[{"data":1,"prerenderedAt":43951},["ShallowReactive",2],{"category-data-学习笔记":3},[4,12666],{"id":5,"title":6,"body":7,"description":12654,"extension":12655,"meta":12656,"navigation":2724,"ogImage":12658,"path":12662,"seo":12663,"stem":12664,"__hash__":12665},"content/blogs/1. transformer复习笔记.md","transformer 复习笔记",{"type":8,"value":9,"toc":12638},"minimark",[10,14,18,35,38,45,48,52,55,315,318,442,758,1190,1196,1467,2386,2389,2618,2621,2861,2868,2871,2874,2879,3211,4558,4575,4801,5260,5263,5358,5361,5948,6309,6665,7022,7295,7298,7301,7634,7791,7794,7797,7989,8128,8131,8643,9042,9046,9049,9661,9665,9668,10668,10671,10674,10965,10968,11125,11136,11730,11733,12440,12443,12621,12624,12629,12634],[11,12,13],"h2",{"id":13},"背景",[15,16,17],"p",{},"距离首次学习 transformer 已经过去一年，最近在回忆 transformer 的相关内容，发现相关内容忘的差不多了，于是决定复习一下并记录下来。",[15,19,20,21,28,29,34],{},"本文主要记录对transformer的理解和思考，代码实现完全参考 ",[22,23,27],"a",{"href":24,"rel":25},"https://zh.d2l.ai/chapter_attention-mechanisms/transformer.html",[26],"nofollow","动手学深度学习"," 部分，部分理论的解释参考 ",[22,30,33],{"href":31,"rel":32},"https://nlp.seas.harvard.edu/annotated-transformer/",[26],"The Annotated Transformer"," 内容。",[11,36,37],{"id":37},"整体框架",[15,39,40],{},[41,42],"img",{"alt":43,"src":44},"transformer架构图","transformer/transformer.png",[11,46,47],{"id":47},"注意力机制",[49,50,51],"h3",{"id":51},"注意力函数",[15,53,54],{},"有两种常见的注意力函数，分别为加性注意力和缩放点积注意力。",[15,56,57,58,314],{},"点积注意力函数与缩放点积注意力函数基本相同，但缩放点积注意力函数有一个额外的缩放因子 ",[59,60,63,102],"span",{"className":61},[62],"katex",[59,64,67],{"className":65},[66],"katex-mathml",[68,69,71],"math",{"xmlns":70},"http://www.w3.org/1998/Math/MathML",[72,73,74,97],"semantics",{},[75,76,77],"mrow",{},[78,79,80,84],"mfrac",{},[81,82,83],"mn",{},"1",[85,86,87],"msqrt",{},[88,89,90,94],"msub",{},[91,92,93],"mi",{},"d",[91,95,96],{},"k",[98,99,101],"annotation",{"encoding":100},"application/x-tex","\\frac{1}{\\sqrt{d_k}}",[59,103,107],{"className":104,"ariaHidden":106},[105],"katex-html","true",[59,108,111,116],{"className":109},[110],"base",[59,112],{"className":113,"style":115},[114],"strut","height:1.3831em;vertical-align:-0.538em;",[59,117,120,125,310],{"className":118},[119],"mord",[59,121],{"className":122},[123,124],"mopen","nulldelimiter",[59,126,128],{"className":127},[78],[59,129,133,301],{"className":130},[131,132],"vlist-t","vlist-t2",[59,134,137,298],{"className":135},[136],"vlist-r",[59,138,142,272,283],{"className":139,"style":141},[140],"vlist","height:0.8451em;",[59,143,145,150],{"style":144},"top:-2.5864em;",[59,146],{"className":147,"style":149},[148],"pstrut","height:3em;",[59,151,157],{"className":152},[153,154,155,156],"sizing","reset-size6","size3","mtight",[59,158,160],{"className":159},[119,156],[59,161,164],{"className":162},[119,163,156],"sqrt",[59,165,167,263],{"className":166},[131,132],[59,168,170,260],{"className":169},[136],[59,171,174,237],{"className":172,"style":173},[140],"height:0.8622em;",[59,175,179,182],{"className":176,"style":178},[177],"svg-align","top:-3em;",[59,180],{"className":181,"style":149},[148],[59,183,186],{"className":184,"style":185},[119,156],"padding-left:0.833em;",[59,187,189,193],{"className":188},[119,156],[59,190,93],{"className":191},[119,192,156],"mathnormal",[59,194,197],{"className":195},[196],"msupsub",[59,198,200,228],{"className":199},[131,132],[59,201,203,223],{"className":202},[136],[59,204,207],{"className":205,"style":206},[140],"height:0.3448em;",[59,208,210,214],{"style":209},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[59,211],{"className":212,"style":213},[148],"height:2.5em;",[59,215,219],{"className":216},[153,217,218,156],"reset-size3","size1",[59,220,96],{"className":221,"style":222},[119,192,156],"margin-right:0.03148em;",[59,224,227],{"className":225},[226],"vlist-s","​",[59,229,231],{"className":230},[136],[59,232,235],{"className":233,"style":234},[140],"height:0.1512em;",[59,236],{},[59,238,240,243],{"style":239},"top:-2.8222em;",[59,241],{"className":242,"style":149},[148],[59,244,248],{"className":245,"style":247},[246,156],"hide-tail","min-width:0.853em;height:1.08em;",[249,250,256],"svg",{"xmlns":251,"width":252,"height":253,"viewBox":254,"preserveAspectRatio":255},"http://www.w3.org/2000/svg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[257,258],"path",{"d":259},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[59,261,227],{"className":262},[226],[59,264,266],{"className":265},[136],[59,267,270],{"className":268,"style":269},[140],"height:0.1778em;",[59,271],{},[59,273,275,278],{"style":274},"top:-3.23em;",[59,276],{"className":277,"style":149},[148],[59,279],{"className":280,"style":282},[281],"frac-line","border-bottom-width:0.04em;",[59,284,286,289],{"style":285},"top:-3.394em;",[59,287],{"className":288,"style":149},[148],[59,290,292],{"className":291},[153,154,155,156],[59,293,295],{"className":294},[119,156],[59,296,83],{"className":297},[119,156],[59,299,227],{"className":300},[226],[59,302,304],{"className":303},[136],[59,305,308],{"className":306,"style":307},[140],"height:0.538em;",[59,309],{},[59,311],{"className":312},[313,124],"mclose","。",[15,316,317],{},"加性注意力通过一个具有单个隐藏层的前馈网络计算兼容性函数。虽然两者在理论复杂度上相似，但在实际应用中，点积注意力更快且更节省空间，因为它可以使用高度优化的矩阵乘法代码实现。",[15,319,320,321,351,352,381,382,411,412,314],{},"在实践中，我们通常从小批量的角度来考虑提高效率，例如基于 ",[59,322,324,338],{"className":323},[62],[59,325,327],{"className":326},[66],[68,328,329],{"xmlns":70},[72,330,331,336],{},[75,332,333],{},[91,334,335],{},"n",[98,337,335],{"encoding":100},[59,339,341],{"className":340,"ariaHidden":106},[105],[59,342,344,348],{"className":343},[110],[59,345],{"className":346,"style":347},[114],"height:0.4306em;",[59,349,335],{"className":350},[119,192]," 个查询和 ",[59,353,355,369],{"className":354},[62],[59,356,358],{"className":357},[66],[68,359,360],{"xmlns":70},[72,361,362,367],{},[75,363,364],{},[91,365,366],{},"m",[98,368,366],{"encoding":100},[59,370,372],{"className":371,"ariaHidden":106},[105],[59,373,375,378],{"className":374},[110],[59,376],{"className":377,"style":347},[114],[59,379,366],{"className":380},[119,192]," 个键-值对计算注意力，其中查询和键的长度为 ",[59,383,385,398],{"className":384},[62],[59,386,388],{"className":387},[66],[68,389,390],{"xmlns":70},[72,391,392,396],{},[75,393,394],{},[91,395,93],{},[98,397,93],{"encoding":100},[59,399,401],{"className":400,"ariaHidden":106},[105],[59,402,404,408],{"className":403},[110],[59,405],{"className":406,"style":407},[114],"height:0.6944em;",[59,409,93],{"className":410},[119,192],"，值的长度为 ",[59,413,415,429],{"className":414},[62],[59,416,418],{"className":417},[66],[68,419,420],{"xmlns":70},[72,421,422,427],{},[75,423,424],{},[91,425,426],{},"v",[98,428,426],{"encoding":100},[59,430,432],{"className":431,"ariaHidden":106},[105],[59,433,435,438],{"className":434},[110],[59,436],{"className":437,"style":347},[114],[59,439,426],{"className":440,"style":441},[119,192],"margin-right:0.03588em;",[15,443,444,445,557,558,657,658,757],{},"查询 ",[59,446,448,483],{"className":447},[62],[59,449,451],{"className":450},[66],[68,452,453],{"xmlns":70},[72,454,455,480],{},[75,456,457,460,464],{},[91,458,459],{},"Q",[461,462,463],"mo",{},"∈",[465,466,467,471],"msup",{},[91,468,470],{"mathvariant":469},"double-struck","R",[75,472,473,475,478],{},[91,474,335],{},[461,476,477],{},"×",[91,479,93],{},[98,481,482],{"encoding":100},"Q \\in \\mathbb{R}^{n \\times d}",[59,484,486,508],{"className":485,"ariaHidden":106},[105],[59,487,489,493,496,501,505],{"className":488},[110],[59,490],{"className":491,"style":492},[114],"height:0.8778em;vertical-align:-0.1944em;",[59,494,459],{"className":495},[119,192],[59,497],{"className":498,"style":500},[499],"mspace","margin-right:0.2778em;",[59,502,463],{"className":503},[504],"mrel",[59,506],{"className":507,"style":500},[499],[59,509,511,515],{"className":510},[110],[59,512],{"className":513,"style":514},[114],"height:0.8491em;",[59,516,518,522],{"className":517},[119],[59,519,470],{"className":520},[119,521],"mathbb",[59,523,525],{"className":524},[196],[59,526,528],{"className":527},[131],[59,529,531],{"className":530},[136],[59,532,534],{"className":533,"style":514},[140],[59,535,537,541],{"style":536},"top:-3.063em;margin-right:0.05em;",[59,538],{"className":539,"style":540},[148],"height:2.7em;",[59,542,544],{"className":543},[153,154,155,156],[59,545,547,550,554],{"className":546},[119,156],[59,548,335],{"className":549},[119,192,156],[59,551,477],{"className":552},[553,156],"mbin",[59,555,93],{"className":556},[119,192,156],"、键 ",[59,559,561,590],{"className":560},[62],[59,562,564],{"className":563},[66],[68,565,566],{"xmlns":70},[72,567,568,587],{},[75,569,570,573,575],{},[91,571,572],{},"K",[461,574,463],{},[465,576,577,579],{},[91,578,470],{"mathvariant":469},[75,580,581,583,585],{},[91,582,366],{},[461,584,477],{},[91,586,93],{},[98,588,589],{"encoding":100},"K \\in \\mathbb{R}^{m \\times d}",[59,591,593,613],{"className":592,"ariaHidden":106},[105],[59,594,596,600,604,607,610],{"className":595},[110],[59,597],{"className":598,"style":599},[114],"height:0.7224em;vertical-align:-0.0391em;",[59,601,572],{"className":602,"style":603},[119,192],"margin-right:0.07153em;",[59,605],{"className":606,"style":500},[499],[59,608,463],{"className":609},[504],[59,611],{"className":612,"style":500},[499],[59,614,616,619],{"className":615},[110],[59,617],{"className":618,"style":514},[114],[59,620,622,625],{"className":621},[119],[59,623,470],{"className":624},[119,521],[59,626,628],{"className":627},[196],[59,629,631],{"className":630},[131],[59,632,634],{"className":633},[136],[59,635,637],{"className":636,"style":514},[140],[59,638,639,642],{"style":536},[59,640],{"className":641,"style":540},[148],[59,643,645],{"className":644},[153,154,155,156],[59,646,648,651,654],{"className":647},[119,156],[59,649,366],{"className":650},[119,192,156],[59,652,477],{"className":653},[553,156],[59,655,93],{"className":656},[119,192,156]," 和值 ",[59,659,661,690],{"className":660},[62],[59,662,664],{"className":663},[66],[68,665,666],{"xmlns":70},[72,667,668,687],{},[75,669,670,673,675],{},[91,671,672],{},"V",[461,674,463],{},[465,676,677,679],{},[91,678,470],{"mathvariant":469},[75,680,681,683,685],{},[91,682,366],{},[461,684,477],{},[91,686,426],{},[98,688,689],{"encoding":100},"V \\in \\mathbb{R}^{m \\times v}",[59,691,693,712],{"className":692,"ariaHidden":106},[105],[59,694,696,699,703,706,709],{"className":695},[110],[59,697],{"className":698,"style":599},[114],[59,700,672],{"className":701,"style":702},[119,192],"margin-right:0.22222em;",[59,704],{"className":705,"style":500},[499],[59,707,463],{"className":708},[504],[59,710],{"className":711,"style":500},[499],[59,713,715,719],{"className":714},[110],[59,716],{"className":717,"style":718},[114],"height:0.7713em;",[59,720,722,725],{"className":721},[119],[59,723,470],{"className":724},[119,521],[59,726,728],{"className":727},[196],[59,729,731],{"className":730},[131],[59,732,734],{"className":733},[136],[59,735,737],{"className":736,"style":718},[140],[59,738,739,742],{"style":536},[59,740],{"className":741,"style":540},[148],[59,743,745],{"className":744},[153,154,155,156],[59,746,748,751,754],{"className":747},[119,156],[59,749,366],{"className":750},[119,192,156],[59,752,477],{"className":753},[553,156],[59,755,426],{"className":756,"style":441},[119,192,156]," 的缩放点积注意力是：",[15,759,760],{},[59,761,763,868],{"className":762},[62],[59,764,766],{"className":765},[66],[68,767,768],{"xmlns":70},[72,769,770,865],{},[75,771,772,775,778,780,783,785,787,790,793,795,799,801,804,806,808,810,813,816,820,823,846,848,850,862],{},[91,773,774],{},"A",[91,776,777],{},"t",[91,779,777],{},[91,781,782],{},"e",[91,784,335],{},[91,786,777],{},[91,788,789],{},"i",[91,791,792],{},"o",[91,794,335],{},[461,796,798],{"stretchy":797},"false","(",[91,800,459],{},[461,802,803],{"separator":106},",",[91,805,572],{},[461,807,803],{"separator":106},[91,809,672],{},[461,811,812],{"stretchy":797},")",[461,814,815],{},"=",[91,817,819],{"mathvariant":818},"normal","softmax",[461,821,822],{},"⁡",[75,824,825,827,844],{},[461,826,798],{"fence":106},[78,828,829,840],{},[75,830,831,833],{},[91,832,459],{},[465,834,835,837],{},[91,836,572],{},[91,838,839],{"mathvariant":818},"⊤",[85,841,842],{},[91,843,93],{},[461,845,812],{"fence":106},[91,847,672],{},[461,849,463],{},[465,851,852,854],{},[91,853,470],{"mathvariant":469},[75,855,856,858,860],{},[91,857,335],{},[461,859,477],{},[91,861,426],{},[91,863,864],{"mathvariant":818},".",[98,866,867],{"encoding":100}," Attention(Q,K,V) = \\operatorname{softmax}\\left(\\frac{QK^\\top}{\\sqrt{d}}\\right)V \\in \\mathbb{R}^{n \\times v}.",[59,869,871,941,1143],{"className":870,"ariaHidden":106},[105],[59,872,874,878,881,885,888,891,894,897,900,903,906,909,913,917,920,923,926,929,932,935,938],{"className":873},[110],[59,875],{"className":876,"style":877},[114],"height:1em;vertical-align:-0.25em;",[59,879,774],{"className":880},[119,192],[59,882,884],{"className":883},[119,192],"tt",[59,886,782],{"className":887},[119,192],[59,889,335],{"className":890},[119,192],[59,892,777],{"className":893},[119,192],[59,895,789],{"className":896},[119,192],[59,898,792],{"className":899},[119,192],[59,901,335],{"className":902},[119,192],[59,904,798],{"className":905},[123],[59,907,459],{"className":908},[119,192],[59,910,803],{"className":911},[912],"mpunct",[59,914],{"className":915,"style":916},[499],"margin-right:0.1667em;",[59,918,572],{"className":919,"style":603},[119,192],[59,921,803],{"className":922},[912],[59,924],{"className":925,"style":916},[499],[59,927,672],{"className":928,"style":702},[119,192],[59,930,812],{"className":931},[313],[59,933],{"className":934,"style":500},[499],[59,936,815],{"className":937},[504],[59,939],{"className":940,"style":500},[499],[59,942,944,948,956,959,1128,1131,1134,1137,1140],{"className":943},[110],[59,945],{"className":946,"style":947},[114],"height:1.8em;vertical-align:-0.65em;",[59,949,952],{"className":950},[951],"mop",[59,953,819],{"className":954},[119,955],"mathrm",[59,957],{"className":958,"style":916},[499],[59,960,963,973,1122],{"className":961},[962],"minner",[59,964,968],{"className":965,"style":967},[123,966],"delimcenter","top:0em;",[59,969,798],{"className":970},[971,972],"delimsizing","size2",[59,974,976,979,1119],{"className":975},[119],[59,977],{"className":978},[123,124],[59,980,982],{"className":981},[78],[59,983,985,1111],{"className":984},[131,132],[59,986,988,1108],{"className":987},[136],[59,989,992,1054,1062],{"className":990,"style":991},[140],"height:1.095em;",[59,993,995,998],{"style":994},"top:-2.5335em;",[59,996],{"className":997,"style":149},[148],[59,999,1001],{"className":1000},[153,154,155,156],[59,1002,1004],{"className":1003},[119,156],[59,1005,1007],{"className":1006},[119,163,156],[59,1008,1010,1045],{"className":1009},[131,132],[59,1011,1013,1042],{"className":1012},[136],[59,1014,1017,1029],{"className":1015,"style":1016},[140],"height:0.9378em;",[59,1018,1020,1023],{"className":1019,"style":178},[177],[59,1021],{"className":1022,"style":149},[148],[59,1024,1026],{"className":1025,"style":185},[119,156],[59,1027,93],{"className":1028},[119,192,156],[59,1030,1032,1035],{"style":1031},"top:-2.8978em;",[59,1033],{"className":1034,"style":149},[148],[59,1036,1038],{"className":1037,"style":247},[246,156],[249,1039,1040],{"xmlns":251,"width":252,"height":253,"viewBox":254,"preserveAspectRatio":255},[257,1041],{"d":259},[59,1043,227],{"className":1044},[226],[59,1046,1048],{"className":1047},[136],[59,1049,1052],{"className":1050,"style":1051},[140],"height:0.1022em;",[59,1053],{},[59,1055,1056,1059],{"style":274},[59,1057],{"className":1058,"style":149},[148],[59,1060],{"className":1061,"style":282},[281],[59,1063,1065,1068],{"style":1064},"top:-3.4461em;",[59,1066],{"className":1067,"style":149},[148],[59,1069,1071],{"className":1070},[153,154,155,156],[59,1072,1074,1077],{"className":1073},[119,156],[59,1075,459],{"className":1076},[119,192,156],[59,1078,1080,1083],{"className":1079},[119,156],[59,1081,572],{"className":1082,"style":603},[119,192,156],[59,1084,1086],{"className":1085},[196],[59,1087,1089],{"className":1088},[131],[59,1090,1092],{"className":1091},[136],[59,1093,1096],{"className":1094,"style":1095},[140],"height:0.927em;",[59,1097,1099,1102],{"style":1098},"top:-2.931em;margin-right:0.0714em;",[59,1100],{"className":1101,"style":213},[148],[59,1103,1105],{"className":1104},[153,217,218,156],[59,1106,839],{"className":1107},[119,156],[59,1109,227],{"className":1110},[226],[59,1112,1114],{"className":1113},[136],[59,1115,1117],{"className":1116,"style":307},[140],[59,1118],{},[59,1120],{"className":1121},[313,124],[59,1123,1125],{"className":1124,"style":967},[313,966],[59,1126,812],{"className":1127},[971,972],[59,1129],{"className":1130,"style":916},[499],[59,1132,672],{"className":1133,"style":702},[119,192],[59,1135],{"className":1136,"style":500},[499],[59,1138,463],{"className":1139},[504],[59,1141],{"className":1142,"style":500},[499],[59,1144,1146,1149,1187],{"className":1145},[110],[59,1147],{"className":1148,"style":718},[114],[59,1150,1152,1155],{"className":1151},[119],[59,1153,470],{"className":1154},[119,521],[59,1156,1158],{"className":1157},[196],[59,1159,1161],{"className":1160},[131],[59,1162,1164],{"className":1163},[136],[59,1165,1167],{"className":1166,"style":718},[140],[59,1168,1169,1172],{"style":536},[59,1170],{"className":1171,"style":540},[148],[59,1173,1175],{"className":1174},[153,154,155,156],[59,1176,1178,1181,1184],{"className":1177},[119,156],[59,1179,335],{"className":1180},[119,192,156],[59,1182,477],{"className":1183},[553,156],[59,1185,426],{"className":1186,"style":441},[119,192,156],[59,1188,864],{"className":1189},[119],[15,1191,1192],{},[41,1193],{"alt":1194,"src":1195},"注意力","transformer/attention.png",[15,1197,1198,1199,1273,1274,1345,1346,1466],{},"缩放点积注意力的输入由维度为 ",[59,1200,1202,1220],{"className":1201},[62],[59,1203,1205],{"className":1204},[66],[68,1206,1207],{"xmlns":70},[72,1208,1209,1217],{},[75,1210,1211],{},[88,1212,1213,1215],{},[91,1214,93],{},[91,1216,96],{},[98,1218,1219],{"encoding":100},"d_k",[59,1221,1223],{"className":1222,"ariaHidden":106},[105],[59,1224,1226,1230],{"className":1225},[110],[59,1227],{"className":1228,"style":1229},[114],"height:0.8444em;vertical-align:-0.15em;",[59,1231,1233,1236],{"className":1232},[119],[59,1234,93],{"className":1235},[119,192],[59,1237,1239],{"className":1238},[196],[59,1240,1242,1264],{"className":1241},[131,132],[59,1243,1245,1261],{"className":1244},[136],[59,1246,1249],{"className":1247,"style":1248},[140],"height:0.3361em;",[59,1250,1252,1255],{"style":1251},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[59,1253],{"className":1254,"style":540},[148],[59,1256,1258],{"className":1257},[153,154,155,156],[59,1259,96],{"className":1260,"style":222},[119,192,156],[59,1262,227],{"className":1263},[226],[59,1265,1267],{"className":1266},[136],[59,1268,1271],{"className":1269,"style":1270},[140],"height:0.15em;",[59,1272],{}," ​的查询和键以及维度为 ",[59,1275,1277,1295],{"className":1276},[62],[59,1278,1280],{"className":1279},[66],[68,1281,1282],{"xmlns":70},[72,1283,1284,1292],{},[75,1285,1286],{},[88,1287,1288,1290],{},[91,1289,93],{},[91,1291,426],{},[98,1293,1294],{"encoding":100},"d_v",[59,1296,1298],{"className":1297,"ariaHidden":106},[105],[59,1299,1301,1304],{"className":1300},[110],[59,1302],{"className":1303,"style":1229},[114],[59,1305,1307,1310],{"className":1306},[119],[59,1308,93],{"className":1309},[119,192],[59,1311,1313],{"className":1312},[196],[59,1314,1316,1337],{"className":1315},[131,132],[59,1317,1319,1334],{"className":1318},[136],[59,1320,1323],{"className":1321,"style":1322},[140],"height:0.1514em;",[59,1324,1325,1328],{"style":1251},[59,1326],{"className":1327,"style":540},[148],[59,1329,1331],{"className":1330},[153,154,155,156],[59,1332,426],{"className":1333,"style":441},[119,192,156],[59,1335,227],{"className":1336},[226],[59,1338,1340],{"className":1339},[136],[59,1341,1343],{"className":1342,"style":1270},[140],[59,1344],{}," ​的值组成。我们计算查询与所有键的点积，将每个点积除以 ",[59,1347,1349,1369],{"className":1348},[62],[59,1350,1352],{"className":1351},[66],[68,1353,1354],{"xmlns":70},[72,1355,1356,1366],{},[75,1357,1358],{},[85,1359,1360],{},[88,1361,1362,1364],{},[91,1363,93],{},[91,1365,96],{},[98,1367,1368],{"encoding":100},"\\sqrt{d_k}",[59,1370,1372],{"className":1371,"ariaHidden":106},[105],[59,1373,1375,1379],{"className":1374},[110],[59,1376],{"className":1377,"style":1378},[114],"height:1.04em;vertical-align:-0.1828em;",[59,1380,1382],{"className":1381},[119,163],[59,1383,1385,1457],{"className":1384},[131,132],[59,1386,1388,1454],{"className":1387},[136],[59,1389,1392,1441],{"className":1390,"style":1391},[140],"height:0.8572em;",[59,1393,1395,1398],{"className":1394,"style":178},[177],[59,1396],{"className":1397,"style":149},[148],[59,1399,1401],{"className":1400,"style":185},[119],[59,1402,1404,1407],{"className":1403},[119],[59,1405,93],{"className":1406},[119,192],[59,1408,1410],{"className":1409},[196],[59,1411,1413,1433],{"className":1412},[131,132],[59,1414,1416,1430],{"className":1415},[136],[59,1417,1419],{"className":1418,"style":1248},[140],[59,1420,1421,1424],{"style":1251},[59,1422],{"className":1423,"style":540},[148],[59,1425,1427],{"className":1426},[153,154,155,156],[59,1428,96],{"className":1429,"style":222},[119,192,156],[59,1431,227],{"className":1432},[226],[59,1434,1436],{"className":1435},[136],[59,1437,1439],{"className":1438,"style":1270},[140],[59,1440],{},[59,1442,1444,1447],{"style":1443},"top:-2.8172em;",[59,1445],{"className":1446,"style":149},[148],[59,1448,1450],{"className":1449,"style":247},[246],[249,1451,1452],{"xmlns":251,"width":252,"height":253,"viewBox":254,"preserveAspectRatio":255},[257,1453],{"d":259},[59,1455,227],{"className":1456},[226],[59,1458,1460],{"className":1459},[136],[59,1461,1464],{"className":1462,"style":1463},[140],"height:0.1828em;",[59,1465],{},"​​，并应用softmax函数以获得值上的权重。",[15,1468,1469,1470,1539,1540,1609,1610,1679,1680,1710,1711,1739,1740,1770,1771,1799,1800,2103,2104,1770,2132,2201,2202,314],{},"对于较小的 ",[59,1471,1473,1490],{"className":1472},[62],[59,1474,1476],{"className":1475},[66],[68,1477,1478],{"xmlns":70},[72,1479,1480,1488],{},[75,1481,1482],{},[88,1483,1484,1486],{},[91,1485,93],{},[91,1487,96],{},[98,1489,1219],{"encoding":100},[59,1491,1493],{"className":1492,"ariaHidden":106},[105],[59,1494,1496,1499],{"className":1495},[110],[59,1497],{"className":1498,"style":1229},[114],[59,1500,1502,1505],{"className":1501},[119],[59,1503,93],{"className":1504},[119,192],[59,1506,1508],{"className":1507},[196],[59,1509,1511,1531],{"className":1510},[131,132],[59,1512,1514,1528],{"className":1513},[136],[59,1515,1517],{"className":1516,"style":1248},[140],[59,1518,1519,1522],{"style":1251},[59,1520],{"className":1521,"style":540},[148],[59,1523,1525],{"className":1524},[153,154,155,156],[59,1526,96],{"className":1527,"style":222},[119,192,156],[59,1529,227],{"className":1530},[226],[59,1532,1534],{"className":1533},[136],[59,1535,1537],{"className":1536,"style":1270},[140],[59,1538],{}," 值来说，加性注意力和点积注意力的表现相似，但加性注意力在 ",[59,1541,1543,1560],{"className":1542},[62],[59,1544,1546],{"className":1545},[66],[68,1547,1548],{"xmlns":70},[72,1549,1550,1558],{},[75,1551,1552],{},[88,1553,1554,1556],{},[91,1555,93],{},[91,1557,96],{},[98,1559,1219],{"encoding":100},[59,1561,1563],{"className":1562,"ariaHidden":106},[105],[59,1564,1566,1569],{"className":1565},[110],[59,1567],{"className":1568,"style":1229},[114],[59,1570,1572,1575],{"className":1571},[119],[59,1573,93],{"className":1574},[119,192],[59,1576,1578],{"className":1577},[196],[59,1579,1581,1601],{"className":1580},[131,132],[59,1582,1584,1598],{"className":1583},[136],[59,1585,1587],{"className":1586,"style":1248},[140],[59,1588,1589,1592],{"style":1251},[59,1590],{"className":1591,"style":540},[148],[59,1593,1595],{"className":1594},[153,154,155,156],[59,1596,96],{"className":1597,"style":222},[119,192,156],[59,1599,227],{"className":1600},[226],[59,1602,1604],{"className":1603},[136],[59,1605,1607],{"className":1606,"style":1270},[140],[59,1608],{}," 值较大时会优于点积注意力。我们怀疑在 ",[59,1611,1613,1630],{"className":1612},[62],[59,1614,1616],{"className":1615},[66],[68,1617,1618],{"xmlns":70},[72,1619,1620,1628],{},[75,1621,1622],{},[88,1623,1624,1626],{},[91,1625,93],{},[91,1627,96],{},[98,1629,1219],{"encoding":100},[59,1631,1633],{"className":1632,"ariaHidden":106},[105],[59,1634,1636,1639],{"className":1635},[110],[59,1637],{"className":1638,"style":1229},[114],[59,1640,1642,1645],{"className":1641},[119],[59,1643,93],{"className":1644},[119,192],[59,1646,1648],{"className":1647},[196],[59,1649,1651,1671],{"className":1650},[131,132],[59,1652,1654,1668],{"className":1653},[136],[59,1655,1657],{"className":1656,"style":1248},[140],[59,1658,1659,1662],{"style":1251},[59,1660],{"className":1661,"style":540},[148],[59,1663,1665],{"className":1664},[153,154,155,156],[59,1666,96],{"className":1667,"style":222},[119,192,156],[59,1669,227],{"className":1670},[226],[59,1672,1674],{"className":1673},[136],[59,1675,1677],{"className":1676,"style":1270},[140],[59,1678],{}," 值较大时，点积的值会很大，导致 softmax 集中在很小的一个区域内。为了更好地说明这一点，我们假设查询 ",[59,1681,1683,1697],{"className":1682},[62],[59,1684,1686],{"className":1685},[66],[68,1687,1688],{"xmlns":70},[72,1689,1690,1695],{},[75,1691,1692],{},[91,1693,1694],{},"q",[98,1696,1694],{"encoding":100},[59,1698,1700],{"className":1699,"ariaHidden":106},[105],[59,1701,1703,1707],{"className":1702},[110],[59,1704],{"className":1705,"style":1706},[114],"height:0.625em;vertical-align:-0.1944em;",[59,1708,1694],{"className":1709,"style":441},[119,192]," 和键 ",[59,1712,1714,1727],{"className":1713},[62],[59,1715,1717],{"className":1716},[66],[68,1718,1719],{"xmlns":70},[72,1720,1721,1725],{},[75,1722,1723],{},[91,1724,96],{},[98,1726,96],{"encoding":100},[59,1728,1730],{"className":1729,"ariaHidden":106},[105],[59,1731,1733,1736],{"className":1732},[110],[59,1734],{"className":1735,"style":407},[114],[59,1737,96],{"className":1738,"style":222},[119,192]," 都符合一个均值为 ",[59,1741,1743,1757],{"className":1742},[62],[59,1744,1746],{"className":1745},[66],[68,1747,1748],{"xmlns":70},[72,1749,1750,1755],{},[75,1751,1752],{},[81,1753,1754],{},"0",[98,1756,1754],{"encoding":100},[59,1758,1760],{"className":1759,"ariaHidden":106},[105],[59,1761,1763,1767],{"className":1762},[110],[59,1764],{"className":1765,"style":1766},[114],"height:0.6444em;",[59,1768,1754],{"className":1769},[119],"，方差为 ",[59,1772,1774,1787],{"className":1773},[62],[59,1775,1777],{"className":1776},[66],[68,1778,1779],{"xmlns":70},[72,1780,1781,1785],{},[75,1782,1783],{},[81,1784,83],{},[98,1786,83],{"encoding":100},[59,1788,1790],{"className":1789,"ariaHidden":106},[105],[59,1791,1793,1796],{"className":1792},[110],[59,1794],{"className":1795,"style":1766},[114],[59,1797,83],{"className":1798},[119]," 的随机变量。他们点积的结果 ",[59,1801,1803,1856],{"className":1802},[62],[59,1804,1806],{"className":1805},[66],[68,1807,1808],{"xmlns":70},[72,1809,1810,1853],{},[75,1811,1812,1814,1817,1819,1821,1841,1847],{},[91,1813,1694],{},[461,1815,1816],{},"⋅",[91,1818,96],{},[461,1820,815],{},[1822,1823,1824,1827,1835],"msubsup",{},[461,1825,1826],{},"∑",[75,1828,1829,1831,1833],{},[91,1830,789],{},[461,1832,815],{},[81,1834,83],{},[88,1836,1837,1839],{},[91,1838,93],{},[91,1840,96],{},[88,1842,1843,1845],{},[91,1844,1694],{},[91,1846,789],{},[88,1848,1849,1851],{},[91,1850,96],{},[91,1852,789],{},[98,1854,1855],{"encoding":100},"q \\cdot k = \\sum_{i=1}^{d_k}q_{i}k_{i}",[59,1857,1859,1879,1897],{"className":1858,"ariaHidden":106},[105],[59,1860,1862,1866,1869,1873,1876],{"className":1861},[110],[59,1863],{"className":1864,"style":1865},[114],"height:0.6389em;vertical-align:-0.1944em;",[59,1867,1694],{"className":1868,"style":441},[119,192],[59,1870],{"className":1871,"style":1872},[499],"margin-right:0.2222em;",[59,1874,1816],{"className":1875},[553],[59,1877],{"className":1878,"style":1872},[499],[59,1880,1882,1885,1888,1891,1894],{"className":1881},[110],[59,1883],{"className":1884,"style":407},[114],[59,1886,96],{"className":1887,"style":222},[119,192],[59,1889],{"className":1890,"style":500},[499],[59,1892,815],{"className":1893},[504],[59,1895],{"className":1896,"style":500},[499],[59,1898,1900,1904,2011,2014,2059],{"className":1899},[110],[59,1901],{"className":1902,"style":1903},[114],"height:1.2887em;vertical-align:-0.2997em;",[59,1905,1907,1913],{"className":1906},[951],[59,1908,1826],{"className":1909,"style":1912},[951,1910,1911],"op-symbol","small-op","position:relative;top:0em;",[59,1914,1916],{"className":1915},[196],[59,1917,1919,2002],{"className":1918},[131,132],[59,1920,1922,1999],{"className":1921},[136],[59,1923,1926,1947],{"className":1924,"style":1925},[140],"height:0.989em;",[59,1927,1929,1932],{"style":1928},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[59,1930],{"className":1931,"style":540},[148],[59,1933,1935],{"className":1934},[153,154,155,156],[59,1936,1938,1941,1944],{"className":1937},[119,156],[59,1939,789],{"className":1940},[119,192,156],[59,1942,815],{"className":1943},[504,156],[59,1945,83],{"className":1946},[119,156],[59,1948,1950,1953],{"style":1949},"top:-3.2029em;margin-right:0.05em;",[59,1951],{"className":1952,"style":540},[148],[59,1954,1956],{"className":1955},[153,154,155,156],[59,1957,1959],{"className":1958},[119,156],[59,1960,1962,1965],{"className":1961},[119,156],[59,1963,93],{"className":1964},[119,192,156],[59,1966,1968],{"className":1967},[196],[59,1969,1971,1991],{"className":1970},[131,132],[59,1972,1974,1988],{"className":1973},[136],[59,1975,1977],{"className":1976,"style":206},[140],[59,1978,1979,1982],{"style":209},[59,1980],{"className":1981,"style":213},[148],[59,1983,1985],{"className":1984},[153,217,218,156],[59,1986,96],{"className":1987,"style":222},[119,192,156],[59,1989,227],{"className":1990},[226],[59,1992,1994],{"className":1993},[136],[59,1995,1997],{"className":1996,"style":234},[140],[59,1998],{},[59,2000,227],{"className":2001},[226],[59,2003,2005],{"className":2004},[136],[59,2006,2009],{"className":2007,"style":2008},[140],"height:0.2997em;",[59,2010],{},[59,2012],{"className":2013,"style":916},[499],[59,2015,2017,2020],{"className":2016},[119],[59,2018,1694],{"className":2019,"style":441},[119,192],[59,2021,2023],{"className":2022},[196],[59,2024,2026,2051],{"className":2025},[131,132],[59,2027,2029,2048],{"className":2028},[136],[59,2030,2033],{"className":2031,"style":2032},[140],"height:0.3117em;",[59,2034,2036,2039],{"style":2035},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[59,2037],{"className":2038,"style":540},[148],[59,2040,2042],{"className":2041},[153,154,155,156],[59,2043,2045],{"className":2044},[119,156],[59,2046,789],{"className":2047},[119,192,156],[59,2049,227],{"className":2050},[226],[59,2052,2054],{"className":2053},[136],[59,2055,2057],{"className":2056,"style":1270},[140],[59,2058],{},[59,2060,2062,2065],{"className":2061},[119],[59,2063,96],{"className":2064,"style":222},[119,192],[59,2066,2068],{"className":2067},[196],[59,2069,2071,2095],{"className":2070},[131,132],[59,2072,2074,2092],{"className":2073},[136],[59,2075,2077],{"className":2076,"style":2032},[140],[59,2078,2080,2083],{"style":2079},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[59,2081],{"className":2082,"style":540},[148],[59,2084,2086],{"className":2085},[153,154,155,156],[59,2087,2089],{"className":2088},[119,156],[59,2090,789],{"className":2091},[119,192,156],[59,2093,227],{"className":2094},[226],[59,2096,2098],{"className":2097},[136],[59,2099,2101],{"className":2100,"style":1270},[140],[59,2102],{}," 的均值为 ",[59,2105,2107,2120],{"className":2106},[62],[59,2108,2110],{"className":2109},[66],[68,2111,2112],{"xmlns":70},[72,2113,2114,2118],{},[75,2115,2116],{},[81,2117,1754],{},[98,2119,1754],{"encoding":100},[59,2121,2123],{"className":2122,"ariaHidden":106},[105],[59,2124,2126,2129],{"className":2125},[110],[59,2127],{"className":2128,"style":1766},[114],[59,2130,1754],{"className":2131},[119],[59,2133,2135,2152],{"className":2134},[62],[59,2136,2138],{"className":2137},[66],[68,2139,2140],{"xmlns":70},[72,2141,2142,2150],{},[75,2143,2144],{},[88,2145,2146,2148],{},[91,2147,93],{},[91,2149,96],{},[98,2151,1219],{"encoding":100},[59,2153,2155],{"className":2154,"ariaHidden":106},[105],[59,2156,2158,2161],{"className":2157},[110],[59,2159],{"className":2160,"style":1229},[114],[59,2162,2164,2167],{"className":2163},[119],[59,2165,93],{"className":2166},[119,192],[59,2168,2170],{"className":2169},[196],[59,2171,2173,2193],{"className":2172},[131,132],[59,2174,2176,2190],{"className":2175},[136],[59,2177,2179],{"className":2178,"style":1248},[140],[59,2180,2181,2184],{"style":1251},[59,2182],{"className":2183,"style":540},[148],[59,2185,2187],{"className":2186},[153,154,155,156],[59,2188,96],{"className":2189,"style":222},[119,192,156],[59,2191,227],{"className":2192},[226],[59,2194,2196],{"className":2195},[136],[59,2197,2199],{"className":2198,"style":1270},[140],[59,2200],{},"，为了抵消这种影响，我们要乘以一个缩放因子 ",[59,2203,2205,2228],{"className":2204},[62],[59,2206,2208],{"className":2207},[66],[68,2209,2210],{"xmlns":70},[72,2211,2212,2226],{},[75,2213,2214],{},[78,2215,2216,2218],{},[81,2217,83],{},[85,2219,2220],{},[88,2221,2222,2224],{},[91,2223,93],{},[91,2225,96],{},[98,2227,101],{"encoding":100},[59,2229,2231],{"className":2230,"ariaHidden":106},[105],[59,2232,2234,2237],{"className":2233},[110],[59,2235],{"className":2236,"style":115},[114],[59,2238,2240,2243,2383],{"className":2239},[119],[59,2241],{"className":2242},[123,124],[59,2244,2246],{"className":2245},[78],[59,2247,2249,2375],{"className":2248},[131,132],[59,2250,2252,2372],{"className":2251},[136],[59,2253,2255,2350,2358],{"className":2254,"style":141},[140],[59,2256,2257,2260],{"style":144},[59,2258],{"className":2259,"style":149},[148],[59,2261,2263],{"className":2262},[153,154,155,156],[59,2264,2266],{"className":2265},[119,156],[59,2267,2269],{"className":2268},[119,163,156],[59,2270,2272,2342],{"className":2271},[131,132],[59,2273,2275,2339],{"className":2274},[136],[59,2276,2278,2327],{"className":2277,"style":173},[140],[59,2279,2281,2284],{"className":2280,"style":178},[177],[59,2282],{"className":2283,"style":149},[148],[59,2285,2287],{"className":2286,"style":185},[119,156],[59,2288,2290,2293],{"className":2289},[119,156],[59,2291,93],{"className":2292},[119,192,156],[59,2294,2296],{"className":2295},[196],[59,2297,2299,2319],{"className":2298},[131,132],[59,2300,2302,2316],{"className":2301},[136],[59,2303,2305],{"className":2304,"style":206},[140],[59,2306,2307,2310],{"style":209},[59,2308],{"className":2309,"style":213},[148],[59,2311,2313],{"className":2312},[153,217,218,156],[59,2314,96],{"className":2315,"style":222},[119,192,156],[59,2317,227],{"className":2318},[226],[59,2320,2322],{"className":2321},[136],[59,2323,2325],{"className":2324,"style":234},[140],[59,2326],{},[59,2328,2329,2332],{"style":239},[59,2330],{"className":2331,"style":149},[148],[59,2333,2335],{"className":2334,"style":247},[246,156],[249,2336,2337],{"xmlns":251,"width":252,"height":253,"viewBox":254,"preserveAspectRatio":255},[257,2338],{"d":259},[59,2340,227],{"className":2341},[226],[59,2343,2345],{"className":2344},[136],[59,2346,2348],{"className":2347,"style":269},[140],[59,2349],{},[59,2351,2352,2355],{"style":274},[59,2353],{"className":2354,"style":149},[148],[59,2356],{"className":2357,"style":282},[281],[59,2359,2360,2363],{"style":285},[59,2361],{"className":2362,"style":149},[148],[59,2364,2366],{"className":2365},[153,154,155,156],[59,2367,2369],{"className":2368},[119,156],[59,2370,83],{"className":2371},[119,156],[59,2373,227],{"className":2374},[226],[59,2376,2378],{"className":2377},[136],[59,2379,2381],{"className":2380,"style":307},[140],[59,2382],{},[59,2384],{"className":2385},[313,124],[15,2387,2388],{},"因为在训练和推理时，并非所有的值都应该加入注意力汇聚操作中，所以我们首先实现 masked_softmax 函数来保证在注意力汇聚时仅加入有意义的值。",[2390,2391,2396],"pre",{"className":2392,"code":2393,"language":2394,"meta":2395,"style":2395},"language-python shiki shiki-themes github-light dracula","def masked_softmax(X, valid_lens):\n    \"\"\"通过在最后一个轴上掩蔽元素来执行softmax操作\"\"\"\n    # X:3D张量，valid_lens:1D或2D张量\n    if valid_lens is None:\n        return nn.functional.softmax(X, dim=-1)\n    else:\n        shape = X.shape\n        if valid_lens.dim() == 1:\n            valid_lens = torch.repeat_interleave(valid_lens, shape[1])\n        else:\n            valid_lens = valid_lens.reshape(-1)\n        # 最后一轴上被掩蔽的元素使用一个非常大的负值替换，从而其softmax输出为0\n        X = d2l.sequence_mask(X.reshape(-1, shape[-1]), valid_lens,\n                              value=-1e6)\n        return nn.functional.softmax(X.reshape(shape), dim=-1)\n","python","",[2397,2398,2399,2427,2434,2441,2460,2481,2489,2500,2517,2533,2541,2558,2564,2589,2602],"code",{"__ignoreMap":2395},[59,2400,2403,2407,2411,2414,2418,2421,2424],{"class":2401,"line":2402},"line",1,[59,2404,2406],{"class":2405},"szJfE","def",[59,2408,2410],{"class":2409},"sCLZk"," masked_softmax",[59,2412,798],{"class":2413},"scbbO",[59,2415,2417],{"class":2416},"syNf4","X",[59,2419,2420],{"class":2413},", ",[59,2422,2423],{"class":2416},"valid_lens",[59,2425,2426],{"class":2413},"):\n",[59,2428,2430],{"class":2401,"line":2429},2,[59,2431,2433],{"class":2432},"seLWX","    \"\"\"通过在最后一个轴上掩蔽元素来执行softmax操作\"\"\"\n",[59,2435,2437],{"class":2401,"line":2436},3,[59,2438,2440],{"class":2439},"sfgPZ","    # X:3D张量，valid_lens:1D或2D张量\n",[59,2442,2444,2447,2450,2453,2457],{"class":2401,"line":2443},4,[59,2445,2446],{"class":2405},"    if",[59,2448,2449],{"class":2413}," valid_lens ",[59,2451,2452],{"class":2405},"is",[59,2454,2456],{"class":2455},"soDru"," None",[59,2458,2459],{"class":2413},":\n",[59,2461,2463,2466,2469,2473,2476,2478],{"class":2401,"line":2462},5,[59,2464,2465],{"class":2405},"        return",[59,2467,2468],{"class":2413}," nn.functional.softmax(X, ",[59,2470,2472],{"class":2471},"sQkXh","dim",[59,2474,2475],{"class":2405},"=-",[59,2477,83],{"class":2455},[59,2479,2480],{"class":2413},")\n",[59,2482,2484,2487],{"class":2401,"line":2483},6,[59,2485,2486],{"class":2405},"    else",[59,2488,2459],{"class":2413},[59,2490,2492,2495,2497],{"class":2401,"line":2491},7,[59,2493,2494],{"class":2413},"        shape ",[59,2496,815],{"class":2405},[59,2498,2499],{"class":2413}," X.shape\n",[59,2501,2503,2506,2509,2512,2515],{"class":2401,"line":2502},8,[59,2504,2505],{"class":2405},"        if",[59,2507,2508],{"class":2413}," valid_lens.dim() ",[59,2510,2511],{"class":2405},"==",[59,2513,2514],{"class":2455}," 1",[59,2516,2459],{"class":2413},[59,2518,2520,2523,2525,2528,2530],{"class":2401,"line":2519},9,[59,2521,2522],{"class":2413},"            valid_lens ",[59,2524,815],{"class":2405},[59,2526,2527],{"class":2413}," torch.repeat_interleave(valid_lens, shape[",[59,2529,83],{"class":2455},[59,2531,2532],{"class":2413},"])\n",[59,2534,2536,2539],{"class":2401,"line":2535},10,[59,2537,2538],{"class":2405},"        else",[59,2540,2459],{"class":2413},[59,2542,2544,2546,2548,2551,2554,2556],{"class":2401,"line":2543},11,[59,2545,2522],{"class":2413},[59,2547,815],{"class":2405},[59,2549,2550],{"class":2413}," valid_lens.reshape(",[59,2552,2553],{"class":2405},"-",[59,2555,83],{"class":2455},[59,2557,2480],{"class":2413},[59,2559,2561],{"class":2401,"line":2560},12,[59,2562,2563],{"class":2439},"        # 最后一轴上被掩蔽的元素使用一个非常大的负值替换，从而其softmax输出为0\n",[59,2565,2567,2570,2572,2575,2577,2579,2582,2584,2586],{"class":2401,"line":2566},13,[59,2568,2569],{"class":2413},"        X ",[59,2571,815],{"class":2405},[59,2573,2574],{"class":2413}," d2l.sequence_mask(X.reshape(",[59,2576,2553],{"class":2405},[59,2578,83],{"class":2455},[59,2580,2581],{"class":2413},", shape[",[59,2583,2553],{"class":2405},[59,2585,83],{"class":2455},[59,2587,2588],{"class":2413},"]), valid_lens,\n",[59,2590,2592,2595,2597,2600],{"class":2401,"line":2591},14,[59,2593,2594],{"class":2471},"                              value",[59,2596,2475],{"class":2405},[59,2598,2599],{"class":2455},"1e6",[59,2601,2480],{"class":2413},[59,2603,2605,2607,2610,2612,2614,2616],{"class":2401,"line":2604},15,[59,2606,2465],{"class":2405},[59,2608,2609],{"class":2413}," nn.functional.softmax(X.reshape(shape), ",[59,2611,2472],{"class":2471},[59,2613,2475],{"class":2405},[59,2615,83],{"class":2455},[59,2617,2480],{"class":2413},[15,2619,2620],{},"下面实现缩放点积注意力。",[2390,2622,2624],{"className":2392,"code":2623,"language":2394,"meta":2395,"style":2395},"class DotProductAttention(nn.Module):\n    \"\"\"缩放点积注意力\"\"\"\n    def __init__(self, dropout, **kwargs):\n        super(DotProductAttention, self).__init__(**kwargs)\n        self.dropout = nn.Dropout(dropout)\n\n    # queries的形状：(batch_size，查询的个数，d)\n    # keys的形状：(batch_size，“键－值”对的个数，d)\n    # values的形状：(batch_size，“键－值”对的个数，值的维度)\n    # valid_lens的形状:(batch_size，)或者(batch_size，查询的个数)\n    def forward(self, queries, keys, values, valid_lens=None):\n        d = queries.shape[-1]\n        # 设置transpose_b=True为了交换keys的最后两个维度\n        scores = torch.bmm(queries, keys.transpose(1,2)) / math.sqrt(d)\n        self.attention_weights = masked_softmax(scores, valid_lens)\n        return torch.bmm(self.dropout(self.attention_weights), values)\n",[2397,2625,2626,2648,2653,2682,2707,2720,2726,2731,2736,2741,2746,2783,2800,2805,2831,2843],{"__ignoreMap":2395},[59,2627,2628,2631,2635,2637,2641,2643,2646],{"class":2401,"line":2402},[59,2629,2630],{"class":2405},"class",[59,2632,2634],{"class":2633},"skCyd"," DotProductAttention",[59,2636,798],{"class":2413},[59,2638,2640],{"class":2639},"sDP9b","nn",[59,2642,864],{"class":2413},[59,2644,2645],{"class":2639},"Module",[59,2647,2426],{"class":2413},[59,2649,2650],{"class":2401,"line":2429},[59,2651,2652],{"class":2432},"    \"\"\"缩放点积注意力\"\"\"\n",[59,2654,2655,2658,2661,2663,2667,2669,2672,2674,2677,2680],{"class":2401,"line":2436},[59,2656,2657],{"class":2405},"    def",[59,2659,2660],{"class":2455}," __init__",[59,2662,798],{"class":2413},[59,2664,2666],{"class":2665},"sD3jg","self",[59,2668,2420],{"class":2413},[59,2670,2671],{"class":2416},"dropout",[59,2673,2420],{"class":2413},[59,2675,2676],{"class":2405},"**",[59,2678,2679],{"class":2416},"kwargs",[59,2681,2426],{"class":2413},[59,2683,2684,2688,2691,2694,2697,2700,2702,2704],{"class":2401,"line":2443},[59,2685,2687],{"class":2686},"sPGBF","        super",[59,2689,2690],{"class":2413},"(DotProductAttention, ",[59,2692,2666],{"class":2693},"sJti5",[59,2695,2696],{"class":2413},").",[59,2698,2699],{"class":2455},"__init__",[59,2701,798],{"class":2413},[59,2703,2676],{"class":2405},[59,2705,2706],{"class":2413},"kwargs)\n",[59,2708,2709,2712,2715,2717],{"class":2401,"line":2462},[59,2710,2711],{"class":2693},"        self",[59,2713,2714],{"class":2413},".dropout ",[59,2716,815],{"class":2405},[59,2718,2719],{"class":2413}," nn.Dropout(dropout)\n",[59,2721,2722],{"class":2401,"line":2483},[59,2723,2725],{"emptyLinePlaceholder":2724},true,"\n",[59,2727,2728],{"class":2401,"line":2491},[59,2729,2730],{"class":2439},"    # queries的形状：(batch_size，查询的个数，d)\n",[59,2732,2733],{"class":2401,"line":2502},[59,2734,2735],{"class":2439},"    # keys的形状：(batch_size，“键－值”对的个数，d)\n",[59,2737,2738],{"class":2401,"line":2519},[59,2739,2740],{"class":2439},"    # values的形状：(batch_size，“键－值”对的个数，值的维度)\n",[59,2742,2743],{"class":2401,"line":2535},[59,2744,2745],{"class":2439},"    # valid_lens的形状:(batch_size，)或者(batch_size，查询的个数)\n",[59,2747,2748,2750,2753,2755,2757,2759,2762,2764,2767,2769,2772,2774,2776,2778,2781],{"class":2401,"line":2543},[59,2749,2657],{"class":2405},[59,2751,2752],{"class":2409}," forward",[59,2754,798],{"class":2413},[59,2756,2666],{"class":2665},[59,2758,2420],{"class":2413},[59,2760,2761],{"class":2416},"queries",[59,2763,2420],{"class":2413},[59,2765,2766],{"class":2416},"keys",[59,2768,2420],{"class":2413},[59,2770,2771],{"class":2416},"values",[59,2773,2420],{"class":2413},[59,2775,2423],{"class":2416},[59,2777,815],{"class":2405},[59,2779,2780],{"class":2455},"None",[59,2782,2426],{"class":2413},[59,2784,2785,2788,2790,2793,2795,2797],{"class":2401,"line":2560},[59,2786,2787],{"class":2413},"        d ",[59,2789,815],{"class":2405},[59,2791,2792],{"class":2413}," queries.shape[",[59,2794,2553],{"class":2405},[59,2796,83],{"class":2455},[59,2798,2799],{"class":2413},"]\n",[59,2801,2802],{"class":2401,"line":2566},[59,2803,2804],{"class":2439},"        # 设置transpose_b=True为了交换keys的最后两个维度\n",[59,2806,2807,2810,2812,2815,2817,2819,2822,2825,2828],{"class":2401,"line":2591},[59,2808,2809],{"class":2413},"        scores ",[59,2811,815],{"class":2405},[59,2813,2814],{"class":2413}," torch.bmm(queries, keys.transpose(",[59,2816,83],{"class":2455},[59,2818,803],{"class":2413},[59,2820,2821],{"class":2455},"2",[59,2823,2824],{"class":2413},")) ",[59,2826,2827],{"class":2405},"/",[59,2829,2830],{"class":2413}," math.sqrt(d)\n",[59,2832,2833,2835,2838,2840],{"class":2401,"line":2604},[59,2834,2711],{"class":2693},[59,2836,2837],{"class":2413},".attention_weights ",[59,2839,815],{"class":2405},[59,2841,2842],{"class":2413}," masked_softmax(scores, valid_lens)\n",[59,2844,2846,2848,2851,2853,2856,2858],{"class":2401,"line":2845},16,[59,2847,2465],{"class":2405},[59,2849,2850],{"class":2413}," torch.bmm(",[59,2852,2666],{"class":2693},[59,2854,2855],{"class":2413},".dropout(",[59,2857,2666],{"class":2693},[59,2859,2860],{"class":2413},".attention_weights), values)\n",[15,2862,2863,2864,2867],{},"代码中函数的 ",[2397,2865,2866],{},"**kwargs","表示任意关键字参数，为了符合 Pytorch 的继承约定。",[49,2869,2870],{"id":2870},"多头注意力",[15,2872,2873],{},"多头注意力机制允许模型在不同位置上同时关注来自不同表示子空间的信息。使用单一注意力头时，平均操作会抑制这种能力。",[15,2875,2876],{},[41,2877],{"alt":2395,"src":2878},"transformer/MHA.png",[15,2880,2881],{},[59,2882,2884,3000],{"className":2883},[62],[59,2885,2887],{"className":2886},[66],[68,2888,2889],{"xmlns":70},[72,2890,2891,2997],{},[75,2892,2893,2917,2919,2921,2923,2925,2927,2929,2931,2933,2949,2951,2966,2968,2971,2973,2987,2989],{},[75,2894,2895,2898,2901,2904,2906,2908,2911,2913,2915],{},[91,2896,2897],{"mathvariant":818},"M",[91,2899,2900],{"mathvariant":818},"u",[91,2902,2903],{"mathvariant":818},"l",[91,2905,777],{"mathvariant":818},[91,2907,789],{"mathvariant":818},[91,2909,2910],{"mathvariant":818},"H",[91,2912,782],{"mathvariant":818},[91,2914,22],{"mathvariant":818},[91,2916,93],{"mathvariant":818},[461,2918,798],{"stretchy":797},[91,2920,459],{},[461,2922,803],{"separator":106},[91,2924,572],{},[461,2926,803],{"separator":106},[91,2928,672],{},[461,2930,812],{"stretchy":797},[461,2932,815],{},[75,2934,2935,2938,2940,2942,2945,2947],{},[91,2936,2937],{"mathvariant":818},"C",[91,2939,792],{"mathvariant":818},[91,2941,335],{"mathvariant":818},[91,2943,2944],{"mathvariant":818},"c",[91,2946,22],{"mathvariant":818},[91,2948,777],{"mathvariant":818},[461,2950,798],{"stretchy":797},[88,2952,2953,2964],{},[75,2954,2955,2958,2960,2962],{},[91,2956,2957],{"mathvariant":818},"h",[91,2959,782],{"mathvariant":818},[91,2961,22],{"mathvariant":818},[91,2963,93],{"mathvariant":818},[81,2965,83],{},[461,2967,803],{"separator":106},[461,2969,2970],{},"…",[461,2972,803],{"separator":106},[88,2974,2975,2985],{},[75,2976,2977,2979,2981,2983],{},[91,2978,2957],{"mathvariant":818},[91,2980,782],{"mathvariant":818},[91,2982,22],{"mathvariant":818},[91,2984,93],{"mathvariant":818},[91,2986,2957],{},[461,2988,812],{"stretchy":797},[465,2990,2991,2994],{},[91,2992,2993],{},"W",[91,2995,2996],{},"O",[98,2998,2999],{"encoding":100},"\\mathrm{MultiHead}(Q, K, V) = \\mathrm{Concat}(\\mathrm{head}_1, \\ldots, \\mathrm{head}_h) W^O",[59,3001,3003,3052],{"className":3002,"ariaHidden":106},[105],[59,3004,3006,3009,3016,3019,3022,3025,3028,3031,3034,3037,3040,3043,3046,3049],{"className":3005},[110],[59,3007],{"className":3008,"style":877},[114],[59,3010,3012],{"className":3011},[119],[59,3013,3015],{"className":3014},[119,955],"MultiHead",[59,3017,798],{"className":3018},[123],[59,3020,459],{"className":3021},[119,192],[59,3023,803],{"className":3024},[912],[59,3026],{"className":3027,"style":916},[499],[59,3029,572],{"className":3030,"style":603},[119,192],[59,3032,803],{"className":3033},[912],[59,3035],{"className":3036,"style":916},[499],[59,3038,672],{"className":3039,"style":702},[119,192],[59,3041,812],{"className":3042},[313],[59,3044],{"className":3045,"style":500},[499],[59,3047,815],{"className":3048},[504],[59,3050],{"className":3051,"style":500},[499],[59,3053,3055,3059,3066,3069,3115,3118,3121,3124,3127,3130,3133,3176,3179],{"className":3054},[110],[59,3056],{"className":3057,"style":3058},[114],"height:1.0913em;vertical-align:-0.25em;",[59,3060,3062],{"className":3061},[119],[59,3063,3065],{"className":3064},[119,955],"Concat",[59,3067,798],{"className":3068},[123],[59,3070,3072,3079],{"className":3071},[119],[59,3073,3075],{"className":3074},[119],[59,3076,3078],{"className":3077},[119,955],"head",[59,3080,3082],{"className":3081},[196],[59,3083,3085,3107],{"className":3084},[131,132],[59,3086,3088,3104],{"className":3087},[136],[59,3089,3092],{"className":3090,"style":3091},[140],"height:0.3011em;",[59,3093,3095,3098],{"style":3094},"top:-2.55em;margin-right:0.05em;",[59,3096],{"className":3097,"style":540},[148],[59,3099,3101],{"className":3100},[153,154,155,156],[59,3102,83],{"className":3103},[119,156],[59,3105,227],{"className":3106},[226],[59,3108,3110],{"className":3109},[136],[59,3111,3113],{"className":3112,"style":1270},[140],[59,3114],{},[59,3116,803],{"className":3117},[912],[59,3119],{"className":3120,"style":916},[499],[59,3122,2970],{"className":3123},[962],[59,3125],{"className":3126,"style":916},[499],[59,3128,803],{"className":3129},[912],[59,3131],{"className":3132,"style":916},[499],[59,3134,3136,3142],{"className":3135},[119],[59,3137,3139],{"className":3138},[119],[59,3140,3078],{"className":3141},[119,955],[59,3143,3145],{"className":3144},[196],[59,3146,3148,3168],{"className":3147},[131,132],[59,3149,3151,3165],{"className":3150},[136],[59,3152,3154],{"className":3153,"style":1248},[140],[59,3155,3156,3159],{"style":3094},[59,3157],{"className":3158,"style":540},[148],[59,3160,3162],{"className":3161},[153,154,155,156],[59,3163,2957],{"className":3164},[119,192,156],[59,3166,227],{"className":3167},[226],[59,3169,3171],{"className":3170},[136],[59,3172,3174],{"className":3173,"style":1270},[140],[59,3175],{},[59,3177,812],{"className":3178},[313],[59,3180,3182,3186],{"className":3181},[119],[59,3183,2993],{"className":3184,"style":3185},[119,192],"margin-right:0.13889em;",[59,3187,3189],{"className":3188},[196],[59,3190,3192],{"className":3191},[131],[59,3193,3195],{"className":3194},[136],[59,3196,3199],{"className":3197,"style":3198},[140],"height:0.8413em;",[59,3200,3201,3204],{"style":536},[59,3202],{"className":3203,"style":540},[148],[59,3205,3207],{"className":3206},[153,154,155,156],[59,3208,2996],{"className":3209,"style":3210},[119,192,156],"margin-right:0.02778em;",[15,3212,3213,3214,3576,3577,3826,3827,3826,4076,4327,4328,864],{},"其中 ",[59,3215,3217,3309],{"className":3216},[62],[59,3218,3220],{"className":3219},[66],[68,3221,3222],{"xmlns":70},[72,3223,3224,3306],{},[75,3225,3226,3240,3242,3262,3264,3266,3274,3276,3280,3282,3290,3292,3294,3296,3304],{},[88,3227,3228,3238],{},[75,3229,3230,3232,3234,3236],{},[91,3231,2957],{"mathvariant":818},[91,3233,782],{"mathvariant":818},[91,3235,22],{"mathvariant":818},[91,3237,93],{"mathvariant":818},[91,3239,789],{},[461,3241,815],{},[75,3243,3244,3246,3248,3250,3252,3254,3256,3258,3260],{},[91,3245,774],{"mathvariant":818},[91,3247,777],{"mathvariant":818},[91,3249,777],{"mathvariant":818},[91,3251,782],{"mathvariant":818},[91,3253,335],{"mathvariant":818},[91,3255,777],{"mathvariant":818},[91,3257,789],{"mathvariant":818},[91,3259,792],{"mathvariant":818},[91,3261,335],{"mathvariant":818},[461,3263,798],{"stretchy":797},[91,3265,459],{},[1822,3267,3268,3270,3272],{},[91,3269,2993],{},[91,3271,789],{},[91,3273,459],{},[461,3275,803],{"separator":106},[3277,3278,3279],"mtext",{},"  ",[91,3281,572],{},[1822,3283,3284,3286,3288],{},[91,3285,2993],{},[91,3287,789],{},[91,3289,572],{},[461,3291,803],{"separator":106},[3277,3293,3279],{},[91,3295,672],{},[1822,3297,3298,3300,3302],{},[91,3299,2993],{},[91,3301,789],{},[91,3303,672],{},[461,3305,812],{"stretchy":797},[98,3307,3308],{"encoding":100},"\\mathrm{head}_i=\\mathrm{Attention}(QW_i^Q,\\; KW_i^K,\\; VW_i^V)",[59,3310,3312,3370],{"className":3311,"ariaHidden":106},[105],[59,3313,3315,3318,3361,3364,3367],{"className":3314},[110],[59,3316],{"className":3317,"style":1229},[114],[59,3319,3321,3327],{"className":3320},[119],[59,3322,3324],{"className":3323},[119],[59,3325,3078],{"className":3326},[119,955],[59,3328,3330],{"className":3329},[196],[59,3331,3333,3353],{"className":3332},[131,132],[59,3334,3336,3350],{"className":3335},[136],[59,3337,3339],{"className":3338,"style":2032},[140],[59,3340,3341,3344],{"style":3094},[59,3342],{"className":3343,"style":540},[148],[59,3345,3347],{"className":3346},[153,154,155,156],[59,3348,789],{"className":3349},[119,192,156],[59,3351,227],{"className":3352},[226],[59,3354,3356],{"className":3355},[136],[59,3357,3359],{"className":3358,"style":1270},[140],[59,3360],{},[59,3362],{"className":3363,"style":500},[499],[59,3365,815],{"className":3366},[504],[59,3368],{"className":3369,"style":500},[499],[59,3371,3373,3377,3384,3387,3390,3445,3448,3451,3454,3457,3510,3513,3516,3519,3522,3573],{"className":3372},[110],[59,3374],{"className":3375,"style":3376},[114],"height:1.2361em;vertical-align:-0.2769em;",[59,3378,3380],{"className":3379},[119],[59,3381,3383],{"className":3382},[119,955],"Attention",[59,3385,798],{"className":3386},[123],[59,3388,459],{"className":3389},[119,192],[59,3391,3393,3396],{"className":3392},[119],[59,3394,2993],{"className":3395,"style":3185},[119,192],[59,3397,3399],{"className":3398},[196],[59,3400,3402,3436],{"className":3401},[131,132],[59,3403,3405,3433],{"className":3404},[136],[59,3406,3409,3421],{"className":3407,"style":3408},[140],"height:0.9592em;",[59,3410,3412,3415],{"style":3411},"top:-2.4231em;margin-left:-0.1389em;margin-right:0.05em;",[59,3413],{"className":3414,"style":540},[148],[59,3416,3418],{"className":3417},[153,154,155,156],[59,3419,789],{"className":3420},[119,192,156],[59,3422,3424,3427],{"style":3423},"top:-3.1809em;margin-right:0.05em;",[59,3425],{"className":3426,"style":540},[148],[59,3428,3430],{"className":3429},[153,154,155,156],[59,3431,459],{"className":3432},[119,192,156],[59,3434,227],{"className":3435},[226],[59,3437,3439],{"className":3438},[136],[59,3440,3443],{"className":3441,"style":3442},[140],"height:0.2769em;",[59,3444],{},[59,3446,803],{"className":3447},[912],[59,3449],{"className":3450,"style":500},[499],[59,3452],{"className":3453,"style":916},[499],[59,3455,572],{"className":3456,"style":603},[119,192],[59,3458,3460,3463],{"className":3459},[119],[59,3461,2993],{"className":3462,"style":3185},[119,192],[59,3464,3466],{"className":3465},[196],[59,3467,3469,3501],{"className":3468},[131,132],[59,3470,3472,3498],{"className":3471},[136],[59,3473,3475,3487],{"className":3474,"style":3198},[140],[59,3476,3478,3481],{"style":3477},"top:-2.4413em;margin-left:-0.1389em;margin-right:0.05em;",[59,3479],{"className":3480,"style":540},[148],[59,3482,3484],{"className":3483},[153,154,155,156],[59,3485,789],{"className":3486},[119,192,156],[59,3488,3489,3492],{"style":536},[59,3490],{"className":3491,"style":540},[148],[59,3493,3495],{"className":3494},[153,154,155,156],[59,3496,572],{"className":3497,"style":603},[119,192,156],[59,3499,227],{"className":3500},[226],[59,3502,3504],{"className":3503},[136],[59,3505,3508],{"className":3506,"style":3507},[140],"height:0.2587em;",[59,3509],{},[59,3511,803],{"className":3512},[912],[59,3514],{"className":3515,"style":500},[499],[59,3517],{"className":3518,"style":916},[499],[59,3520,672],{"className":3521,"style":702},[119,192],[59,3523,3525,3528],{"className":3524},[119],[59,3526,2993],{"className":3527,"style":3185},[119,192],[59,3529,3531],{"className":3530},[196],[59,3532,3534,3565],{"className":3533},[131,132],[59,3535,3537,3562],{"className":3536},[136],[59,3538,3540,3551],{"className":3539,"style":3198},[140],[59,3541,3542,3545],{"style":3477},[59,3543],{"className":3544,"style":540},[148],[59,3546,3548],{"className":3547},[153,154,155,156],[59,3549,789],{"className":3550},[119,192,156],[59,3552,3553,3556],{"style":536},[59,3554],{"className":3555,"style":540},[148],[59,3557,3559],{"className":3558},[153,154,155,156],[59,3560,672],{"className":3561,"style":702},[119,192,156],[59,3563,227],{"className":3564},[226],[59,3566,3568],{"className":3567},[136],[59,3569,3571],{"className":3570,"style":3507},[140],[59,3572],{},[59,3574,812],{"className":3575},[313],"，投影参数是矩阵\n",[59,3578,3580,3632],{"className":3579},[62],[59,3581,3583],{"className":3582},[66],[68,3584,3585],{"xmlns":70},[72,3586,3587,3629],{},[75,3588,3589,3597,3599],{},[1822,3590,3591,3593,3595],{},[91,3592,2993],{},[91,3594,789],{},[91,3596,459],{},[461,3598,463],{},[465,3600,3601,3603],{},[91,3602,470],{"mathvariant":469},[75,3604,3605,3621,3623],{},[88,3606,3607,3609],{},[91,3608,93],{},[75,3610,3611,3613,3615,3617,3619],{},[91,3612,366],{"mathvariant":818},[91,3614,792],{"mathvariant":818},[91,3616,93],{"mathvariant":818},[91,3618,782],{"mathvariant":818},[91,3620,2903],{"mathvariant":818},[461,3622,477],{},[88,3624,3625,3627],{},[91,3626,93],{},[91,3628,96],{},[98,3630,3631],{"encoding":100},"W_i^Q \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}",[59,3633,3635,3701],{"className":3634,"ariaHidden":106},[105],[59,3636,3638,3641,3692,3695,3698],{"className":3637},[110],[59,3639],{"className":3640,"style":3376},[114],[59,3642,3644,3647],{"className":3643},[119],[59,3645,2993],{"className":3646,"style":3185},[119,192],[59,3648,3650],{"className":3649},[196],[59,3651,3653,3684],{"className":3652},[131,132],[59,3654,3656,3681],{"className":3655},[136],[59,3657,3659,3670],{"className":3658,"style":3408},[140],[59,3660,3661,3664],{"style":3411},[59,3662],{"className":3663,"style":540},[148],[59,3665,3667],{"className":3666},[153,154,155,156],[59,3668,789],{"className":3669},[119,192,156],[59,3671,3672,3675],{"style":3423},[59,3673],{"className":3674,"style":540},[148],[59,3676,3678],{"className":3677},[153,154,155,156],[59,3679,459],{"className":3680},[119,192,156],[59,3682,227],{"className":3683},[226],[59,3685,3687],{"className":3686},[136],[59,3688,3690],{"className":3689,"style":3442},[140],[59,3691],{},[59,3693],{"className":3694,"style":500},[499],[59,3696,463],{"className":3697},[504],[59,3699],{"className":3700,"style":500},[499],[59,3702,3704,3707],{"className":3703},[110],[59,3705],{"className":3706,"style":514},[114],[59,3708,3710,3713],{"className":3709},[119],[59,3711,470],{"className":3712},[119,521],[59,3714,3716],{"className":3715},[196],[59,3717,3719],{"className":3718},[131],[59,3720,3722],{"className":3721},[136],[59,3723,3725],{"className":3724,"style":514},[140],[59,3726,3727,3730],{"style":536},[59,3728],{"className":3729,"style":540},[148],[59,3731,3733],{"className":3732},[153,154,155,156],[59,3734,3736,3783,3786],{"className":3735},[119,156],[59,3737,3739,3742],{"className":3738},[119,156],[59,3740,93],{"className":3741},[119,192,156],[59,3743,3745],{"className":3744},[196],[59,3746,3748,3775],{"className":3747},[131,132],[59,3749,3751,3772],{"className":3750},[136],[59,3752,3754],{"className":3753,"style":206},[140],[59,3755,3756,3759],{"style":209},[59,3757],{"className":3758,"style":213},[148],[59,3760,3762],{"className":3761},[153,217,218,156],[59,3763,3765],{"className":3764},[119,156],[59,3766,3768],{"className":3767},[119,156],[59,3769,3771],{"className":3770},[119,955,156],"model",[59,3773,227],{"className":3774},[226],[59,3776,3778],{"className":3777},[136],[59,3779,3781],{"className":3780,"style":234},[140],[59,3782],{},[59,3784,477],{"className":3785},[553,156],[59,3787,3789,3792],{"className":3788},[119,156],[59,3790,93],{"className":3791},[119,192,156],[59,3793,3795],{"className":3794},[196],[59,3796,3798,3818],{"className":3797},[131,132],[59,3799,3801,3815],{"className":3800},[136],[59,3802,3804],{"className":3803,"style":206},[140],[59,3805,3806,3809],{"style":209},[59,3807],{"className":3808,"style":213},[148],[59,3810,3812],{"className":3811},[153,217,218,156],[59,3813,96],{"className":3814,"style":222},[119,192,156],[59,3816,227],{"className":3817},[226],[59,3819,3821],{"className":3820},[136],[59,3822,3824],{"className":3823,"style":234},[140],[59,3825],{},",\n",[59,3828,3830,3882],{"className":3829},[62],[59,3831,3833],{"className":3832},[66],[68,3834,3835],{"xmlns":70},[72,3836,3837,3879],{},[75,3838,3839,3847,3849],{},[1822,3840,3841,3843,3845],{},[91,3842,2993],{},[91,3844,789],{},[91,3846,572],{},[461,3848,463],{},[465,3850,3851,3853],{},[91,3852,470],{"mathvariant":469},[75,3854,3855,3871,3873],{},[88,3856,3857,3859],{},[91,3858,93],{},[75,3860,3861,3863,3865,3867,3869],{},[91,3862,366],{"mathvariant":818},[91,3864,792],{"mathvariant":818},[91,3866,93],{"mathvariant":818},[91,3868,782],{"mathvariant":818},[91,3870,2903],{"mathvariant":818},[461,3872,477],{},[88,3874,3875,3877],{},[91,3876,93],{},[91,3878,96],{},[98,3880,3881],{"encoding":100},"W_i^K \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}",[59,3883,3885,3952],{"className":3884,"ariaHidden":106},[105],[59,3886,3888,3892,3943,3946,3949],{"className":3887},[110],[59,3889],{"className":3890,"style":3891},[114],"height:1.1em;vertical-align:-0.2587em;",[59,3893,3895,3898],{"className":3894},[119],[59,3896,2993],{"className":3897,"style":3185},[119,192],[59,3899,3901],{"className":3900},[196],[59,3902,3904,3935],{"className":3903},[131,132],[59,3905,3907,3932],{"className":3906},[136],[59,3908,3910,3921],{"className":3909,"style":3198},[140],[59,3911,3912,3915],{"style":3477},[59,3913],{"className":3914,"style":540},[148],[59,3916,3918],{"className":3917},[153,154,155,156],[59,3919,789],{"className":3920},[119,192,156],[59,3922,3923,3926],{"style":536},[59,3924],{"className":3925,"style":540},[148],[59,3927,3929],{"className":3928},[153,154,155,156],[59,3930,572],{"className":3931,"style":603},[119,192,156],[59,3933,227],{"className":3934},[226],[59,3936,3938],{"className":3937},[136],[59,3939,3941],{"className":3940,"style":3507},[140],[59,3942],{},[59,3944],{"className":3945,"style":500},[499],[59,3947,463],{"className":3948},[504],[59,3950],{"className":3951,"style":500},[499],[59,3953,3955,3958],{"className":3954},[110],[59,3956],{"className":3957,"style":514},[114],[59,3959,3961,3964],{"className":3960},[119],[59,3962,470],{"className":3963},[119,521],[59,3965,3967],{"className":3966},[196],[59,3968,3970],{"className":3969},[131],[59,3971,3973],{"className":3972},[136],[59,3974,3976],{"className":3975,"style":514},[140],[59,3977,3978,3981],{"style":536},[59,3979],{"className":3980,"style":540},[148],[59,3982,3984],{"className":3983},[153,154,155,156],[59,3985,3987,4033,4036],{"className":3986},[119,156],[59,3988,3990,3993],{"className":3989},[119,156],[59,3991,93],{"className":3992},[119,192,156],[59,3994,3996],{"className":3995},[196],[59,3997,3999,4025],{"className":3998},[131,132],[59,4000,4002,4022],{"className":4001},[136],[59,4003,4005],{"className":4004,"style":206},[140],[59,4006,4007,4010],{"style":209},[59,4008],{"className":4009,"style":213},[148],[59,4011,4013],{"className":4012},[153,217,218,156],[59,4014,4016],{"className":4015},[119,156],[59,4017,4019],{"className":4018},[119,156],[59,4020,3771],{"className":4021},[119,955,156],[59,4023,227],{"className":4024},[226],[59,4026,4028],{"className":4027},[136],[59,4029,4031],{"className":4030,"style":234},[140],[59,4032],{},[59,4034,477],{"className":4035},[553,156],[59,4037,4039,4042],{"className":4038},[119,156],[59,4040,93],{"className":4041},[119,192,156],[59,4043,4045],{"className":4044},[196],[59,4046,4048,4068],{"className":4047},[131,132],[59,4049,4051,4065],{"className":4050},[136],[59,4052,4054],{"className":4053,"style":206},[140],[59,4055,4056,4059],{"style":209},[59,4057],{"className":4058,"style":213},[148],[59,4060,4062],{"className":4061},[153,217,218,156],[59,4063,96],{"className":4064,"style":222},[119,192,156],[59,4066,227],{"className":4067},[226],[59,4069,4071],{"className":4070},[136],[59,4072,4074],{"className":4073,"style":234},[140],[59,4075],{},[59,4077,4079,4131],{"className":4078},[62],[59,4080,4082],{"className":4081},[66],[68,4083,4084],{"xmlns":70},[72,4085,4086,4128],{},[75,4087,4088,4096,4098],{},[1822,4089,4090,4092,4094],{},[91,4091,2993],{},[91,4093,789],{},[91,4095,672],{},[461,4097,463],{},[465,4099,4100,4102],{},[91,4101,470],{"mathvariant":469},[75,4103,4104,4120,4122],{},[88,4105,4106,4108],{},[91,4107,93],{},[75,4109,4110,4112,4114,4116,4118],{},[91,4111,366],{"mathvariant":818},[91,4113,792],{"mathvariant":818},[91,4115,93],{"mathvariant":818},[91,4117,782],{"mathvariant":818},[91,4119,2903],{"mathvariant":818},[461,4121,477],{},[88,4123,4124,4126],{},[91,4125,93],{},[91,4127,426],{},[98,4129,4130],{"encoding":100},"W_i^V \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_v}",[59,4132,4134,4200],{"className":4133,"ariaHidden":106},[105],[59,4135,4137,4140,4191,4194,4197],{"className":4136},[110],[59,4138],{"className":4139,"style":3891},[114],[59,4141,4143,4146],{"className":4142},[119],[59,4144,2993],{"className":4145,"style":3185},[119,192],[59,4147,4149],{"className":4148},[196],[59,4150,4152,4183],{"className":4151},[131,132],[59,4153,4155,4180],{"className":4154},[136],[59,4156,4158,4169],{"className":4157,"style":3198},[140],[59,4159,4160,4163],{"style":3477},[59,4161],{"className":4162,"style":540},[148],[59,4164,4166],{"className":4165},[153,154,155,156],[59,4167,789],{"className":4168},[119,192,156],[59,4170,4171,4174],{"style":536},[59,4172],{"className":4173,"style":540},[148],[59,4175,4177],{"className":4176},[153,154,155,156],[59,4178,672],{"className":4179,"style":702},[119,192,156],[59,4181,227],{"className":4182},[226],[59,4184,4186],{"className":4185},[136],[59,4187,4189],{"className":4188,"style":3507},[140],[59,4190],{},[59,4192],{"className":4193,"style":500},[499],[59,4195,463],{"className":4196},[504],[59,4198],{"className":4199,"style":500},[499],[59,4201,4203,4206],{"className":4202},[110],[59,4204],{"className":4205,"style":514},[114],[59,4207,4209,4212],{"className":4208},[119],[59,4210,470],{"className":4211},[119,521],[59,4213,4215],{"className":4214},[196],[59,4216,4218],{"className":4217},[131],[59,4219,4221],{"className":4220},[136],[59,4222,4224],{"className":4223,"style":514},[140],[59,4225,4226,4229],{"style":536},[59,4227],{"className":4228,"style":540},[148],[59,4230,4232],{"className":4231},[153,154,155,156],[59,4233,4235,4281,4284],{"className":4234},[119,156],[59,4236,4238,4241],{"className":4237},[119,156],[59,4239,93],{"className":4240},[119,192,156],[59,4242,4244],{"className":4243},[196],[59,4245,4247,4273],{"className":4246},[131,132],[59,4248,4250,4270],{"className":4249},[136],[59,4251,4253],{"className":4252,"style":206},[140],[59,4254,4255,4258],{"style":209},[59,4256],{"className":4257,"style":213},[148],[59,4259,4261],{"className":4260},[153,217,218,156],[59,4262,4264],{"className":4263},[119,156],[59,4265,4267],{"className":4266},[119,156],[59,4268,3771],{"className":4269},[119,955,156],[59,4271,227],{"className":4272},[226],[59,4274,4276],{"className":4275},[136],[59,4277,4279],{"className":4278,"style":234},[140],[59,4280],{},[59,4282,477],{"className":4283},[553,156],[59,4285,4287,4290],{"className":4286},[119,156],[59,4288,93],{"className":4289},[119,192,156],[59,4291,4293],{"className":4292},[196],[59,4294,4296,4318],{"className":4295},[131,132],[59,4297,4299,4315],{"className":4298},[136],[59,4300,4303],{"className":4301,"style":4302},[140],"height:0.1645em;",[59,4304,4306,4309],{"style":4305},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[59,4307],{"className":4308,"style":213},[148],[59,4310,4312],{"className":4311},[153,217,218,156],[59,4313,426],{"className":4314,"style":441},[119,192,156],[59,4316,227],{"className":4317},[226],[59,4319,4321],{"className":4320},[136],[59,4322,4325],{"className":4323,"style":4324},[140],"height:0.143em;",[59,4326],{},"\n和\n",[59,4329,4331,4383],{"className":4330},[62],[59,4332,4334],{"className":4333},[66],[68,4335,4336],{"xmlns":70},[72,4337,4338,4380],{},[75,4339,4340,4346,4348],{},[465,4341,4342,4344],{},[91,4343,2993],{},[91,4345,2996],{},[461,4347,463],{},[465,4349,4350,4352],{},[91,4351,470],{"mathvariant":469},[75,4353,4354,4356,4362,4364],{},[91,4355,2957],{},[88,4357,4358,4360],{},[91,4359,93],{},[91,4361,426],{},[461,4363,477],{},[88,4365,4366,4368],{},[91,4367,93],{},[75,4369,4370,4372,4374,4376,4378],{},[91,4371,366],{"mathvariant":818},[91,4373,792],{"mathvariant":818},[91,4375,93],{"mathvariant":818},[91,4377,782],{"mathvariant":818},[91,4379,2903],{"mathvariant":818},[98,4381,4382],{"encoding":100},"W^O \\in \\mathbb{R}^{h d_v \\times d_{\\mathrm{model}}}",[59,4384,4386,4431],{"className":4385,"ariaHidden":106},[105],[59,4387,4389,4393,4422,4425,4428],{"className":4388},[110],[59,4390],{"className":4391,"style":4392},[114],"height:0.8804em;vertical-align:-0.0391em;",[59,4394,4396,4399],{"className":4395},[119],[59,4397,2993],{"className":4398,"style":3185},[119,192],[59,4400,4402],{"className":4401},[196],[59,4403,4405],{"className":4404},[131],[59,4406,4408],{"className":4407},[136],[59,4409,4411],{"className":4410,"style":3198},[140],[59,4412,4413,4416],{"style":536},[59,4414],{"className":4415,"style":540},[148],[59,4417,4419],{"className":4418},[153,154,155,156],[59,4420,2996],{"className":4421,"style":3210},[119,192,156],[59,4423],{"className":4424,"style":500},[499],[59,4426,463],{"className":4427},[504],[59,4429],{"className":4430,"style":500},[499],[59,4432,4434,4437],{"className":4433},[110],[59,4435],{"className":4436,"style":514},[114],[59,4438,4440,4443],{"className":4439},[119],[59,4441,470],{"className":4442},[119,521],[59,4444,4446],{"className":4445},[196],[59,4447,4449],{"className":4448},[131],[59,4450,4452],{"className":4451},[136],[59,4453,4455],{"className":4454,"style":514},[140],[59,4456,4457,4460],{"style":536},[59,4458],{"className":4459,"style":540},[148],[59,4461,4463],{"className":4462},[153,154,155,156],[59,4464,4466,4469,4509,4512],{"className":4465},[119,156],[59,4467,2957],{"className":4468},[119,192,156],[59,4470,4472,4475],{"className":4471},[119,156],[59,4473,93],{"className":4474},[119,192,156],[59,4476,4478],{"className":4477},[196],[59,4479,4481,4501],{"className":4480},[131,132],[59,4482,4484,4498],{"className":4483},[136],[59,4485,4487],{"className":4486,"style":4302},[140],[59,4488,4489,4492],{"style":4305},[59,4490],{"className":4491,"style":213},[148],[59,4493,4495],{"className":4494},[153,217,218,156],[59,4496,426],{"className":4497,"style":441},[119,192,156],[59,4499,227],{"className":4500},[226],[59,4502,4504],{"className":4503},[136],[59,4505,4507],{"className":4506,"style":4324},[140],[59,4508],{},[59,4510,477],{"className":4511},[553,156],[59,4513,4515,4518],{"className":4514},[119,156],[59,4516,93],{"className":4517},[119,192,156],[59,4519,4521],{"className":4520},[196],[59,4522,4524,4550],{"className":4523},[131,132],[59,4525,4527,4547],{"className":4526},[136],[59,4528,4530],{"className":4529,"style":206},[140],[59,4531,4532,4535],{"style":209},[59,4533],{"className":4534,"style":213},[148],[59,4536,4538],{"className":4537},[153,217,218,156],[59,4539,4541],{"className":4540},[119,156],[59,4542,4544],{"className":4543},[119,156],[59,4545,3771],{"className":4546},[119,955,156],[59,4548,227],{"className":4549},[226],[59,4551,4553],{"className":4552},[136],[59,4554,4556],{"className":4555,"style":234},[140],[59,4557],{},[15,4559,4560,4561,4564,4565,4568,4569,4571,4572,4574],{},"为了能使多个头并行计算，我们定义两个转置函数 ",[2397,4562,4563],{},"transpose_output"," 和 ",[2397,4566,4567],{},"transpose_qkv","，其中 ",[2397,4570,4563],{}," 还原了 ",[2397,4573,4567],{}," 的结果。",[2390,4576,4578],{"className":2392,"code":4577,"language":2394,"meta":2395,"style":2395},"def transpose_qkv(X, num_heads):\n    \"\"\"为了多注意力头的并行计算而变换形状\"\"\"\n    # 输入X的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n    # 输出X的形状:(batch_size，查询或者“键－值”对的个数，num_heads，num_hiddens/num_heads)\n    X = X.reshape(X.shape[0], X.shape[1], num_heads, -1)\n\n    # 输出X的形状:(batch_size，num_heads，查询或者“键－值”对的个数, num_hiddens/num_heads)\n    X = X.permute(0, 2, 1, 3)\n\n    # 最终输出的形状:(batch_size*num_heads,查询或者“键－值”对的个数, num_hiddens/num_heads)\n    return X.reshape(-1, X.shape[2], X.shape[3])\n\ndef transpose_output(X, num_heads):\n    \"\"\"逆转transpose_qkv函数的操作\"\"\"\n    X = X.reshape(-1, num_heads, X.shape[1], X.shape[2])\n    X = X.permute(0, 2, 1, 3)\n    return X.reshape(X.shape[0], X.shape[1], -1)\n",[2397,4579,4580,4598,4603,4608,4613,4639,4643,4648,4674,4678,4683,4706,4710,4727,4732,4755,4779],{"__ignoreMap":2395},[59,4581,4582,4584,4587,4589,4591,4593,4596],{"class":2401,"line":2402},[59,4583,2406],{"class":2405},[59,4585,4586],{"class":2409}," transpose_qkv",[59,4588,798],{"class":2413},[59,4590,2417],{"class":2416},[59,4592,2420],{"class":2413},[59,4594,4595],{"class":2416},"num_heads",[59,4597,2426],{"class":2413},[59,4599,4600],{"class":2401,"line":2429},[59,4601,4602],{"class":2432},"    \"\"\"为了多注意力头的并行计算而变换形状\"\"\"\n",[59,4604,4605],{"class":2401,"line":2436},[59,4606,4607],{"class":2439},"    # 输入X的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n",[59,4609,4610],{"class":2401,"line":2443},[59,4611,4612],{"class":2439},"    # 输出X的形状:(batch_size，查询或者“键－值”对的个数，num_heads，num_hiddens/num_heads)\n",[59,4614,4615,4618,4620,4623,4625,4628,4630,4633,4635,4637],{"class":2401,"line":2462},[59,4616,4617],{"class":2413},"    X ",[59,4619,815],{"class":2405},[59,4621,4622],{"class":2413}," X.reshape(X.shape[",[59,4624,1754],{"class":2455},[59,4626,4627],{"class":2413},"], X.shape[",[59,4629,83],{"class":2455},[59,4631,4632],{"class":2413},"], num_heads, ",[59,4634,2553],{"class":2405},[59,4636,83],{"class":2455},[59,4638,2480],{"class":2413},[59,4640,4641],{"class":2401,"line":2483},[59,4642,2725],{"emptyLinePlaceholder":2724},[59,4644,4645],{"class":2401,"line":2491},[59,4646,4647],{"class":2439},"    # 输出X的形状:(batch_size，num_heads，查询或者“键－值”对的个数, num_hiddens/num_heads)\n",[59,4649,4650,4652,4654,4657,4659,4661,4663,4665,4667,4669,4672],{"class":2401,"line":2502},[59,4651,4617],{"class":2413},[59,4653,815],{"class":2405},[59,4655,4656],{"class":2413}," X.permute(",[59,4658,1754],{"class":2455},[59,4660,2420],{"class":2413},[59,4662,2821],{"class":2455},[59,4664,2420],{"class":2413},[59,4666,83],{"class":2455},[59,4668,2420],{"class":2413},[59,4670,4671],{"class":2455},"3",[59,4673,2480],{"class":2413},[59,4675,4676],{"class":2401,"line":2519},[59,4677,2725],{"emptyLinePlaceholder":2724},[59,4679,4680],{"class":2401,"line":2535},[59,4681,4682],{"class":2439},"    # 最终输出的形状:(batch_size*num_heads,查询或者“键－值”对的个数, num_hiddens/num_heads)\n",[59,4684,4685,4688,4691,4693,4695,4698,4700,4702,4704],{"class":2401,"line":2543},[59,4686,4687],{"class":2405},"    return",[59,4689,4690],{"class":2413}," X.reshape(",[59,4692,2553],{"class":2405},[59,4694,83],{"class":2455},[59,4696,4697],{"class":2413},", X.shape[",[59,4699,2821],{"class":2455},[59,4701,4627],{"class":2413},[59,4703,4671],{"class":2455},[59,4705,2532],{"class":2413},[59,4707,4708],{"class":2401,"line":2560},[59,4709,2725],{"emptyLinePlaceholder":2724},[59,4711,4712,4714,4717,4719,4721,4723,4725],{"class":2401,"line":2566},[59,4713,2406],{"class":2405},[59,4715,4716],{"class":2409}," transpose_output",[59,4718,798],{"class":2413},[59,4720,2417],{"class":2416},[59,4722,2420],{"class":2413},[59,4724,4595],{"class":2416},[59,4726,2426],{"class":2413},[59,4728,4729],{"class":2401,"line":2591},[59,4730,4731],{"class":2432},"    \"\"\"逆转transpose_qkv函数的操作\"\"\"\n",[59,4733,4734,4736,4738,4740,4742,4744,4747,4749,4751,4753],{"class":2401,"line":2604},[59,4735,4617],{"class":2413},[59,4737,815],{"class":2405},[59,4739,4690],{"class":2413},[59,4741,2553],{"class":2405},[59,4743,83],{"class":2455},[59,4745,4746],{"class":2413},", num_heads, X.shape[",[59,4748,83],{"class":2455},[59,4750,4627],{"class":2413},[59,4752,2821],{"class":2455},[59,4754,2532],{"class":2413},[59,4756,4757,4759,4761,4763,4765,4767,4769,4771,4773,4775,4777],{"class":2401,"line":2845},[59,4758,4617],{"class":2413},[59,4760,815],{"class":2405},[59,4762,4656],{"class":2413},[59,4764,1754],{"class":2455},[59,4766,2420],{"class":2413},[59,4768,2821],{"class":2455},[59,4770,2420],{"class":2413},[59,4772,83],{"class":2455},[59,4774,2420],{"class":2413},[59,4776,4671],{"class":2455},[59,4778,2480],{"class":2413},[59,4780,4782,4784,4786,4788,4790,4792,4795,4797,4799],{"class":2401,"line":4781},17,[59,4783,4687],{"class":2405},[59,4785,4622],{"class":2413},[59,4787,1754],{"class":2455},[59,4789,4627],{"class":2413},[59,4791,83],{"class":2455},[59,4793,4794],{"class":2413},"], ",[59,4796,2553],{"class":2405},[59,4798,83],{"class":2455},[59,4800,2480],{"class":2413},[2390,4802,4804],{"className":2392,"code":4803,"language":2394,"meta":2395,"style":2395},"#@save\nclass MultiHeadAttention(nn.Module):\n    \"\"\"多头注意力\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 num_heads, dropout, bias=False, **kwargs):\n        super(MultiHeadAttention, self).__init__(**kwargs)\n        self.num_heads = num_heads\n        self.attention = d2l.DotProductAttention(dropout)\n        self.W_q = nn.Linear(query_size, num_hiddens, bias=bias)\n        self.W_k = nn.Linear(key_size, num_hiddens, bias=bias)\n        self.W_v = nn.Linear(value_size, num_hiddens, bias=bias)\n        self.W_o = nn.Linear(num_hiddens, num_hiddens, bias=bias)\n\n    def forward(self, queries, keys, values, valid_lens):\n        # queries，keys，values的形状:\n        # (batch_size，查询或者“键－值”对的个数，num_hiddens)\n        # valid_lens　的形状:\n        # (batch_size，)或(batch_size，查询的个数)\n        # 经过变换后，输出的queries，keys，values 的形状:\n        # (batch_size*num_heads，查询或者“键－值”对的个数，num_hiddens/num_heads)\n        queries = transpose_qkv(self.W_q(queries), self.num_heads)\n        keys = transpose_qkv(self.W_k(keys), self.num_heads)\n        values = transpose_qkv(self.W_v(values), self.num_heads)\n\n        if valid_lens is not None:\n            # 在轴0，将第一项（标量或者矢量）复制num_heads次，\n            # 然后如此复制第二项，然后诸如此类。\n            valid_lens = torch.repeat_interleave(\n                valid_lens, repeats=self.num_heads, dim=0)\n\n        # output的形状:(batch_size*num_heads，查询的个数， num_hiddens/num_heads)\n        output = self.attention(queries, keys, values, valid_lens)\n\n        # output_concat的形状:(batch_size，查询的个数，num_hiddens)\n        output_concat = transpose_output(output, self.num_heads)\n        return self.W_o(output_concat)\n",[2397,4805,4806,4811,4828,4833,4865,4892,4911,4923,4935,4954,4972,4990,5008,5012,5040,5045,5050,5055,5061,5067,5073,5094,5113,5132,5137,5153,5159,5165,5175,5199,5204,5210,5224,5229,5235,5250],{"__ignoreMap":2395},[59,4807,4808],{"class":2401,"line":2402},[59,4809,4810],{"class":2439},"#@save\n",[59,4812,4813,4815,4818,4820,4822,4824,4826],{"class":2401,"line":2429},[59,4814,2630],{"class":2405},[59,4816,4817],{"class":2633}," MultiHeadAttention",[59,4819,798],{"class":2413},[59,4821,2640],{"class":2639},[59,4823,864],{"class":2413},[59,4825,2645],{"class":2639},[59,4827,2426],{"class":2413},[59,4829,4830],{"class":2401,"line":2436},[59,4831,4832],{"class":2432},"    \"\"\"多头注意力\"\"\"\n",[59,4834,4835,4837,4839,4841,4843,4845,4848,4850,4853,4855,4858,4860,4863],{"class":2401,"line":2443},[59,4836,2657],{"class":2405},[59,4838,2660],{"class":2455},[59,4840,798],{"class":2413},[59,4842,2666],{"class":2665},[59,4844,2420],{"class":2413},[59,4846,4847],{"class":2416},"key_size",[59,4849,2420],{"class":2413},[59,4851,4852],{"class":2416},"query_size",[59,4854,2420],{"class":2413},[59,4856,4857],{"class":2416},"value_size",[59,4859,2420],{"class":2413},[59,4861,4862],{"class":2416},"num_hiddens",[59,4864,3826],{"class":2413},[59,4866,4867,4870,4872,4874,4876,4879,4881,4884,4886,4888,4890],{"class":2401,"line":2462},[59,4868,4869],{"class":2416},"                 num_heads",[59,4871,2420],{"class":2413},[59,4873,2671],{"class":2416},[59,4875,2420],{"class":2413},[59,4877,4878],{"class":2416},"bias",[59,4880,815],{"class":2405},[59,4882,4883],{"class":2455},"False",[59,4885,2420],{"class":2413},[59,4887,2676],{"class":2405},[59,4889,2679],{"class":2416},[59,4891,2426],{"class":2413},[59,4893,4894,4896,4899,4901,4903,4905,4907,4909],{"class":2401,"line":2483},[59,4895,2687],{"class":2686},[59,4897,4898],{"class":2413},"(MultiHeadAttention, ",[59,4900,2666],{"class":2693},[59,4902,2696],{"class":2413},[59,4904,2699],{"class":2455},[59,4906,798],{"class":2413},[59,4908,2676],{"class":2405},[59,4910,2706],{"class":2413},[59,4912,4913,4915,4918,4920],{"class":2401,"line":2491},[59,4914,2711],{"class":2693},[59,4916,4917],{"class":2413},".num_heads ",[59,4919,815],{"class":2405},[59,4921,4922],{"class":2413}," num_heads\n",[59,4924,4925,4927,4930,4932],{"class":2401,"line":2502},[59,4926,2711],{"class":2693},[59,4928,4929],{"class":2413},".attention ",[59,4931,815],{"class":2405},[59,4933,4934],{"class":2413}," d2l.DotProductAttention(dropout)\n",[59,4936,4937,4939,4942,4944,4947,4949,4951],{"class":2401,"line":2519},[59,4938,2711],{"class":2693},[59,4940,4941],{"class":2413},".W_q ",[59,4943,815],{"class":2405},[59,4945,4946],{"class":2413}," nn.Linear(query_size, num_hiddens, ",[59,4948,4878],{"class":2471},[59,4950,815],{"class":2405},[59,4952,4953],{"class":2413},"bias)\n",[59,4955,4956,4958,4961,4963,4966,4968,4970],{"class":2401,"line":2535},[59,4957,2711],{"class":2693},[59,4959,4960],{"class":2413},".W_k ",[59,4962,815],{"class":2405},[59,4964,4965],{"class":2413}," nn.Linear(key_size, num_hiddens, ",[59,4967,4878],{"class":2471},[59,4969,815],{"class":2405},[59,4971,4953],{"class":2413},[59,4973,4974,4976,4979,4981,4984,4986,4988],{"class":2401,"line":2543},[59,4975,2711],{"class":2693},[59,4977,4978],{"class":2413},".W_v ",[59,4980,815],{"class":2405},[59,4982,4983],{"class":2413}," nn.Linear(value_size, num_hiddens, ",[59,4985,4878],{"class":2471},[59,4987,815],{"class":2405},[59,4989,4953],{"class":2413},[59,4991,4992,4994,4997,4999,5002,5004,5006],{"class":2401,"line":2560},[59,4993,2711],{"class":2693},[59,4995,4996],{"class":2413},".W_o ",[59,4998,815],{"class":2405},[59,5000,5001],{"class":2413}," nn.Linear(num_hiddens, num_hiddens, ",[59,5003,4878],{"class":2471},[59,5005,815],{"class":2405},[59,5007,4953],{"class":2413},[59,5009,5010],{"class":2401,"line":2566},[59,5011,2725],{"emptyLinePlaceholder":2724},[59,5013,5014,5016,5018,5020,5022,5024,5026,5028,5030,5032,5034,5036,5038],{"class":2401,"line":2591},[59,5015,2657],{"class":2405},[59,5017,2752],{"class":2409},[59,5019,798],{"class":2413},[59,5021,2666],{"class":2665},[59,5023,2420],{"class":2413},[59,5025,2761],{"class":2416},[59,5027,2420],{"class":2413},[59,5029,2766],{"class":2416},[59,5031,2420],{"class":2413},[59,5033,2771],{"class":2416},[59,5035,2420],{"class":2413},[59,5037,2423],{"class":2416},[59,5039,2426],{"class":2413},[59,5041,5042],{"class":2401,"line":2604},[59,5043,5044],{"class":2439},"        # queries，keys，values的形状:\n",[59,5046,5047],{"class":2401,"line":2845},[59,5048,5049],{"class":2439},"        # (batch_size，查询或者“键－值”对的个数，num_hiddens)\n",[59,5051,5052],{"class":2401,"line":4781},[59,5053,5054],{"class":2439},"        # valid_lens　的形状:\n",[59,5056,5058],{"class":2401,"line":5057},18,[59,5059,5060],{"class":2439},"        # (batch_size，)或(batch_size，查询的个数)\n",[59,5062,5064],{"class":2401,"line":5063},19,[59,5065,5066],{"class":2439},"        # 经过变换后，输出的queries，keys，values 的形状:\n",[59,5068,5070],{"class":2401,"line":5069},20,[59,5071,5072],{"class":2439},"        # (batch_size*num_heads，查询或者“键－值”对的个数，num_hiddens/num_heads)\n",[59,5074,5076,5079,5081,5084,5086,5089,5091],{"class":2401,"line":5075},21,[59,5077,5078],{"class":2413},"        queries ",[59,5080,815],{"class":2405},[59,5082,5083],{"class":2413}," transpose_qkv(",[59,5085,2666],{"class":2693},[59,5087,5088],{"class":2413},".W_q(queries), ",[59,5090,2666],{"class":2693},[59,5092,5093],{"class":2413},".num_heads)\n",[59,5095,5097,5100,5102,5104,5106,5109,5111],{"class":2401,"line":5096},22,[59,5098,5099],{"class":2413},"        keys ",[59,5101,815],{"class":2405},[59,5103,5083],{"class":2413},[59,5105,2666],{"class":2693},[59,5107,5108],{"class":2413},".W_k(keys), ",[59,5110,2666],{"class":2693},[59,5112,5093],{"class":2413},[59,5114,5116,5119,5121,5123,5125,5128,5130],{"class":2401,"line":5115},23,[59,5117,5118],{"class":2413},"        values ",[59,5120,815],{"class":2405},[59,5122,5083],{"class":2413},[59,5124,2666],{"class":2693},[59,5126,5127],{"class":2413},".W_v(values), ",[59,5129,2666],{"class":2693},[59,5131,5093],{"class":2413},[59,5133,5135],{"class":2401,"line":5134},24,[59,5136,2725],{"emptyLinePlaceholder":2724},[59,5138,5140,5142,5144,5146,5149,5151],{"class":2401,"line":5139},25,[59,5141,2505],{"class":2405},[59,5143,2449],{"class":2413},[59,5145,2452],{"class":2405},[59,5147,5148],{"class":2405}," not",[59,5150,2456],{"class":2455},[59,5152,2459],{"class":2413},[59,5154,5156],{"class":2401,"line":5155},26,[59,5157,5158],{"class":2439},"            # 在轴0，将第一项（标量或者矢量）复制num_heads次，\n",[59,5160,5162],{"class":2401,"line":5161},27,[59,5163,5164],{"class":2439},"            # 然后如此复制第二项，然后诸如此类。\n",[59,5166,5168,5170,5172],{"class":2401,"line":5167},28,[59,5169,2522],{"class":2413},[59,5171,815],{"class":2405},[59,5173,5174],{"class":2413}," torch.repeat_interleave(\n",[59,5176,5178,5181,5184,5186,5188,5191,5193,5195,5197],{"class":2401,"line":5177},29,[59,5179,5180],{"class":2413},"                valid_lens, ",[59,5182,5183],{"class":2471},"repeats",[59,5185,815],{"class":2405},[59,5187,2666],{"class":2693},[59,5189,5190],{"class":2413},".num_heads, ",[59,5192,2472],{"class":2471},[59,5194,815],{"class":2405},[59,5196,1754],{"class":2455},[59,5198,2480],{"class":2413},[59,5200,5202],{"class":2401,"line":5201},30,[59,5203,2725],{"emptyLinePlaceholder":2724},[59,5205,5207],{"class":2401,"line":5206},31,[59,5208,5209],{"class":2439},"        # output的形状:(batch_size*num_heads，查询的个数， num_hiddens/num_heads)\n",[59,5211,5213,5216,5218,5221],{"class":2401,"line":5212},32,[59,5214,5215],{"class":2413},"        output ",[59,5217,815],{"class":2405},[59,5219,5220],{"class":2693}," self",[59,5222,5223],{"class":2413},".attention(queries, keys, values, valid_lens)\n",[59,5225,5227],{"class":2401,"line":5226},33,[59,5228,2725],{"emptyLinePlaceholder":2724},[59,5230,5232],{"class":2401,"line":5231},34,[59,5233,5234],{"class":2439},"        # output_concat的形状:(batch_size，查询的个数，num_hiddens)\n",[59,5236,5238,5241,5243,5246,5248],{"class":2401,"line":5237},35,[59,5239,5240],{"class":2413},"        output_concat ",[59,5242,815],{"class":2405},[59,5244,5245],{"class":2413}," transpose_output(output, ",[59,5247,2666],{"class":2693},[59,5249,5093],{"class":2413},[59,5251,5253,5255,5257],{"class":2401,"line":5252},36,[59,5254,2465],{"class":2405},[59,5256,5220],{"class":2693},[59,5258,5259],{"class":2413},".W_o(output_concat)\n",[11,5261,5262],{"id":5262},"位置编码",[15,5264,5265,5266,5357],{},"在处理 token 序列时 RNN 采用了按顺序逐个处理 token 的，但注意力机制为了并行计算放弃了顺序处理，这会导致其失去位置信息。为了使模型能够使用到位置信息，我们需要在 token 中注入一些和位置相关的信息，我们使用位置编码来注入绝对的或相对的位置信息。位置编码与嵌入的维度相同，均为 ",[59,5267,5269,5295],{"className":5268},[62],[59,5270,5272],{"className":5271},[66],[68,5273,5274],{"xmlns":70},[72,5275,5276,5292],{},[75,5277,5278,5284,5286,5288,5290],{},[88,5279,5280,5282],{},[91,5281,93],{},[91,5283,366],{},[91,5285,792],{},[91,5287,93],{},[91,5289,782],{},[91,5291,2903],{},[98,5293,5294],{"encoding":100},"d_model",[59,5296,5298],{"className":5297,"ariaHidden":106},[105],[59,5299,5301,5304,5344,5347,5350,5353],{"className":5300},[110],[59,5302],{"className":5303,"style":1229},[114],[59,5305,5307,5310],{"className":5306},[119],[59,5308,93],{"className":5309},[119,192],[59,5311,5313],{"className":5312},[196],[59,5314,5316,5336],{"className":5315},[131,132],[59,5317,5319,5333],{"className":5318},[136],[59,5320,5322],{"className":5321,"style":1322},[140],[59,5323,5324,5327],{"style":1251},[59,5325],{"className":5326,"style":540},[148],[59,5328,5330],{"className":5329},[153,154,155,156],[59,5331,366],{"className":5332},[119,192,156],[59,5334,227],{"className":5335},[226],[59,5337,5339],{"className":5338},[136],[59,5340,5342],{"className":5341,"style":1270},[140],[59,5343],{},[59,5345,792],{"className":5346},[119,192],[59,5348,93],{"className":5349},[119,192],[59,5351,782],{"className":5352},[119,192],[59,5354,2903],{"className":5355,"style":5356},[119,192],"margin-right:0.01968em;","，因此两者可以相加。位置编码可以通过学习得到也可以直接固定得到。",[15,5359,5360],{},"在这里，我们使用的是基于正弦函数和余弦函数的位置编码。",[15,5362,5363,5364,5529,5530,5558,5559,5587,5588,5760,5761,5822,5823,5852,5853,5890,5891,5947],{},"假设输入表示 ",[59,5365,5367,5410],{"className":5366},[62],[59,5368,5370],{"className":5369},[66],[68,5371,5372],{"xmlns":70},[72,5373,5374,5407],{},[75,5375,5376,5379,5381],{},[91,5377,2417],{"mathvariant":5378},"bold",[461,5380,463],{},[465,5382,5383,5385],{},[91,5384,470],{"mathvariant":469},[75,5386,5387,5389,5391],{},[91,5388,335],{},[461,5390,477],{},[88,5392,5393,5395],{},[91,5394,93],{},[75,5396,5397,5399,5401,5403,5405],{},[91,5398,366],{},[91,5400,792],{},[91,5402,93],{},[91,5404,782],{},[91,5406,2903],{},[98,5408,5409],{"encoding":100},"\\mathbf{X} \\in \\mathbb{R}^{n \\times d_{model}}",[59,5411,5413,5433],{"className":5412,"ariaHidden":106},[105],[59,5414,5416,5420,5424,5427,5430],{"className":5415},[110],[59,5417],{"className":5418,"style":5419},[114],"height:0.7252em;vertical-align:-0.0391em;",[59,5421,2417],{"className":5422},[119,5423],"mathbf",[59,5425],{"className":5426,"style":500},[499],[59,5428,463],{"className":5429},[504],[59,5431],{"className":5432,"style":500},[499],[59,5434,5436,5439],{"className":5435},[110],[59,5437],{"className":5438,"style":514},[114],[59,5440,5442,5445],{"className":5441},[119],[59,5443,470],{"className":5444},[119,521],[59,5446,5448],{"className":5447},[196],[59,5449,5451],{"className":5450},[131],[59,5452,5454],{"className":5453},[136],[59,5455,5457],{"className":5456,"style":514},[140],[59,5458,5459,5462],{"style":536},[59,5460],{"className":5461,"style":540},[148],[59,5463,5465],{"className":5464},[153,154,155,156],[59,5466,5468,5471,5474],{"className":5467},[119,156],[59,5469,335],{"className":5470},[119,192,156],[59,5472,477],{"className":5473},[553,156],[59,5475,5477,5480],{"className":5476},[119,156],[59,5478,93],{"className":5479},[119,192,156],[59,5481,5483],{"className":5482},[196],[59,5484,5486,5521],{"className":5485},[131,132],[59,5487,5489,5518],{"className":5488},[136],[59,5490,5492],{"className":5491,"style":206},[140],[59,5493,5494,5497],{"style":209},[59,5495],{"className":5496,"style":213},[148],[59,5498,5500],{"className":5499},[153,217,218,156],[59,5501,5503,5506,5509,5512,5515],{"className":5502},[119,156],[59,5504,366],{"className":5505},[119,192,156],[59,5507,792],{"className":5508},[119,192,156],[59,5510,93],{"className":5511},[119,192,156],[59,5513,782],{"className":5514},[119,192,156],[59,5516,2903],{"className":5517,"style":5356},[119,192,156],[59,5519,227],{"className":5520},[226],[59,5522,5524],{"className":5523},[136],[59,5525,5527],{"className":5526,"style":234},[140],[59,5528],{}," 包含一个序列中 ",[59,5531,5533,5546],{"className":5532},[62],[59,5534,5536],{"className":5535},[66],[68,5537,5538],{"xmlns":70},[72,5539,5540,5544],{},[75,5541,5542],{},[91,5543,335],{},[98,5545,335],{"encoding":100},[59,5547,5549],{"className":5548,"ariaHidden":106},[105],[59,5550,5552,5555],{"className":5551},[110],[59,5553],{"className":5554,"style":347},[114],[59,5556,335],{"className":5557},[119,192]," 个词元的 ",[59,5560,5562,5575],{"className":5561},[62],[59,5563,5565],{"className":5564},[66],[68,5566,5567],{"xmlns":70},[72,5568,5569,5573],{},[75,5570,5571],{},[91,5572,93],{},[98,5574,93],{"encoding":100},[59,5576,5578],{"className":5577,"ariaHidden":106},[105],[59,5579,5581,5584],{"className":5580},[110],[59,5582],{"className":5583,"style":407},[114],[59,5585,93],{"className":5586},[119,192]," 维嵌入表示。位置编码使用相同形状的位置嵌入矩阵 ",[59,5589,5591,5639],{"className":5590},[62],[59,5592,5594],{"className":5593},[66],[68,5595,5596],{"xmlns":70},[72,5597,5598,5636],{},[75,5599,5600,5608,5610],{},[75,5601,5602,5605],{},[91,5603,5604],{"mathvariant":5378},"P",[91,5606,5607],{"mathvariant":5378},"E",[461,5609,463],{},[465,5611,5612,5614],{},[91,5613,470],{"mathvariant":469},[75,5615,5616,5618,5620],{},[91,5617,335],{},[461,5619,477],{},[88,5621,5622,5624],{},[91,5623,93],{},[75,5625,5626,5628,5630,5632,5634],{},[91,5627,366],{},[91,5629,792],{},[91,5631,93],{},[91,5633,782],{},[91,5635,2903],{},[98,5637,5638],{"encoding":100},"\\mathbf{PE} \\in \\mathbb{R}^{n \\times d_{model}}",[59,5640,5642,5664],{"className":5641,"ariaHidden":106},[105],[59,5643,5645,5648,5655,5658,5661],{"className":5644},[110],[59,5646],{"className":5647,"style":5419},[114],[59,5649,5651],{"className":5650},[119],[59,5652,5654],{"className":5653},[119,5423],"PE",[59,5656],{"className":5657,"style":500},[499],[59,5659,463],{"className":5660},[504],[59,5662],{"className":5663,"style":500},[499],[59,5665,5667,5670],{"className":5666},[110],[59,5668],{"className":5669,"style":514},[114],[59,5671,5673,5676],{"className":5672},[119],[59,5674,470],{"className":5675},[119,521],[59,5677,5679],{"className":5678},[196],[59,5680,5682],{"className":5681},[131],[59,5683,5685],{"className":5684},[136],[59,5686,5688],{"className":5687,"style":514},[140],[59,5689,5690,5693],{"style":536},[59,5691],{"className":5692,"style":540},[148],[59,5694,5696],{"className":5695},[153,154,155,156],[59,5697,5699,5702,5705],{"className":5698},[119,156],[59,5700,335],{"className":5701},[119,192,156],[59,5703,477],{"className":5704},[553,156],[59,5706,5708,5711],{"className":5707},[119,156],[59,5709,93],{"className":5710},[119,192,156],[59,5712,5714],{"className":5713},[196],[59,5715,5717,5752],{"className":5716},[131,132],[59,5718,5720,5749],{"className":5719},[136],[59,5721,5723],{"className":5722,"style":206},[140],[59,5724,5725,5728],{"style":209},[59,5726],{"className":5727,"style":213},[148],[59,5729,5731],{"className":5730},[153,217,218,156],[59,5732,5734,5737,5740,5743,5746],{"className":5733},[119,156],[59,5735,366],{"className":5736},[119,192,156],[59,5738,792],{"className":5739},[119,192,156],[59,5741,93],{"className":5742},[119,192,156],[59,5744,782],{"className":5745},[119,192,156],[59,5747,2903],{"className":5748,"style":5356},[119,192,156],[59,5750,227],{"className":5751},[226],[59,5753,5755],{"className":5754},[136],[59,5756,5758],{"className":5757,"style":234},[140],[59,5759],{}," 输出 ",[59,5762,5764,5787],{"className":5763},[62],[59,5765,5767],{"className":5766},[66],[68,5768,5769],{"xmlns":70},[72,5770,5771,5784],{},[75,5772,5773,5775,5778],{},[91,5774,2417],{"mathvariant":5378},[461,5776,5777],{},"+",[75,5779,5780,5782],{},[91,5781,5604],{"mathvariant":5378},[91,5783,5607],{"mathvariant":5378},[98,5785,5786],{"encoding":100},"\\mathbf{X} + \\mathbf{PE}",[59,5788,5790,5809],{"className":5789,"ariaHidden":106},[105],[59,5791,5793,5797,5800,5803,5806],{"className":5792},[110],[59,5794],{"className":5795,"style":5796},[114],"height:0.7694em;vertical-align:-0.0833em;",[59,5798,2417],{"className":5799},[119,5423],[59,5801],{"className":5802,"style":1872},[499],[59,5804,5777],{"className":5805},[553],[59,5807],{"className":5808,"style":1872},[499],[59,5810,5812,5816],{"className":5811},[110],[59,5813],{"className":5814,"style":5815},[114],"height:0.6861em;",[59,5817,5819],{"className":5818},[119],[59,5820,5654],{"className":5821},[119,5423],"，矩阵第 ",[59,5824,5826,5839],{"className":5825},[62],[59,5827,5829],{"className":5828},[66],[68,5830,5831],{"xmlns":70},[72,5832,5833,5837],{},[75,5834,5835],{},[91,5836,789],{},[98,5838,789],{"encoding":100},[59,5840,5842],{"className":5841,"ariaHidden":106},[105],[59,5843,5845,5849],{"className":5844},[110],[59,5846],{"className":5847,"style":5848},[114],"height:0.6595em;",[59,5850,789],{"className":5851},[119,192]," 行、第 ",[59,5854,5856,5873],{"className":5855},[62],[59,5857,5859],{"className":5858},[66],[68,5860,5861],{"xmlns":70},[72,5862,5863,5870],{},[75,5864,5865,5867],{},[81,5866,2821],{},[91,5868,5869],{},"j",[98,5871,5872],{"encoding":100},"2j",[59,5874,5876],{"className":5875,"ariaHidden":106},[105],[59,5877,5879,5883,5886],{"className":5878},[110],[59,5880],{"className":5881,"style":5882},[114],"height:0.854em;vertical-align:-0.1944em;",[59,5884,2821],{"className":5885},[119],[59,5887,5869],{"className":5888,"style":5889},[119,192],"margin-right:0.05724em;"," 列和 ",[59,5892,5894,5914],{"className":5893},[62],[59,5895,5897],{"className":5896},[66],[68,5898,5899],{"xmlns":70},[72,5900,5901,5911],{},[75,5902,5903,5905,5907,5909],{},[81,5904,2821],{},[91,5906,5869],{},[461,5908,5777],{},[81,5910,83],{},[98,5912,5913],{"encoding":100},"2j+1",[59,5915,5917,5938],{"className":5916,"ariaHidden":106},[105],[59,5918,5920,5923,5926,5929,5932,5935],{"className":5919},[110],[59,5921],{"className":5922,"style":5882},[114],[59,5924,2821],{"className":5925},[119],[59,5927,5869],{"className":5928,"style":5889},[119,192],[59,5930],{"className":5931,"style":1872},[499],[59,5933,5777],{"className":5934},[553],[59,5936],{"className":5937,"style":1872},[499],[59,5939,5941,5944],{"className":5940},[110],[59,5942],{"className":5943,"style":1766},[114],[59,5945,83],{"className":5946},[119]," 列上的元素为：",[15,5949,5950],{},[59,5951,5953,6031],{"className":5952},[62],[59,5954,5956],{"className":5955},[66],[68,5957,5958],{"xmlns":70},[72,5959,5960,6028],{},[75,5961,5962,5964,5982,5984,5987,5989],{},[91,5963,5604],{},[88,5965,5966,5968],{},[91,5967,5607],{},[75,5969,5970,5972,5974,5976,5978,5980],{},[461,5971,798],{"stretchy":797},[91,5973,789],{},[461,5975,803],{"separator":106},[81,5977,2821],{},[91,5979,5869],{},[461,5981,812],{"stretchy":797},[461,5983,815],{},[91,5985,5986],{},"sin",[461,5988,822],{},[75,5990,5991,5993,6026],{},[461,5992,798],{"fence":106},[78,5994,5995,5997],{},[91,5996,789],{},[465,5998,5999,6002],{},[81,6000,6001],{},"10000",[75,6003,6004,6006,6008,6010],{},[81,6005,2821],{},[91,6007,5869],{},[91,6009,2827],{"mathvariant":818},[88,6011,6012,6014],{},[91,6013,93],{},[75,6015,6016,6018,6020,6022,6024],{},[91,6017,366],{},[91,6019,792],{},[91,6021,93],{},[91,6023,782],{},[91,6025,2903],{},[461,6027,812],{"fence":106},[98,6029,6030],{"encoding":100}," PE_{(i,2j)} = \\sin\\left(\\frac{i}{10000^{2j/d_{model}}}\\right)",[59,6032,6034,6114],{"className":6033,"ariaHidden":106},[105],[59,6035,6037,6041,6044,6105,6108,6111],{"className":6036},[110],[59,6038],{"className":6039,"style":6040},[114],"height:1.0385em;vertical-align:-0.3552em;",[59,6042,5604],{"className":6043,"style":3185},[119,192],[59,6045,6047,6051],{"className":6046},[119],[59,6048,5607],{"className":6049,"style":6050},[119,192],"margin-right:0.05764em;",[59,6052,6054],{"className":6053},[196],[59,6055,6057,6096],{"className":6056},[131,132],[59,6058,6060,6093],{"className":6059},[136],[59,6061,6063],{"className":6062,"style":206},[140],[59,6064,6066,6069],{"style":6065},"top:-2.5198em;margin-left:-0.0576em;margin-right:0.05em;",[59,6067],{"className":6068,"style":540},[148],[59,6070,6072],{"className":6071},[153,154,155,156],[59,6073,6075,6078,6081,6084,6087,6090],{"className":6074},[119,156],[59,6076,798],{"className":6077},[123,156],[59,6079,789],{"className":6080},[119,192,156],[59,6082,803],{"className":6083},[912,156],[59,6085,2821],{"className":6086},[119,156],[59,6088,5869],{"className":6089,"style":5889},[119,192,156],[59,6091,812],{"className":6092},[313,156],[59,6094,227],{"className":6095},[226],[59,6097,6099],{"className":6098},[136],[59,6100,6103],{"className":6101,"style":6102},[140],"height:0.3552em;",[59,6104],{},[59,6106],{"className":6107,"style":500},[499],[59,6109,815],{"className":6110},[504],[59,6112],{"className":6113,"style":500},[499],[59,6115,6117,6120,6123,6126],{"className":6116},[110],[59,6118],{"className":6119,"style":947},[114],[59,6121,5986],{"className":6122},[951],[59,6124],{"className":6125,"style":916},[499],[59,6127,6129,6135,6303],{"className":6128},[962],[59,6130,6132],{"className":6131,"style":967},[123,966],[59,6133,798],{"className":6134},[971,972],[59,6136,6138,6141,6300],{"className":6137},[119],[59,6139],{"className":6140},[123,124],[59,6142,6144],{"className":6143},[78],[59,6145,6147,6291],{"className":6146},[131,132],[59,6148,6150,6288],{"className":6149},[136],[59,6151,6154,6266,6274],{"className":6152,"style":6153},[140],"height:0.8557em;",[59,6155,6157,6160],{"style":6156},"top:-2.5648em;",[59,6158],{"className":6159,"style":149},[148],[59,6161,6163],{"className":6162},[153,154,155,156],[59,6164,6166,6170],{"className":6165},[119,156],[59,6167,6169],{"className":6168},[119,156],"1000",[59,6171,6173,6176],{"className":6172},[119,156],[59,6174,1754],{"className":6175},[119,156],[59,6177,6179],{"className":6178},[196],[59,6180,6182],{"className":6181},[131],[59,6183,6185],{"className":6184},[136],[59,6186,6189],{"className":6187,"style":6188},[140],"height:0.8932em;",[59,6190,6192,6196],{"style":6191},"top:-2.8932em;margin-right:0.0714em;",[59,6193],{"className":6194,"style":6195},[148],"height:2.5357em;",[59,6197,6199],{"className":6198},[153,217,218,156],[59,6200,6202,6205,6208,6211],{"className":6201},[119,156],[59,6203,2821],{"className":6204},[119,156],[59,6206,5869],{"className":6207,"style":5889},[119,192,156],[59,6209,2827],{"className":6210},[119,156],[59,6212,6214,6217],{"className":6213},[119,156],[59,6215,93],{"className":6216},[119,192,156],[59,6218,6220],{"className":6219},[196],[59,6221,6223,6257],{"className":6222},[131,132],[59,6224,6226,6254],{"className":6225},[136],[59,6227,6229],{"className":6228,"style":206},[140],[59,6230,6232,6236],{"style":6231},"top:-2.3448em;margin-left:0em;margin-right:0.1em;",[59,6233],{"className":6234,"style":6235},[148],"height:2.6944em;",[59,6237,6239,6242,6245,6248,6251],{"className":6238},[119,156],[59,6240,366],{"className":6241},[119,192,156],[59,6243,792],{"className":6244},[119,192,156],[59,6246,93],{"className":6247},[119,192,156],[59,6249,782],{"className":6250},[119,192,156],[59,6252,2903],{"className":6253,"style":5356},[119,192,156],[59,6255,227],{"className":6256},[226],[59,6258,6260],{"className":6259},[136],[59,6261,6264],{"className":6262,"style":6263},[140],"height:0.3496em;",[59,6265],{},[59,6267,6268,6271],{"style":274},[59,6269],{"className":6270,"style":149},[148],[59,6272],{"className":6273,"style":282},[281],[59,6275,6276,6279],{"style":285},[59,6277],{"className":6278,"style":149},[148],[59,6280,6282],{"className":6281},[153,154,155,156],[59,6283,6285],{"className":6284},[119,156],[59,6286,789],{"className":6287},[119,192,156],[59,6289,227],{"className":6290},[226],[59,6292,6294],{"className":6293},[136],[59,6295,6298],{"className":6296,"style":6297},[140],"height:0.4352em;",[59,6299],{},[59,6301],{"className":6302},[313,124],[59,6304,6306],{"className":6305,"style":967},[313,966],[59,6307,812],{"className":6308},[971,972],[15,6310,6311],{},[59,6312,6314,6395],{"className":6313},[62],[59,6315,6317],{"className":6316},[66],[68,6318,6319],{"xmlns":70},[72,6320,6321,6392],{},[75,6322,6323,6325,6347,6349,6352,6354],{},[91,6324,5604],{},[88,6326,6327,6329],{},[91,6328,5607],{},[75,6330,6331,6333,6335,6337,6339,6341,6343,6345],{},[461,6332,798],{"stretchy":797},[91,6334,789],{},[461,6336,803],{"separator":106},[81,6338,2821],{},[91,6340,5869],{},[461,6342,5777],{},[81,6344,83],{},[461,6346,812],{"stretchy":797},[461,6348,815],{},[91,6350,6351],{},"cos",[461,6353,822],{},[75,6355,6356,6358,6390],{},[461,6357,798],{"fence":106},[78,6359,6360,6362],{},[91,6361,789],{},[465,6363,6364,6366],{},[81,6365,6001],{},[75,6367,6368,6370,6372,6374],{},[81,6369,2821],{},[91,6371,5869],{},[91,6373,2827],{"mathvariant":818},[88,6375,6376,6378],{},[91,6377,93],{},[75,6379,6380,6382,6384,6386,6388],{},[91,6381,366],{},[91,6383,792],{},[91,6385,93],{},[91,6387,782],{},[91,6389,2903],{},[461,6391,812],{"fence":106},[98,6393,6394],{"encoding":100}," PE_{(i,2j+1)} = \\cos\\left(\\frac{i}{10000^{2j/d_{model}}}\\right)",[59,6396,6398,6480],{"className":6397,"ariaHidden":106},[105],[59,6399,6401,6404,6407,6471,6474,6477],{"className":6400},[110],[59,6402],{"className":6403,"style":6040},[114],[59,6405,5604],{"className":6406,"style":3185},[119,192],[59,6408,6410,6413],{"className":6409},[119],[59,6411,5607],{"className":6412,"style":6050},[119,192],[59,6414,6416],{"className":6415},[196],[59,6417,6419,6463],{"className":6418},[131,132],[59,6420,6422,6460],{"className":6421},[136],[59,6423,6425],{"className":6424,"style":206},[140],[59,6426,6427,6430],{"style":6065},[59,6428],{"className":6429,"style":540},[148],[59,6431,6433],{"className":6432},[153,154,155,156],[59,6434,6436,6439,6442,6445,6448,6451,6454,6457],{"className":6435},[119,156],[59,6437,798],{"className":6438},[123,156],[59,6440,789],{"className":6441},[119,192,156],[59,6443,803],{"className":6444},[912,156],[59,6446,2821],{"className":6447},[119,156],[59,6449,5869],{"className":6450,"style":5889},[119,192,156],[59,6452,5777],{"className":6453},[553,156],[59,6455,83],{"className":6456},[119,156],[59,6458,812],{"className":6459},[313,156],[59,6461,227],{"className":6462},[226],[59,6464,6466],{"className":6465},[136],[59,6467,6469],{"className":6468,"style":6102},[140],[59,6470],{},[59,6472],{"className":6473,"style":500},[499],[59,6475,815],{"className":6476},[504],[59,6478],{"className":6479,"style":500},[499],[59,6481,6483,6486,6489,6492],{"className":6482},[110],[59,6484],{"className":6485,"style":947},[114],[59,6487,6351],{"className":6488},[951],[59,6490],{"className":6491,"style":916},[499],[59,6493,6495,6501,6659],{"className":6494},[962],[59,6496,6498],{"className":6497,"style":967},[123,966],[59,6499,798],{"className":6500},[971,972],[59,6502,6504,6507,6656],{"className":6503},[119],[59,6505],{"className":6506},[123,124],[59,6508,6510],{"className":6509},[78],[59,6511,6513,6648],{"className":6512},[131,132],[59,6514,6516,6645],{"className":6515},[136],[59,6517,6519,6623,6631],{"className":6518,"style":6153},[140],[59,6520,6521,6524],{"style":6156},[59,6522],{"className":6523,"style":149},[148],[59,6525,6527],{"className":6526},[153,154,155,156],[59,6528,6530,6533],{"className":6529},[119,156],[59,6531,6169],{"className":6532},[119,156],[59,6534,6536,6539],{"className":6535},[119,156],[59,6537,1754],{"className":6538},[119,156],[59,6540,6542],{"className":6541},[196],[59,6543,6545],{"className":6544},[131],[59,6546,6548],{"className":6547},[136],[59,6549,6551],{"className":6550,"style":6188},[140],[59,6552,6553,6556],{"style":6191},[59,6554],{"className":6555,"style":6195},[148],[59,6557,6559],{"className":6558},[153,217,218,156],[59,6560,6562,6565,6568,6571],{"className":6561},[119,156],[59,6563,2821],{"className":6564},[119,156],[59,6566,5869],{"className":6567,"style":5889},[119,192,156],[59,6569,2827],{"className":6570},[119,156],[59,6572,6574,6577],{"className":6573},[119,156],[59,6575,93],{"className":6576},[119,192,156],[59,6578,6580],{"className":6579},[196],[59,6581,6583,6615],{"className":6582},[131,132],[59,6584,6586,6612],{"className":6585},[136],[59,6587,6589],{"className":6588,"style":206},[140],[59,6590,6591,6594],{"style":6231},[59,6592],{"className":6593,"style":6235},[148],[59,6595,6597,6600,6603,6606,6609],{"className":6596},[119,156],[59,6598,366],{"className":6599},[119,192,156],[59,6601,792],{"className":6602},[119,192,156],[59,6604,93],{"className":6605},[119,192,156],[59,6607,782],{"className":6608},[119,192,156],[59,6610,2903],{"className":6611,"style":5356},[119,192,156],[59,6613,227],{"className":6614},[226],[59,6616,6618],{"className":6617},[136],[59,6619,6621],{"className":6620,"style":6263},[140],[59,6622],{},[59,6624,6625,6628],{"style":274},[59,6626],{"className":6627,"style":149},[148],[59,6629],{"className":6630,"style":282},[281],[59,6632,6633,6636],{"style":285},[59,6634],{"className":6635,"style":149},[148],[59,6637,6639],{"className":6638},[153,154,155,156],[59,6640,6642],{"className":6641},[119,156],[59,6643,789],{"className":6644},[119,192,156],[59,6646,227],{"className":6647},[226],[59,6649,6651],{"className":6650},[136],[59,6652,6654],{"className":6653,"style":6297},[140],[59,6655],{},[59,6657],{"className":6658},[313,124],[59,6660,6662],{"className":6661,"style":967},[313,966],[59,6663,812],{"className":6664},[971,972],[15,6666,6667,6668,6696,6697,6725,6726,6761,6762,6818,6819,6847,6848,6941,6942,7021],{},"其中，",[59,6669,6671,6684],{"className":6670},[62],[59,6672,6674],{"className":6673},[66],[68,6675,6676],{"xmlns":70},[72,6677,6678,6682],{},[75,6679,6680],{},[91,6681,789],{},[98,6683,789],{"encoding":100},[59,6685,6687],{"className":6686,"ariaHidden":106},[105],[59,6688,6690,6693],{"className":6689},[110],[59,6691],{"className":6692,"style":5848},[114],[59,6694,789],{"className":6695},[119,192]," 表示位置，",[59,6698,6700,6713],{"className":6699},[62],[59,6701,6703],{"className":6702},[66],[68,6704,6705],{"xmlns":70},[72,6706,6707,6711],{},[75,6708,6709],{},[91,6710,5869],{},[98,6712,5869],{"encoding":100},[59,6714,6716],{"className":6715,"ariaHidden":106},[105],[59,6717,6719,6722],{"className":6718},[110],[59,6720],{"className":6721,"style":5882},[114],[59,6723,5869],{"className":6724,"style":5889},[119,192]," 表示维度。也就是说，位置编码的每一个维度都对应一个正弦曲线。其波长从 ",[59,6727,6729,6746],{"className":6728},[62],[59,6730,6732],{"className":6731},[66],[68,6733,6734],{"xmlns":70},[72,6735,6736,6743],{},[75,6737,6738,6740],{},[81,6739,2821],{},[91,6741,6742],{},"π",[98,6744,6745],{"encoding":100},"2\\pi",[59,6747,6749],{"className":6748,"ariaHidden":106},[105],[59,6750,6752,6755,6758],{"className":6751},[110],[59,6753],{"className":6754,"style":1766},[114],[59,6756,2821],{"className":6757},[119],[59,6759,6742],{"className":6760,"style":441},[119,192]," 到 ",[59,6763,6765,6785],{"className":6764},[62],[59,6766,6768],{"className":6767},[66],[68,6769,6770],{"xmlns":70},[72,6771,6772,6782],{},[75,6773,6774,6776,6778,6780],{},[81,6775,6001],{},[461,6777,1816],{},[81,6779,2821],{},[91,6781,6742],{},[98,6783,6784],{"encoding":100},"10000 \\cdot 2\\pi",[59,6786,6788,6806],{"className":6787,"ariaHidden":106},[105],[59,6789,6791,6794,6797,6800,6803],{"className":6790},[110],[59,6792],{"className":6793,"style":1766},[114],[59,6795,6001],{"className":6796},[119],[59,6798],{"className":6799,"style":1872},[499],[59,6801,1816],{"className":6802},[553],[59,6804],{"className":6805,"style":1872},[499],[59,6807,6809,6812,6815],{"className":6808},[110],[59,6810],{"className":6811,"style":1766},[114],[59,6813,2821],{"className":6814},[119],[59,6816,6742],{"className":6817,"style":441},[119,192]," 按几何级数增长。我们选择这个函数，是因为我们假设它能够使模型更容易学习关注相对位置，因为对于任意固定的偏移量 ",[59,6820,6822,6835],{"className":6821},[62],[59,6823,6825],{"className":6824},[66],[68,6826,6827],{"xmlns":70},[72,6828,6829,6833],{},[75,6830,6831],{},[91,6832,96],{},[98,6834,96],{"encoding":100},[59,6836,6838],{"className":6837,"ariaHidden":106},[105],[59,6839,6841,6844],{"className":6840},[110],[59,6842],{"className":6843,"style":407},[114],[59,6845,96],{"className":6846,"style":222},[119,192],"，",[59,6849,6851,6877],{"className":6850},[62],[59,6852,6854],{"className":6853},[66],[68,6855,6856],{"xmlns":70},[72,6857,6858,6874],{},[75,6859,6860,6862],{},[91,6861,5604],{},[88,6863,6864,6866],{},[91,6865,5607],{},[75,6867,6868,6870,6872],{},[91,6869,789],{},[461,6871,5777],{},[91,6873,96],{},[98,6875,6876],{"encoding":100},"PE_{i+k}",[59,6878,6880],{"className":6879,"ariaHidden":106},[105],[59,6881,6883,6887,6890],{"className":6882},[110],[59,6884],{"className":6885,"style":6886},[114],"height:0.8917em;vertical-align:-0.2083em;",[59,6888,5604],{"className":6889,"style":3185},[119,192],[59,6891,6893,6896],{"className":6892},[119],[59,6894,5607],{"className":6895,"style":6050},[119,192],[59,6897,6899],{"className":6898},[196],[59,6900,6902,6932],{"className":6901},[131,132],[59,6903,6905,6929],{"className":6904},[136],[59,6906,6908],{"className":6907,"style":1248},[140],[59,6909,6911,6914],{"style":6910},"top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;",[59,6912],{"className":6913,"style":540},[148],[59,6915,6917],{"className":6916},[153,154,155,156],[59,6918,6920,6923,6926],{"className":6919},[119,156],[59,6921,789],{"className":6922},[119,192,156],[59,6924,5777],{"className":6925},[553,156],[59,6927,96],{"className":6928,"style":222},[119,192,156],[59,6930,227],{"className":6931},[226],[59,6933,6935],{"className":6934},[136],[59,6936,6939],{"className":6937,"style":6938},[140],"height:0.2083em;",[59,6940],{}," 都可以表示为 ",[59,6943,6945,6965],{"className":6944},[62],[59,6946,6948],{"className":6947},[66],[68,6949,6950],{"xmlns":70},[72,6951,6952,6962],{},[75,6953,6954,6956],{},[91,6955,5604],{},[88,6957,6958,6960],{},[91,6959,5607],{},[91,6961,789],{},[98,6963,6964],{"encoding":100},"PE_{i}",[59,6966,6968],{"className":6967,"ariaHidden":106},[105],[59,6969,6971,6975,6978],{"className":6970},[110],[59,6972],{"className":6973,"style":6974},[114],"height:0.8333em;vertical-align:-0.15em;",[59,6976,5604],{"className":6977,"style":3185},[119,192],[59,6979,6981,6984],{"className":6980},[119],[59,6982,5607],{"className":6983,"style":6050},[119,192],[59,6985,6987],{"className":6986},[196],[59,6988,6990,7013],{"className":6989},[131,132],[59,6991,6993,7010],{"className":6992},[136],[59,6994,6996],{"className":6995,"style":2032},[140],[59,6997,6998,7001],{"style":6910},[59,6999],{"className":7000,"style":540},[148],[59,7002,7004],{"className":7003},[153,154,155,156],[59,7005,7007],{"className":7006},[119,156],[59,7008,789],{"className":7009},[119,192,156],[59,7011,227],{"className":7012},[226],[59,7014,7016],{"className":7015},[136],[59,7017,7019],{"className":7018,"style":1270},[140],[59,7020],{}," 的线性函数。",[2390,7023,7025],{"className":2392,"code":7024,"language":2394,"meta":2395,"style":2395},"class PositionalEncoding(nn.Block):\n    \"\"\"位置编码\"\"\"\n    def __init__(self, num_hiddens, dropout, max_len=1000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(dropout)\n        # 创建一个足够长的P\n        self.P = np.zeros((1, max_len, num_hiddens))\n        X = np.arange(max_len).reshape(-1, 1) / np.power(\n            10000, np.arange(0, num_hiddens, 2) / num_hiddens)\n        self.P[:, :, 0::2] = np.sin(X)\n        self.P[:, :, 1::2] = np.cos(X)\n\n    def forward(self, X):\n        X = X + self.P[:, :X.shape[1], :].as_in_ctx(X.ctx)\n        return self.dropout(X)\n",[2397,7026,7027,7045,7050,7079,7095,7105,7110,7127,7152,7174,7206,7233,7237,7253,7286],{"__ignoreMap":2395},[59,7028,7029,7031,7034,7036,7038,7040,7043],{"class":2401,"line":2402},[59,7030,2630],{"class":2405},[59,7032,7033],{"class":2633}," PositionalEncoding",[59,7035,798],{"class":2413},[59,7037,2640],{"class":2639},[59,7039,864],{"class":2413},[59,7041,7042],{"class":2639},"Block",[59,7044,2426],{"class":2413},[59,7046,7047],{"class":2401,"line":2429},[59,7048,7049],{"class":2432},"    \"\"\"位置编码\"\"\"\n",[59,7051,7052,7054,7056,7058,7060,7062,7064,7066,7068,7070,7073,7075,7077],{"class":2401,"line":2436},[59,7053,2657],{"class":2405},[59,7055,2660],{"class":2455},[59,7057,798],{"class":2413},[59,7059,2666],{"class":2665},[59,7061,2420],{"class":2413},[59,7063,4862],{"class":2416},[59,7065,2420],{"class":2413},[59,7067,2671],{"class":2416},[59,7069,2420],{"class":2413},[59,7071,7072],{"class":2416},"max_len",[59,7074,815],{"class":2405},[59,7076,6169],{"class":2455},[59,7078,2426],{"class":2413},[59,7080,7081,7083,7086,7088,7090,7092],{"class":2401,"line":2443},[59,7082,2687],{"class":2686},[59,7084,7085],{"class":2413},"(PositionalEncoding, ",[59,7087,2666],{"class":2693},[59,7089,2696],{"class":2413},[59,7091,2699],{"class":2455},[59,7093,7094],{"class":2413},"()\n",[59,7096,7097,7099,7101,7103],{"class":2401,"line":2462},[59,7098,2711],{"class":2693},[59,7100,2714],{"class":2413},[59,7102,815],{"class":2405},[59,7104,2719],{"class":2413},[59,7106,7107],{"class":2401,"line":2483},[59,7108,7109],{"class":2439},"        # 创建一个足够长的P\n",[59,7111,7112,7114,7117,7119,7122,7124],{"class":2401,"line":2491},[59,7113,2711],{"class":2693},[59,7115,7116],{"class":2413},".P ",[59,7118,815],{"class":2405},[59,7120,7121],{"class":2413}," np.zeros((",[59,7123,83],{"class":2455},[59,7125,7126],{"class":2413},", max_len, num_hiddens))\n",[59,7128,7129,7131,7133,7136,7138,7140,7142,7144,7147,7149],{"class":2401,"line":2502},[59,7130,2569],{"class":2413},[59,7132,815],{"class":2405},[59,7134,7135],{"class":2413}," np.arange(max_len).reshape(",[59,7137,2553],{"class":2405},[59,7139,83],{"class":2455},[59,7141,2420],{"class":2413},[59,7143,83],{"class":2455},[59,7145,7146],{"class":2413},") ",[59,7148,2827],{"class":2405},[59,7150,7151],{"class":2413}," np.power(\n",[59,7153,7154,7157,7160,7162,7165,7167,7169,7171],{"class":2401,"line":2519},[59,7155,7156],{"class":2455},"            10000",[59,7158,7159],{"class":2413},", np.arange(",[59,7161,1754],{"class":2455},[59,7163,7164],{"class":2413},", num_hiddens, ",[59,7166,2821],{"class":2455},[59,7168,7146],{"class":2413},[59,7170,2827],{"class":2405},[59,7172,7173],{"class":2413}," num_hiddens)\n",[59,7175,7176,7178,7181,7185,7187,7189,7191,7193,7196,7198,7201,7203],{"class":2401,"line":2535},[59,7177,2711],{"class":2693},[59,7179,7180],{"class":2413},".P[",[59,7182,7184],{"class":7183},"sDoOe",":",[59,7186,2420],{"class":2413},[59,7188,7184],{"class":7183},[59,7190,2420],{"class":2413},[59,7192,1754],{"class":2455},[59,7194,7195],{"class":7183},"::",[59,7197,2821],{"class":2455},[59,7199,7200],{"class":2413},"] ",[59,7202,815],{"class":2405},[59,7204,7205],{"class":2413}," np.sin(X)\n",[59,7207,7208,7210,7212,7214,7216,7218,7220,7222,7224,7226,7228,7230],{"class":2401,"line":2543},[59,7209,2711],{"class":2693},[59,7211,7180],{"class":2413},[59,7213,7184],{"class":7183},[59,7215,2420],{"class":2413},[59,7217,7184],{"class":7183},[59,7219,2420],{"class":2413},[59,7221,83],{"class":2455},[59,7223,7195],{"class":7183},[59,7225,2821],{"class":2455},[59,7227,7200],{"class":2413},[59,7229,815],{"class":2405},[59,7231,7232],{"class":2413}," np.cos(X)\n",[59,7234,7235],{"class":2401,"line":2560},[59,7236,2725],{"emptyLinePlaceholder":2724},[59,7238,7239,7241,7243,7245,7247,7249,7251],{"class":2401,"line":2566},[59,7240,2657],{"class":2405},[59,7242,2752],{"class":2409},[59,7244,798],{"class":2413},[59,7246,2666],{"class":2665},[59,7248,2420],{"class":2413},[59,7250,2417],{"class":2416},[59,7252,2426],{"class":2413},[59,7254,7255,7257,7259,7262,7264,7266,7268,7270,7272,7274,7277,7279,7281,7283],{"class":2401,"line":2591},[59,7256,2569],{"class":2413},[59,7258,815],{"class":2405},[59,7260,7261],{"class":2413}," X ",[59,7263,5777],{"class":2405},[59,7265,5220],{"class":2693},[59,7267,7180],{"class":2413},[59,7269,7184],{"class":7183},[59,7271,2420],{"class":2413},[59,7273,7184],{"class":7183},[59,7275,7276],{"class":2413},"X.shape[",[59,7278,83],{"class":2455},[59,7280,4794],{"class":2413},[59,7282,7184],{"class":7183},[59,7284,7285],{"class":2413},"].as_in_ctx(X.ctx)\n",[59,7287,7288,7290,7292],{"class":2401,"line":2604},[59,7289,2465],{"class":2405},[59,7291,5220],{"class":2693},[59,7293,7294],{"class":2413},".dropout(X)\n",[11,7296,7297],{"id":7297},"前馈神经网络",[15,7299,7300],{},"除了注意力子层外，我们的编码器和解码器的每一层中都包含一个全连接的前馈神经网络，该网络分别且相同地应用于每个位置。这包括两个线性变换，中间使用ReLU激活函数。",[15,7302,7303],{},[59,7304,7306,7379],{"className":7305},[62],[59,7307,7309],{"className":7308},[66],[68,7310,7311],{"xmlns":70},[72,7312,7313,7376],{},[75,7314,7315,7318,7320,7323,7325,7328,7330,7332,7335,7337,7339,7341,7343,7345,7351,7353,7360,7362,7368,7370],{},[91,7316,7317],{},"F",[91,7319,7317],{},[91,7321,7322],{},"N",[461,7324,798],{"stretchy":797},[91,7326,7327],{},"x",[461,7329,812],{"stretchy":797},[461,7331,815],{},[91,7333,7334],{},"max",[461,7336,822],{},[461,7338,798],{"stretchy":797},[81,7340,1754],{},[461,7342,803],{"separator":106},[91,7344,7327],{},[88,7346,7347,7349],{},[91,7348,2993],{},[81,7350,83],{},[461,7352,5777],{},[88,7354,7355,7358],{},[91,7356,7357],{},"b",[81,7359,83],{},[461,7361,812],{"stretchy":797},[88,7363,7364,7366],{},[91,7365,2993],{},[81,7367,2821],{},[461,7369,5777],{},[88,7371,7372,7374],{},[91,7373,7357],{},[81,7375,2821],{},[98,7377,7378],{"encoding":100}," FFN(x) = \\max(0,xW_1+b_1)W_2+b_2",[59,7380,7382,7416,7490,7588],{"className":7381,"ariaHidden":106},[105],[59,7383,7385,7388,7391,7394,7398,7401,7404,7407,7410,7413],{"className":7384},[110],[59,7386],{"className":7387,"style":877},[114],[59,7389,7317],{"className":7390,"style":3185},[119,192],[59,7392,7317],{"className":7393,"style":3185},[119,192],[59,7395,7322],{"className":7396,"style":7397},[119,192],"margin-right:0.10903em;",[59,7399,798],{"className":7400},[123],[59,7402,7327],{"className":7403},[119,192],[59,7405,812],{"className":7406},[313],[59,7408],{"className":7409,"style":500},[499],[59,7411,815],{"className":7412},[504],[59,7414],{"className":7415,"style":500},[499],[59,7417,7419,7422,7425,7428,7431,7434,7437,7440,7481,7484,7487],{"className":7418},[110],[59,7420],{"className":7421,"style":877},[114],[59,7423,7334],{"className":7424},[951],[59,7426,798],{"className":7427},[123],[59,7429,1754],{"className":7430},[119],[59,7432,803],{"className":7433},[912],[59,7435],{"className":7436,"style":916},[499],[59,7438,7327],{"className":7439},[119,192],[59,7441,7443,7446],{"className":7442},[119],[59,7444,2993],{"className":7445,"style":3185},[119,192],[59,7447,7449],{"className":7448},[196],[59,7450,7452,7473],{"className":7451},[131,132],[59,7453,7455,7470],{"className":7454},[136],[59,7456,7458],{"className":7457,"style":3091},[140],[59,7459,7461,7464],{"style":7460},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[59,7462],{"className":7463,"style":540},[148],[59,7465,7467],{"className":7466},[153,154,155,156],[59,7468,83],{"className":7469},[119,156],[59,7471,227],{"className":7472},[226],[59,7474,7476],{"className":7475},[136],[59,7477,7479],{"className":7478,"style":1270},[140],[59,7480],{},[59,7482],{"className":7483,"style":1872},[499],[59,7485,5777],{"className":7486},[553],[59,7488],{"className":7489,"style":1872},[499],[59,7491,7493,7496,7536,7539,7579,7582,7585],{"className":7492},[110],[59,7494],{"className":7495,"style":877},[114],[59,7497,7499,7502],{"className":7498},[119],[59,7500,7357],{"className":7501},[119,192],[59,7503,7505],{"className":7504},[196],[59,7506,7508,7528],{"className":7507},[131,132],[59,7509,7511,7525],{"className":7510},[136],[59,7512,7514],{"className":7513,"style":3091},[140],[59,7515,7516,7519],{"style":1251},[59,7517],{"className":7518,"style":540},[148],[59,7520,7522],{"className":7521},[153,154,155,156],[59,7523,83],{"className":7524},[119,156],[59,7526,227],{"className":7527},[226],[59,7529,7531],{"className":7530},[136],[59,7532,7534],{"className":7533,"style":1270},[140],[59,7535],{},[59,7537,812],{"className":7538},[313],[59,7540,7542,7545],{"className":7541},[119],[59,7543,2993],{"className":7544,"style":3185},[119,192],[59,7546,7548],{"className":7547},[196],[59,7549,7551,7571],{"className":7550},[131,132],[59,7552,7554,7568],{"className":7553},[136],[59,7555,7557],{"className":7556,"style":3091},[140],[59,7558,7559,7562],{"style":7460},[59,7560],{"className":7561,"style":540},[148],[59,7563,7565],{"className":7564},[153,154,155,156],[59,7566,2821],{"className":7567},[119,156],[59,7569,227],{"className":7570},[226],[59,7572,7574],{"className":7573},[136],[59,7575,7577],{"className":7576,"style":1270},[140],[59,7578],{},[59,7580],{"className":7581,"style":1872},[499],[59,7583,5777],{"className":7584},[553],[59,7586],{"className":7587,"style":1872},[499],[59,7589,7591,7594],{"className":7590},[110],[59,7592],{"className":7593,"style":1229},[114],[59,7595,7597,7600],{"className":7596},[119],[59,7598,7357],{"className":7599},[119,192],[59,7601,7603],{"className":7602},[196],[59,7604,7606,7626],{"className":7605},[131,132],[59,7607,7609,7623],{"className":7608},[136],[59,7610,7612],{"className":7611,"style":3091},[140],[59,7613,7614,7617],{"style":1251},[59,7615],{"className":7616,"style":540},[148],[59,7618,7620],{"className":7619},[153,154,155,156],[59,7621,2821],{"className":7622},[119,156],[59,7624,227],{"className":7625},[226],[59,7627,7629],{"className":7628},[136],[59,7630,7632],{"className":7631,"style":1270},[140],[59,7633],{},[2390,7635,7637],{"className":2392,"code":7636,"language":2394,"meta":2395,"style":2395},"class PositionWiseFFN(nn.Module):\n    \"\"\"基于位置的前馈网络\"\"\"\n    def __init__(self, ffn_num_input, ffn_num_hiddens, ffn_num_outputs,\n                 **kwargs):\n        super(PositionWiseFFN, self).__init__(**kwargs)\n        self.dense1 = nn.Linear(ffn_num_input, ffn_num_hiddens)\n        self.relu = nn.ReLU()\n        self.dense2 = nn.Linear(ffn_num_hiddens, ffn_num_outputs)\n\n    def forward(self, X):\n        return self.dense2(self.relu(self.dense1(X)))\n",[2397,7638,7639,7656,7661,7688,7697,7716,7728,7740,7752,7756,7772],{"__ignoreMap":2395},[59,7640,7641,7643,7646,7648,7650,7652,7654],{"class":2401,"line":2402},[59,7642,2630],{"class":2405},[59,7644,7645],{"class":2633}," PositionWiseFFN",[59,7647,798],{"class":2413},[59,7649,2640],{"class":2639},[59,7651,864],{"class":2413},[59,7653,2645],{"class":2639},[59,7655,2426],{"class":2413},[59,7657,7658],{"class":2401,"line":2429},[59,7659,7660],{"class":2432},"    \"\"\"基于位置的前馈网络\"\"\"\n",[59,7662,7663,7665,7667,7669,7671,7673,7676,7678,7681,7683,7686],{"class":2401,"line":2436},[59,7664,2657],{"class":2405},[59,7666,2660],{"class":2455},[59,7668,798],{"class":2413},[59,7670,2666],{"class":2665},[59,7672,2420],{"class":2413},[59,7674,7675],{"class":2416},"ffn_num_input",[59,7677,2420],{"class":2413},[59,7679,7680],{"class":2416},"ffn_num_hiddens",[59,7682,2420],{"class":2413},[59,7684,7685],{"class":2416},"ffn_num_outputs",[59,7687,3826],{"class":2413},[59,7689,7690,7693,7695],{"class":2401,"line":2443},[59,7691,7692],{"class":2405},"                 **",[59,7694,2679],{"class":2416},[59,7696,2426],{"class":2413},[59,7698,7699,7701,7704,7706,7708,7710,7712,7714],{"class":2401,"line":2462},[59,7700,2687],{"class":2686},[59,7702,7703],{"class":2413},"(PositionWiseFFN, ",[59,7705,2666],{"class":2693},[59,7707,2696],{"class":2413},[59,7709,2699],{"class":2455},[59,7711,798],{"class":2413},[59,7713,2676],{"class":2405},[59,7715,2706],{"class":2413},[59,7717,7718,7720,7723,7725],{"class":2401,"line":2483},[59,7719,2711],{"class":2693},[59,7721,7722],{"class":2413},".dense1 ",[59,7724,815],{"class":2405},[59,7726,7727],{"class":2413}," nn.Linear(ffn_num_input, ffn_num_hiddens)\n",[59,7729,7730,7732,7735,7737],{"class":2401,"line":2491},[59,7731,2711],{"class":2693},[59,7733,7734],{"class":2413},".relu ",[59,7736,815],{"class":2405},[59,7738,7739],{"class":2413}," nn.ReLU()\n",[59,7741,7742,7744,7747,7749],{"class":2401,"line":2502},[59,7743,2711],{"class":2693},[59,7745,7746],{"class":2413},".dense2 ",[59,7748,815],{"class":2405},[59,7750,7751],{"class":2413}," nn.Linear(ffn_num_hiddens, ffn_num_outputs)\n",[59,7753,7754],{"class":2401,"line":2519},[59,7755,2725],{"emptyLinePlaceholder":2724},[59,7757,7758,7760,7762,7764,7766,7768,7770],{"class":2401,"line":2535},[59,7759,2657],{"class":2405},[59,7761,2752],{"class":2409},[59,7763,798],{"class":2413},[59,7765,2666],{"class":2665},[59,7767,2420],{"class":2413},[59,7769,2417],{"class":2416},[59,7771,2426],{"class":2413},[59,7773,7774,7776,7778,7781,7783,7786,7788],{"class":2401,"line":2543},[59,7775,2465],{"class":2405},[59,7777,5220],{"class":2693},[59,7779,7780],{"class":2413},".dense2(",[59,7782,2666],{"class":2693},[59,7784,7785],{"class":2413},".relu(",[59,7787,2666],{"class":2693},[59,7789,7790],{"class":2413},".dense1(X)))\n",[11,7792,7793],{"id":7793},"残差连接和层规范化",[15,7795,7796],{},"我们在每个子层之间加入了一个残差连接和层归一化。",[15,7798,7799,7800,4568,7925,7988],{},"因此，每个子层的输出都是 ",[59,7801,7803,7871],{"className":7802},[62],[59,7804,7806],{"className":7805},[66],[68,7807,7808],{"xmlns":70},[72,7809,7810,7868],{},[75,7811,7812,7835,7837,7839,7841,7860,7862,7864,7866],{},[75,7813,7814,7817,7819,7822,7824,7827,7829,7831,7833],{},[91,7815,7816],{"mathvariant":818},"L",[91,7818,22],{"mathvariant":818},[91,7820,7821],{"mathvariant":818},"y",[91,7823,782],{"mathvariant":818},[91,7825,7826],{"mathvariant":818},"r",[91,7828,7322],{"mathvariant":818},[91,7830,792],{"mathvariant":818},[91,7832,7826],{"mathvariant":818},[91,7834,366],{"mathvariant":818},[461,7836,798],{"stretchy":797},[91,7838,7327],{},[461,7840,5777],{},[75,7842,7843,7846,7848,7850,7852,7854,7856,7858],{},[91,7844,7845],{"mathvariant":818},"S",[91,7847,2900],{"mathvariant":818},[91,7849,7357],{"mathvariant":818},[91,7851,2903],{"mathvariant":818},[91,7853,22],{"mathvariant":818},[91,7855,7821],{"mathvariant":818},[91,7857,782],{"mathvariant":818},[91,7859,7826],{"mathvariant":818},[461,7861,798],{"stretchy":797},[91,7863,7327],{},[461,7865,812],{"stretchy":797},[461,7867,812],{"stretchy":797},[98,7869,7870],{"encoding":100},"\\mathrm{LayerNorm}(x + \\mathrm{Sublayer}(x))",[59,7872,7874,7902],{"className":7873,"ariaHidden":106},[105],[59,7875,7877,7880,7887,7890,7893,7896,7899],{"className":7876},[110],[59,7878],{"className":7879,"style":877},[114],[59,7881,7883],{"className":7882},[119],[59,7884,7886],{"className":7885},[119,955],"LayerNorm",[59,7888,798],{"className":7889},[123],[59,7891,7327],{"className":7892},[119,192],[59,7894],{"className":7895,"style":1872},[499],[59,7897,5777],{"className":7898},[553],[59,7900],{"className":7901,"style":1872},[499],[59,7903,7905,7908,7915,7918,7921],{"className":7904},[110],[59,7906],{"className":7907,"style":877},[114],[59,7909,7911],{"className":7910},[119],[59,7912,7914],{"className":7913},[119,955],"Sublayer",[59,7916,798],{"className":7917},[123],[59,7919,7327],{"className":7920},[119,192],[59,7922,7924],{"className":7923},[313],"))",[59,7926,7928,7964],{"className":7927},[62],[59,7929,7931],{"className":7930},[66],[68,7932,7933],{"xmlns":70},[72,7934,7935,7961],{},[75,7936,7937,7955,7957,7959],{},[75,7938,7939,7941,7943,7945,7947,7949,7951,7953],{},[91,7940,7845],{"mathvariant":818},[91,7942,2900],{"mathvariant":818},[91,7944,7357],{"mathvariant":818},[91,7946,2903],{"mathvariant":818},[91,7948,22],{"mathvariant":818},[91,7950,7821],{"mathvariant":818},[91,7952,782],{"mathvariant":818},[91,7954,7826],{"mathvariant":818},[461,7956,798],{"stretchy":797},[91,7958,7327],{},[461,7960,812],{"stretchy":797},[98,7962,7963],{"encoding":100},"\\mathrm{Sublayer}(x)",[59,7965,7967],{"className":7966,"ariaHidden":106},[105],[59,7968,7970,7973,7979,7982,7985],{"className":7969},[110],[59,7971],{"className":7972,"style":877},[114],[59,7974,7976],{"className":7975},[119],[59,7977,7914],{"className":7978},[119,955],[59,7980,798],{"className":7981},[123],[59,7983,7327],{"className":7984},[119,192],[59,7986,812],{"className":7987},[313]," 是由该子层本身实现的函数。我们在每个子层的输出上应用 dropout，然后再将其加到子层输入并进行归一化。",[2390,7990,7992],{"className":2392,"code":7991,"language":2394,"meta":2395,"style":2395},"class AddNorm(nn.Module):\n    \"\"\"残差连接后进行层规范化\"\"\"\n    def __init__(self, normalized_shape, dropout, **kwargs):\n        super(AddNorm, self).__init__(**kwargs)\n        self.dropout = nn.Dropout(dropout)\n        self.ln = nn.LayerNorm(normalized_shape)\n\n    def forward(self, X, Y):\n        return self.ln(self.dropout(Y) + X)\n",[2397,7993,7994,8011,8016,8043,8062,8072,8084,8088,8109],{"__ignoreMap":2395},[59,7995,7996,7998,8001,8003,8005,8007,8009],{"class":2401,"line":2402},[59,7997,2630],{"class":2405},[59,7999,8000],{"class":2633}," AddNorm",[59,8002,798],{"class":2413},[59,8004,2640],{"class":2639},[59,8006,864],{"class":2413},[59,8008,2645],{"class":2639},[59,8010,2426],{"class":2413},[59,8012,8013],{"class":2401,"line":2429},[59,8014,8015],{"class":2432},"    \"\"\"残差连接后进行层规范化\"\"\"\n",[59,8017,8018,8020,8022,8024,8026,8028,8031,8033,8035,8037,8039,8041],{"class":2401,"line":2436},[59,8019,2657],{"class":2405},[59,8021,2660],{"class":2455},[59,8023,798],{"class":2413},[59,8025,2666],{"class":2665},[59,8027,2420],{"class":2413},[59,8029,8030],{"class":2416},"normalized_shape",[59,8032,2420],{"class":2413},[59,8034,2671],{"class":2416},[59,8036,2420],{"class":2413},[59,8038,2676],{"class":2405},[59,8040,2679],{"class":2416},[59,8042,2426],{"class":2413},[59,8044,8045,8047,8050,8052,8054,8056,8058,8060],{"class":2401,"line":2443},[59,8046,2687],{"class":2686},[59,8048,8049],{"class":2413},"(AddNorm, ",[59,8051,2666],{"class":2693},[59,8053,2696],{"class":2413},[59,8055,2699],{"class":2455},[59,8057,798],{"class":2413},[59,8059,2676],{"class":2405},[59,8061,2706],{"class":2413},[59,8063,8064,8066,8068,8070],{"class":2401,"line":2462},[59,8065,2711],{"class":2693},[59,8067,2714],{"class":2413},[59,8069,815],{"class":2405},[59,8071,2719],{"class":2413},[59,8073,8074,8076,8079,8081],{"class":2401,"line":2483},[59,8075,2711],{"class":2693},[59,8077,8078],{"class":2413},".ln ",[59,8080,815],{"class":2405},[59,8082,8083],{"class":2413}," nn.LayerNorm(normalized_shape)\n",[59,8085,8086],{"class":2401,"line":2491},[59,8087,2725],{"emptyLinePlaceholder":2724},[59,8089,8090,8092,8094,8096,8098,8100,8102,8104,8107],{"class":2401,"line":2502},[59,8091,2657],{"class":2405},[59,8093,2752],{"class":2409},[59,8095,798],{"class":2413},[59,8097,2666],{"class":2665},[59,8099,2420],{"class":2413},[59,8101,2417],{"class":2416},[59,8103,2420],{"class":2413},[59,8105,8106],{"class":2416},"Y",[59,8108,2426],{"class":2413},[59,8110,8111,8113,8115,8118,8120,8123,8125],{"class":2401,"line":2519},[59,8112,2465],{"class":2405},[59,8114,5220],{"class":2693},[59,8116,8117],{"class":2413},".ln(",[59,8119,2666],{"class":2693},[59,8121,8122],{"class":2413},".dropout(Y) ",[59,8124,5777],{"class":2405},[59,8126,8127],{"class":2413}," X)\n",[11,8129,8130],{"id":8130},"编码器-解码器架构",[15,8132,8133,8134,8284,8285,8461,8462,8491,8492,8642],{},"大多数主流的神经序列转换模型都采用编码器—解码器结构。其中，编码器将符号表示组成的输入序列 ",[59,8135,8137,8171],{"className":8136},[62],[59,8138,8140],{"className":8139},[66],[68,8141,8142],{"xmlns":70},[72,8143,8144,8168],{},[75,8145,8146,8148,8154,8156,8158,8160,8166],{},[461,8147,798],{"stretchy":797},[88,8149,8150,8152],{},[91,8151,7327],{},[81,8153,83],{},[461,8155,803],{"separator":106},[461,8157,2970],{},[461,8159,803],{"separator":106},[88,8161,8162,8164],{},[91,8163,7327],{},[91,8165,335],{},[461,8167,812],{"stretchy":797},[98,8169,8170],{"encoding":100},"(x_1, \\ldots, x_n)",[59,8172,8174],{"className":8173,"ariaHidden":106},[105],[59,8175,8177,8180,8183,8223,8226,8229,8232,8235,8238,8241,8281],{"className":8176},[110],[59,8178],{"className":8179,"style":877},[114],[59,8181,798],{"className":8182},[123],[59,8184,8186,8189],{"className":8185},[119],[59,8187,7327],{"className":8188},[119,192],[59,8190,8192],{"className":8191},[196],[59,8193,8195,8215],{"className":8194},[131,132],[59,8196,8198,8212],{"className":8197},[136],[59,8199,8201],{"className":8200,"style":3091},[140],[59,8202,8203,8206],{"style":1251},[59,8204],{"className":8205,"style":540},[148],[59,8207,8209],{"className":8208},[153,154,155,156],[59,8210,83],{"className":8211},[119,156],[59,8213,227],{"className":8214},[226],[59,8216,8218],{"className":8217},[136],[59,8219,8221],{"className":8220,"style":1270},[140],[59,8222],{},[59,8224,803],{"className":8225},[912],[59,8227],{"className":8228,"style":916},[499],[59,8230,2970],{"className":8231},[962],[59,8233],{"className":8234,"style":916},[499],[59,8236,803],{"className":8237},[912],[59,8239],{"className":8240,"style":916},[499],[59,8242,8244,8247],{"className":8243},[119],[59,8245,7327],{"className":8246},[119,192],[59,8248,8250],{"className":8249},[196],[59,8251,8253,8273],{"className":8252},[131,132],[59,8254,8256,8270],{"className":8255},[136],[59,8257,8259],{"className":8258,"style":1322},[140],[59,8260,8261,8264],{"style":1251},[59,8262],{"className":8263,"style":540},[148],[59,8265,8267],{"className":8266},[153,154,155,156],[59,8268,335],{"className":8269},[119,192,156],[59,8271,227],{"className":8272},[226],[59,8274,8276],{"className":8275},[136],[59,8277,8279],{"className":8278,"style":1270},[140],[59,8280],{},[59,8282,812],{"className":8283},[313]," 映射为连续表示序列 ",[59,8286,8288,8327],{"className":8287},[62],[59,8289,8291],{"className":8290},[66],[68,8292,8293],{"xmlns":70},[72,8294,8295,8324],{},[75,8296,8297,8300,8302,8304,8310,8312,8314,8316,8322],{},[91,8298,8299],{"mathvariant":5378},"z",[461,8301,815],{},[461,8303,798],{"stretchy":797},[88,8305,8306,8308],{},[91,8307,8299],{},[81,8309,83],{},[461,8311,803],{"separator":106},[461,8313,2970],{},[461,8315,803],{"separator":106},[88,8317,8318,8320],{},[91,8319,8299],{},[91,8321,335],{},[461,8323,812],{"stretchy":797},[98,8325,8326],{"encoding":100},"\\mathbf{z} = (z_1, \\ldots, z_n)",[59,8328,8330,8349],{"className":8329,"ariaHidden":106},[105],[59,8331,8333,8337,8340,8343,8346],{"className":8332},[110],[59,8334],{"className":8335,"style":8336},[114],"height:0.4444em;",[59,8338,8299],{"className":8339},[119,5423],[59,8341],{"className":8342,"style":500},[499],[59,8344,815],{"className":8345},[504],[59,8347],{"className":8348,"style":500},[499],[59,8350,8352,8355,8358,8400,8403,8406,8409,8412,8415,8418,8458],{"className":8351},[110],[59,8353],{"className":8354,"style":877},[114],[59,8356,798],{"className":8357},[123],[59,8359,8361,8365],{"className":8360},[119],[59,8362,8299],{"className":8363,"style":8364},[119,192],"margin-right:0.04398em;",[59,8366,8368],{"className":8367},[196],[59,8369,8371,8392],{"className":8370},[131,132],[59,8372,8374,8389],{"className":8373},[136],[59,8375,8377],{"className":8376,"style":3091},[140],[59,8378,8380,8383],{"style":8379},"top:-2.55em;margin-left:-0.044em;margin-right:0.05em;",[59,8381],{"className":8382,"style":540},[148],[59,8384,8386],{"className":8385},[153,154,155,156],[59,8387,83],{"className":8388},[119,156],[59,8390,227],{"className":8391},[226],[59,8393,8395],{"className":8394},[136],[59,8396,8398],{"className":8397,"style":1270},[140],[59,8399],{},[59,8401,803],{"className":8402},[912],[59,8404],{"className":8405,"style":916},[499],[59,8407,2970],{"className":8408},[962],[59,8410],{"className":8411,"style":916},[499],[59,8413,803],{"className":8414},[912],[59,8416],{"className":8417,"style":916},[499],[59,8419,8421,8424],{"className":8420},[119],[59,8422,8299],{"className":8423,"style":8364},[119,192],[59,8425,8427],{"className":8426},[196],[59,8428,8430,8450],{"className":8429},[131,132],[59,8431,8433,8447],{"className":8432},[136],[59,8434,8436],{"className":8435,"style":1322},[140],[59,8437,8438,8441],{"style":8379},[59,8439],{"className":8440,"style":540},[148],[59,8442,8444],{"className":8443},[153,154,155,156],[59,8445,335],{"className":8446},[119,192,156],[59,8448,227],{"className":8449},[226],[59,8451,8453],{"className":8452},[136],[59,8454,8456],{"className":8455,"style":1270},[140],[59,8457],{},[59,8459,812],{"className":8460},[313],"。给定 ",[59,8463,8465,8479],{"className":8464},[62],[59,8466,8468],{"className":8467},[66],[68,8469,8470],{"xmlns":70},[72,8471,8472,8476],{},[75,8473,8474],{},[91,8475,8299],{"mathvariant":5378},[98,8477,8478],{"encoding":100},"\\mathbf{z}",[59,8480,8482],{"className":8481,"ariaHidden":106},[105],[59,8483,8485,8488],{"className":8484},[110],[59,8486],{"className":8487,"style":8336},[114],[59,8489,8299],{"className":8490},[119,5423],"，解码器随后一次生成一个元素的输出符号序列 ",[59,8493,8495,8529],{"className":8494},[62],[59,8496,8498],{"className":8497},[66],[68,8499,8500],{"xmlns":70},[72,8501,8502,8526],{},[75,8503,8504,8506,8512,8514,8516,8518,8524],{},[461,8505,798],{"stretchy":797},[88,8507,8508,8510],{},[91,8509,7821],{},[81,8511,83],{},[461,8513,803],{"separator":106},[461,8515,2970],{},[461,8517,803],{"separator":106},[88,8519,8520,8522],{},[91,8521,7821],{},[91,8523,366],{},[461,8525,812],{"stretchy":797},[98,8527,8528],{"encoding":100},"(y_1, \\ldots, y_m)",[59,8530,8532],{"className":8531,"ariaHidden":106},[105],[59,8533,8535,8538,8541,8581,8584,8587,8590,8593,8596,8599,8639],{"className":8534},[110],[59,8536],{"className":8537,"style":877},[114],[59,8539,798],{"className":8540},[123],[59,8542,8544,8547],{"className":8543},[119],[59,8545,7821],{"className":8546,"style":441},[119,192],[59,8548,8550],{"className":8549},[196],[59,8551,8553,8573],{"className":8552},[131,132],[59,8554,8556,8570],{"className":8555},[136],[59,8557,8559],{"className":8558,"style":3091},[140],[59,8560,8561,8564],{"style":2035},[59,8562],{"className":8563,"style":540},[148],[59,8565,8567],{"className":8566},[153,154,155,156],[59,8568,83],{"className":8569},[119,156],[59,8571,227],{"className":8572},[226],[59,8574,8576],{"className":8575},[136],[59,8577,8579],{"className":8578,"style":1270},[140],[59,8580],{},[59,8582,803],{"className":8583},[912],[59,8585],{"className":8586,"style":916},[499],[59,8588,2970],{"className":8589},[962],[59,8591],{"className":8592,"style":916},[499],[59,8594,803],{"className":8595},[912],[59,8597],{"className":8598,"style":916},[499],[59,8600,8602,8605],{"className":8601},[119],[59,8603,7821],{"className":8604,"style":441},[119,192],[59,8606,8608],{"className":8607},[196],[59,8609,8611,8631],{"className":8610},[131,132],[59,8612,8614,8628],{"className":8613},[136],[59,8615,8617],{"className":8616,"style":1322},[140],[59,8618,8619,8622],{"style":2035},[59,8620],{"className":8621,"style":540},[148],[59,8623,8625],{"className":8624},[153,154,155,156],[59,8626,366],{"className":8627},[119,192,156],[59,8629,227],{"className":8630},[226],[59,8632,8634],{"className":8633},[136],[59,8635,8637],{"className":8636,"style":1270},[140],[59,8638],{},[59,8640,812],{"className":8641},[313],"。在每一个时间步，模型都是自回归的，即在生成下一个符号时，会将之前已经生成的符号作为额外输入。",[2390,8644,8646],{"className":2392,"code":8645,"language":2394,"meta":2395,"style":2395},"class Encoder(nn.Module):\n    \"\"\"编码器-解码器架构的基本编码器接口\"\"\"\n    def __init__(self, **kwargs):\n        super(Encoder, self).__init__(**kwargs)\n\n    def forward(self, X, *args):\n        raise NotImplementedError\n\nclass Decoder(nn.Module):\n    \"\"\"编码器-解码器架构的基本解码器接口\"\"\"\n    def __init__(self, **kwargs):\n        super(Decoder, self).__init__(**kwargs)\n\n    def init_state(self, enc_outputs, *args):\n        raise NotImplementedError\n\n    def forward(self, X, state):\n        raise NotImplementedError\n\nclass EncoderDecoder(nn.Module):\n    \"\"\"编码器-解码器架构的基类\"\"\"\n    def __init__(self, encoder, decoder, **kwargs):\n        super(EncoderDecoder, self).__init__(**kwargs)\n        self.encoder = encoder\n        self.decoder = decoder\n\n    def forward(self, enc_X, dec_X, *args):\n        enc_outputs = self.encoder(enc_X, *args)\n        dec_state = self.decoder.init_state(enc_outputs, *args)\n        return self.decoder(dec_X, dec_state)\n",[2397,8647,8648,8665,8670,8688,8707,8711,8735,8743,8747,8764,8769,8787,8806,8810,8834,8840,8844,8865,8871,8875,8892,8897,8925,8944,8956,8968,8972,9000,9017,9033],{"__ignoreMap":2395},[59,8649,8650,8652,8655,8657,8659,8661,8663],{"class":2401,"line":2402},[59,8651,2630],{"class":2405},[59,8653,8654],{"class":2633}," Encoder",[59,8656,798],{"class":2413},[59,8658,2640],{"class":2639},[59,8660,864],{"class":2413},[59,8662,2645],{"class":2639},[59,8664,2426],{"class":2413},[59,8666,8667],{"class":2401,"line":2429},[59,8668,8669],{"class":2432},"    \"\"\"编码器-解码器架构的基本编码器接口\"\"\"\n",[59,8671,8672,8674,8676,8678,8680,8682,8684,8686],{"class":2401,"line":2436},[59,8673,2657],{"class":2405},[59,8675,2660],{"class":2455},[59,8677,798],{"class":2413},[59,8679,2666],{"class":2665},[59,8681,2420],{"class":2413},[59,8683,2676],{"class":2405},[59,8685,2679],{"class":2416},[59,8687,2426],{"class":2413},[59,8689,8690,8692,8695,8697,8699,8701,8703,8705],{"class":2401,"line":2443},[59,8691,2687],{"class":2686},[59,8693,8694],{"class":2413},"(Encoder, ",[59,8696,2666],{"class":2693},[59,8698,2696],{"class":2413},[59,8700,2699],{"class":2455},[59,8702,798],{"class":2413},[59,8704,2676],{"class":2405},[59,8706,2706],{"class":2413},[59,8708,8709],{"class":2401,"line":2462},[59,8710,2725],{"emptyLinePlaceholder":2724},[59,8712,8713,8715,8717,8719,8721,8723,8725,8727,8730,8733],{"class":2401,"line":2483},[59,8714,2657],{"class":2405},[59,8716,2752],{"class":2409},[59,8718,798],{"class":2413},[59,8720,2666],{"class":2665},[59,8722,2420],{"class":2413},[59,8724,2417],{"class":2416},[59,8726,2420],{"class":2413},[59,8728,8729],{"class":2405},"*",[59,8731,8732],{"class":2416},"args",[59,8734,2426],{"class":2413},[59,8736,8737,8740],{"class":2401,"line":2491},[59,8738,8739],{"class":2405},"        raise",[59,8741,8742],{"class":2686}," NotImplementedError\n",[59,8744,8745],{"class":2401,"line":2502},[59,8746,2725],{"emptyLinePlaceholder":2724},[59,8748,8749,8751,8754,8756,8758,8760,8762],{"class":2401,"line":2519},[59,8750,2630],{"class":2405},[59,8752,8753],{"class":2633}," Decoder",[59,8755,798],{"class":2413},[59,8757,2640],{"class":2639},[59,8759,864],{"class":2413},[59,8761,2645],{"class":2639},[59,8763,2426],{"class":2413},[59,8765,8766],{"class":2401,"line":2535},[59,8767,8768],{"class":2432},"    \"\"\"编码器-解码器架构的基本解码器接口\"\"\"\n",[59,8770,8771,8773,8775,8777,8779,8781,8783,8785],{"class":2401,"line":2543},[59,8772,2657],{"class":2405},[59,8774,2660],{"class":2455},[59,8776,798],{"class":2413},[59,8778,2666],{"class":2665},[59,8780,2420],{"class":2413},[59,8782,2676],{"class":2405},[59,8784,2679],{"class":2416},[59,8786,2426],{"class":2413},[59,8788,8789,8791,8794,8796,8798,8800,8802,8804],{"class":2401,"line":2560},[59,8790,2687],{"class":2686},[59,8792,8793],{"class":2413},"(Decoder, ",[59,8795,2666],{"class":2693},[59,8797,2696],{"class":2413},[59,8799,2699],{"class":2455},[59,8801,798],{"class":2413},[59,8803,2676],{"class":2405},[59,8805,2706],{"class":2413},[59,8807,8808],{"class":2401,"line":2566},[59,8809,2725],{"emptyLinePlaceholder":2724},[59,8811,8812,8814,8817,8819,8821,8823,8826,8828,8830,8832],{"class":2401,"line":2591},[59,8813,2657],{"class":2405},[59,8815,8816],{"class":2409}," init_state",[59,8818,798],{"class":2413},[59,8820,2666],{"class":2665},[59,8822,2420],{"class":2413},[59,8824,8825],{"class":2416},"enc_outputs",[59,8827,2420],{"class":2413},[59,8829,8729],{"class":2405},[59,8831,8732],{"class":2416},[59,8833,2426],{"class":2413},[59,8835,8836,8838],{"class":2401,"line":2604},[59,8837,8739],{"class":2405},[59,8839,8742],{"class":2686},[59,8841,8842],{"class":2401,"line":2845},[59,8843,2725],{"emptyLinePlaceholder":2724},[59,8845,8846,8848,8850,8852,8854,8856,8858,8860,8863],{"class":2401,"line":4781},[59,8847,2657],{"class":2405},[59,8849,2752],{"class":2409},[59,8851,798],{"class":2413},[59,8853,2666],{"class":2665},[59,8855,2420],{"class":2413},[59,8857,2417],{"class":2416},[59,8859,2420],{"class":2413},[59,8861,8862],{"class":2416},"state",[59,8864,2426],{"class":2413},[59,8866,8867,8869],{"class":2401,"line":5057},[59,8868,8739],{"class":2405},[59,8870,8742],{"class":2686},[59,8872,8873],{"class":2401,"line":5063},[59,8874,2725],{"emptyLinePlaceholder":2724},[59,8876,8877,8879,8882,8884,8886,8888,8890],{"class":2401,"line":5069},[59,8878,2630],{"class":2405},[59,8880,8881],{"class":2633}," EncoderDecoder",[59,8883,798],{"class":2413},[59,8885,2640],{"class":2639},[59,8887,864],{"class":2413},[59,8889,2645],{"class":2639},[59,8891,2426],{"class":2413},[59,8893,8894],{"class":2401,"line":5075},[59,8895,8896],{"class":2432},"    \"\"\"编码器-解码器架构的基类\"\"\"\n",[59,8898,8899,8901,8903,8905,8907,8909,8912,8914,8917,8919,8921,8923],{"class":2401,"line":5096},[59,8900,2657],{"class":2405},[59,8902,2660],{"class":2455},[59,8904,798],{"class":2413},[59,8906,2666],{"class":2665},[59,8908,2420],{"class":2413},[59,8910,8911],{"class":2416},"encoder",[59,8913,2420],{"class":2413},[59,8915,8916],{"class":2416},"decoder",[59,8918,2420],{"class":2413},[59,8920,2676],{"class":2405},[59,8922,2679],{"class":2416},[59,8924,2426],{"class":2413},[59,8926,8927,8929,8932,8934,8936,8938,8940,8942],{"class":2401,"line":5115},[59,8928,2687],{"class":2686},[59,8930,8931],{"class":2413},"(EncoderDecoder, ",[59,8933,2666],{"class":2693},[59,8935,2696],{"class":2413},[59,8937,2699],{"class":2455},[59,8939,798],{"class":2413},[59,8941,2676],{"class":2405},[59,8943,2706],{"class":2413},[59,8945,8946,8948,8951,8953],{"class":2401,"line":5134},[59,8947,2711],{"class":2693},[59,8949,8950],{"class":2413},".encoder ",[59,8952,815],{"class":2405},[59,8954,8955],{"class":2413}," encoder\n",[59,8957,8958,8960,8963,8965],{"class":2401,"line":5139},[59,8959,2711],{"class":2693},[59,8961,8962],{"class":2413},".decoder ",[59,8964,815],{"class":2405},[59,8966,8967],{"class":2413}," decoder\n",[59,8969,8970],{"class":2401,"line":5155},[59,8971,2725],{"emptyLinePlaceholder":2724},[59,8973,8974,8976,8978,8980,8982,8984,8987,8989,8992,8994,8996,8998],{"class":2401,"line":5161},[59,8975,2657],{"class":2405},[59,8977,2752],{"class":2409},[59,8979,798],{"class":2413},[59,8981,2666],{"class":2665},[59,8983,2420],{"class":2413},[59,8985,8986],{"class":2416},"enc_X",[59,8988,2420],{"class":2413},[59,8990,8991],{"class":2416},"dec_X",[59,8993,2420],{"class":2413},[59,8995,8729],{"class":2405},[59,8997,8732],{"class":2416},[59,8999,2426],{"class":2413},[59,9001,9002,9005,9007,9009,9012,9014],{"class":2401,"line":5167},[59,9003,9004],{"class":2413},"        enc_outputs ",[59,9006,815],{"class":2405},[59,9008,5220],{"class":2693},[59,9010,9011],{"class":2413},".encoder(enc_X, ",[59,9013,8729],{"class":2405},[59,9015,9016],{"class":2413},"args)\n",[59,9018,9019,9022,9024,9026,9029,9031],{"class":2401,"line":5177},[59,9020,9021],{"class":2413},"        dec_state ",[59,9023,815],{"class":2405},[59,9025,5220],{"class":2693},[59,9027,9028],{"class":2413},".decoder.init_state(enc_outputs, ",[59,9030,8729],{"class":2405},[59,9032,9016],{"class":2413},[59,9034,9035,9037,9039],{"class":2401,"line":5201},[59,9036,2465],{"class":2405},[59,9038,5220],{"class":2693},[59,9040,9041],{"class":2413},".decoder(dec_X, dec_state)\n",[11,9043,9045],{"id":9044},"transformer-编码器","transformer 编码器",[15,9047,9048],{},"代码实现如下：",[2390,9050,9052],{"className":2392,"code":9051,"language":2394,"meta":2395,"style":2395},"class EncoderBlock(nn.Module):\n    \"\"\"Transformer编码器块\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n                 dropout, use_bias=False, **kwargs):\n        super(EncoderBlock, self).__init__(**kwargs)\n        self.attention = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout,\n            use_bias)\n        self.addnorm1 = AddNorm(norm_shape, dropout)\n        self.ffn = PositionWiseFFN(\n            ffn_num_input, ffn_num_hiddens, num_hiddens)\n        self.addnorm2 = AddNorm(norm_shape, dropout)\n\n    def forward(self, X, valid_lens):\n        Y = self.addnorm1(X, self.attention(X, X, X, valid_lens))\n        return self.addnorm2(Y, self.ffn(Y))\n\nclass TransformerEncoder(Encoder):\n    \"\"\"Transformer编码器\"\"\"\n    def __init__(self, vocab_size, key_size, query_size, value_size,\n                 num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,\n                 num_heads, num_layers, dropout, use_bias=False, **kwargs):\n        super(TransformerEncoder, self).__init__(**kwargs)\n        self.num_hiddens = num_hiddens\n        self.embedding = nn.Embedding(vocab_size, num_hiddens)\n        self.pos_encoding = PositionalEncoding(num_hiddens, dropout)\n        self.blks = nn.Sequential()\n        for i in range(num_layers):\n            self.blks.add_module(\"block\"+str(i),\n                EncoderBlock(key_size, query_size, value_size, num_hiddens,\n                             norm_shape, ffn_num_input, ffn_num_hiddens,\n                             num_heads, dropout, use_bias))\n\n    def forward(self, X, valid_lens, *args):\n        # 因为位置编码值在-1和1之间，\n        # 因此嵌入值乘以嵌入维度的平方根进行缩放，\n        # 然后再与位置编码相加。\n        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))\n        self.attention_weights = [None] * len(self.blks)\n        for i, blk in enumerate(self.blks):\n            X = blk(X, valid_lens)\n            self.attention_weights[i] = blk.attention.attention.attention_weights\n        return X\n",[2397,9053,9054,9071,9076,9104,9123,9145,9164,9175,9180,9185,9197,9209,9214,9225,9229,9249,9266,9280,9284,9298,9303,9332,9352,9381,9400,9412,9424,9436,9448,9466,9492,9497,9502,9507,9511,9537,9542,9548,9554,9581,9609,9629,9640,9653],{"__ignoreMap":2395},[59,9055,9056,9058,9061,9063,9065,9067,9069],{"class":2401,"line":2402},[59,9057,2630],{"class":2405},[59,9059,9060],{"class":2633}," EncoderBlock",[59,9062,798],{"class":2413},[59,9064,2640],{"class":2639},[59,9066,864],{"class":2413},[59,9068,2645],{"class":2639},[59,9070,2426],{"class":2413},[59,9072,9073],{"class":2401,"line":2429},[59,9074,9075],{"class":2432},"    \"\"\"Transformer编码器块\"\"\"\n",[59,9077,9078,9080,9082,9084,9086,9088,9090,9092,9094,9096,9098,9100,9102],{"class":2401,"line":2436},[59,9079,2657],{"class":2405},[59,9081,2660],{"class":2455},[59,9083,798],{"class":2413},[59,9085,2666],{"class":2665},[59,9087,2420],{"class":2413},[59,9089,4847],{"class":2416},[59,9091,2420],{"class":2413},[59,9093,4852],{"class":2416},[59,9095,2420],{"class":2413},[59,9097,4857],{"class":2416},[59,9099,2420],{"class":2413},[59,9101,4862],{"class":2416},[59,9103,3826],{"class":2413},[59,9105,9106,9109,9111,9113,9115,9117,9119,9121],{"class":2401,"line":2443},[59,9107,9108],{"class":2416},"                 norm_shape",[59,9110,2420],{"class":2413},[59,9112,7675],{"class":2416},[59,9114,2420],{"class":2413},[59,9116,7680],{"class":2416},[59,9118,2420],{"class":2413},[59,9120,4595],{"class":2416},[59,9122,3826],{"class":2413},[59,9124,9125,9128,9130,9133,9135,9137,9139,9141,9143],{"class":2401,"line":2462},[59,9126,9127],{"class":2416},"                 dropout",[59,9129,2420],{"class":2413},[59,9131,9132],{"class":2416},"use_bias",[59,9134,815],{"class":2405},[59,9136,4883],{"class":2455},[59,9138,2420],{"class":2413},[59,9140,2676],{"class":2405},[59,9142,2679],{"class":2416},[59,9144,2426],{"class":2413},[59,9146,9147,9149,9152,9154,9156,9158,9160,9162],{"class":2401,"line":2483},[59,9148,2687],{"class":2686},[59,9150,9151],{"class":2413},"(EncoderBlock, ",[59,9153,2666],{"class":2693},[59,9155,2696],{"class":2413},[59,9157,2699],{"class":2455},[59,9159,798],{"class":2413},[59,9161,2676],{"class":2405},[59,9163,2706],{"class":2413},[59,9165,9166,9168,9170,9172],{"class":2401,"line":2491},[59,9167,2711],{"class":2693},[59,9169,4929],{"class":2413},[59,9171,815],{"class":2405},[59,9173,9174],{"class":2413}," d2l.MultiHeadAttention(\n",[59,9176,9177],{"class":2401,"line":2502},[59,9178,9179],{"class":2413},"            key_size, query_size, value_size, num_hiddens, num_heads, dropout,\n",[59,9181,9182],{"class":2401,"line":2519},[59,9183,9184],{"class":2413},"            use_bias)\n",[59,9186,9187,9189,9192,9194],{"class":2401,"line":2535},[59,9188,2711],{"class":2693},[59,9190,9191],{"class":2413},".addnorm1 ",[59,9193,815],{"class":2405},[59,9195,9196],{"class":2413}," AddNorm(norm_shape, dropout)\n",[59,9198,9199,9201,9204,9206],{"class":2401,"line":2543},[59,9200,2711],{"class":2693},[59,9202,9203],{"class":2413},".ffn ",[59,9205,815],{"class":2405},[59,9207,9208],{"class":2413}," PositionWiseFFN(\n",[59,9210,9211],{"class":2401,"line":2560},[59,9212,9213],{"class":2413},"            ffn_num_input, ffn_num_hiddens, num_hiddens)\n",[59,9215,9216,9218,9221,9223],{"class":2401,"line":2566},[59,9217,2711],{"class":2693},[59,9219,9220],{"class":2413},".addnorm2 ",[59,9222,815],{"class":2405},[59,9224,9196],{"class":2413},[59,9226,9227],{"class":2401,"line":2591},[59,9228,2725],{"emptyLinePlaceholder":2724},[59,9230,9231,9233,9235,9237,9239,9241,9243,9245,9247],{"class":2401,"line":2604},[59,9232,2657],{"class":2405},[59,9234,2752],{"class":2409},[59,9236,798],{"class":2413},[59,9238,2666],{"class":2665},[59,9240,2420],{"class":2413},[59,9242,2417],{"class":2416},[59,9244,2420],{"class":2413},[59,9246,2423],{"class":2416},[59,9248,2426],{"class":2413},[59,9250,9251,9254,9256,9258,9261,9263],{"class":2401,"line":2845},[59,9252,9253],{"class":2413},"        Y ",[59,9255,815],{"class":2405},[59,9257,5220],{"class":2693},[59,9259,9260],{"class":2413},".addnorm1(X, ",[59,9262,2666],{"class":2693},[59,9264,9265],{"class":2413},".attention(X, X, X, valid_lens))\n",[59,9267,9268,9270,9272,9275,9277],{"class":2401,"line":4781},[59,9269,2465],{"class":2405},[59,9271,5220],{"class":2693},[59,9273,9274],{"class":2413},".addnorm2(Y, ",[59,9276,2666],{"class":2693},[59,9278,9279],{"class":2413},".ffn(Y))\n",[59,9281,9282],{"class":2401,"line":5057},[59,9283,2725],{"emptyLinePlaceholder":2724},[59,9285,9286,9288,9291,9293,9296],{"class":2401,"line":5063},[59,9287,2630],{"class":2405},[59,9289,9290],{"class":2633}," TransformerEncoder",[59,9292,798],{"class":2413},[59,9294,9295],{"class":2639},"Encoder",[59,9297,2426],{"class":2413},[59,9299,9300],{"class":2401,"line":5069},[59,9301,9302],{"class":2432},"    \"\"\"Transformer编码器\"\"\"\n",[59,9304,9305,9307,9309,9311,9313,9315,9318,9320,9322,9324,9326,9328,9330],{"class":2401,"line":5075},[59,9306,2657],{"class":2405},[59,9308,2660],{"class":2455},[59,9310,798],{"class":2413},[59,9312,2666],{"class":2665},[59,9314,2420],{"class":2413},[59,9316,9317],{"class":2416},"vocab_size",[59,9319,2420],{"class":2413},[59,9321,4847],{"class":2416},[59,9323,2420],{"class":2413},[59,9325,4852],{"class":2416},[59,9327,2420],{"class":2413},[59,9329,4857],{"class":2416},[59,9331,3826],{"class":2413},[59,9333,9334,9337,9339,9342,9344,9346,9348,9350],{"class":2401,"line":5096},[59,9335,9336],{"class":2416},"                 num_hiddens",[59,9338,2420],{"class":2413},[59,9340,9341],{"class":2416},"norm_shape",[59,9343,2420],{"class":2413},[59,9345,7675],{"class":2416},[59,9347,2420],{"class":2413},[59,9349,7680],{"class":2416},[59,9351,3826],{"class":2413},[59,9353,9354,9356,9358,9361,9363,9365,9367,9369,9371,9373,9375,9377,9379],{"class":2401,"line":5115},[59,9355,4869],{"class":2416},[59,9357,2420],{"class":2413},[59,9359,9360],{"class":2416},"num_layers",[59,9362,2420],{"class":2413},[59,9364,2671],{"class":2416},[59,9366,2420],{"class":2413},[59,9368,9132],{"class":2416},[59,9370,815],{"class":2405},[59,9372,4883],{"class":2455},[59,9374,2420],{"class":2413},[59,9376,2676],{"class":2405},[59,9378,2679],{"class":2416},[59,9380,2426],{"class":2413},[59,9382,9383,9385,9388,9390,9392,9394,9396,9398],{"class":2401,"line":5134},[59,9384,2687],{"class":2686},[59,9386,9387],{"class":2413},"(TransformerEncoder, ",[59,9389,2666],{"class":2693},[59,9391,2696],{"class":2413},[59,9393,2699],{"class":2455},[59,9395,798],{"class":2413},[59,9397,2676],{"class":2405},[59,9399,2706],{"class":2413},[59,9401,9402,9404,9407,9409],{"class":2401,"line":5139},[59,9403,2711],{"class":2693},[59,9405,9406],{"class":2413},".num_hiddens ",[59,9408,815],{"class":2405},[59,9410,9411],{"class":2413}," num_hiddens\n",[59,9413,9414,9416,9419,9421],{"class":2401,"line":5155},[59,9415,2711],{"class":2693},[59,9417,9418],{"class":2413},".embedding ",[59,9420,815],{"class":2405},[59,9422,9423],{"class":2413}," nn.Embedding(vocab_size, num_hiddens)\n",[59,9425,9426,9428,9431,9433],{"class":2401,"line":5161},[59,9427,2711],{"class":2693},[59,9429,9430],{"class":2413},".pos_encoding ",[59,9432,815],{"class":2405},[59,9434,9435],{"class":2413}," PositionalEncoding(num_hiddens, dropout)\n",[59,9437,9438,9440,9443,9445],{"class":2401,"line":5167},[59,9439,2711],{"class":2693},[59,9441,9442],{"class":2413},".blks ",[59,9444,815],{"class":2405},[59,9446,9447],{"class":2413}," nn.Sequential()\n",[59,9449,9450,9453,9456,9459,9463],{"class":2401,"line":5177},[59,9451,9452],{"class":2405},"        for",[59,9454,9455],{"class":2413}," i ",[59,9457,9458],{"class":2405},"in",[59,9460,9462],{"class":9461},"sDgm9"," range",[59,9464,9465],{"class":2413},"(num_layers):\n",[59,9467,9468,9471,9474,9478,9482,9484,9486,9489],{"class":2401,"line":5201},[59,9469,9470],{"class":2693},"            self",[59,9472,9473],{"class":2413},".blks.add_module(",[59,9475,9477],{"class":9476},"sMWOi","\"",[59,9479,9481],{"class":9480},"sEzAm","block",[59,9483,9477],{"class":9476},[59,9485,5777],{"class":2405},[59,9487,9488],{"class":2686},"str",[59,9490,9491],{"class":2413},"(i),\n",[59,9493,9494],{"class":2401,"line":5206},[59,9495,9496],{"class":2413},"                EncoderBlock(key_size, query_size, value_size, num_hiddens,\n",[59,9498,9499],{"class":2401,"line":5212},[59,9500,9501],{"class":2413},"                             norm_shape, ffn_num_input, ffn_num_hiddens,\n",[59,9503,9504],{"class":2401,"line":5226},[59,9505,9506],{"class":2413},"                             num_heads, dropout, use_bias))\n",[59,9508,9509],{"class":2401,"line":5231},[59,9510,2725],{"emptyLinePlaceholder":2724},[59,9512,9513,9515,9517,9519,9521,9523,9525,9527,9529,9531,9533,9535],{"class":2401,"line":5237},[59,9514,2657],{"class":2405},[59,9516,2752],{"class":2409},[59,9518,798],{"class":2413},[59,9520,2666],{"class":2665},[59,9522,2420],{"class":2413},[59,9524,2417],{"class":2416},[59,9526,2420],{"class":2413},[59,9528,2423],{"class":2416},[59,9530,2420],{"class":2413},[59,9532,8729],{"class":2405},[59,9534,8732],{"class":2416},[59,9536,2426],{"class":2413},[59,9538,9539],{"class":2401,"line":5252},[59,9540,9541],{"class":2439},"        # 因为位置编码值在-1和1之间，\n",[59,9543,9545],{"class":2401,"line":9544},37,[59,9546,9547],{"class":2439},"        # 因此嵌入值乘以嵌入维度的平方根进行缩放，\n",[59,9549,9551],{"class":2401,"line":9550},38,[59,9552,9553],{"class":2439},"        # 然后再与位置编码相加。\n",[59,9555,9557,9559,9561,9563,9566,9568,9571,9573,9576,9578],{"class":2401,"line":9556},39,[59,9558,2569],{"class":2413},[59,9560,815],{"class":2405},[59,9562,5220],{"class":2693},[59,9564,9565],{"class":2413},".pos_encoding(",[59,9567,2666],{"class":2693},[59,9569,9570],{"class":2413},".embedding(X) ",[59,9572,8729],{"class":2405},[59,9574,9575],{"class":2413}," math.sqrt(",[59,9577,2666],{"class":2693},[59,9579,9580],{"class":2413},".num_hiddens))\n",[59,9582,9584,9586,9588,9590,9593,9595,9597,9599,9602,9604,9606],{"class":2401,"line":9583},40,[59,9585,2711],{"class":2693},[59,9587,2837],{"class":2413},[59,9589,815],{"class":2405},[59,9591,9592],{"class":2413}," [",[59,9594,2780],{"class":2455},[59,9596,7200],{"class":2413},[59,9598,8729],{"class":2405},[59,9600,9601],{"class":9461}," len",[59,9603,798],{"class":2413},[59,9605,2666],{"class":2693},[59,9607,9608],{"class":2413},".blks)\n",[59,9610,9612,9614,9617,9619,9622,9624,9626],{"class":2401,"line":9611},41,[59,9613,9452],{"class":2405},[59,9615,9616],{"class":2413}," i, blk ",[59,9618,9458],{"class":2405},[59,9620,9621],{"class":9461}," enumerate",[59,9623,798],{"class":2413},[59,9625,2666],{"class":2693},[59,9627,9628],{"class":2413},".blks):\n",[59,9630,9632,9635,9637],{"class":2401,"line":9631},42,[59,9633,9634],{"class":2413},"            X ",[59,9636,815],{"class":2405},[59,9638,9639],{"class":2413}," blk(X, valid_lens)\n",[59,9641,9643,9645,9648,9650],{"class":2401,"line":9642},43,[59,9644,9470],{"class":2693},[59,9646,9647],{"class":2413},".attention_weights[i] ",[59,9649,815],{"class":2405},[59,9651,9652],{"class":2413}," blk.attention.attention.attention_weights\n",[59,9654,9656,9658],{"class":2401,"line":9655},44,[59,9657,2465],{"class":2405},[59,9659,9660],{"class":2413}," X\n",[11,9662,9664],{"id":9663},"transformer-解码器","transformer 解码器",[15,9666,9667],{},"代码如下：",[2390,9669,9671],{"className":2392,"code":9670,"language":2394,"meta":2395,"style":2395},"class DecoderBlock(nn.Module):\n    \"\"\"解码器中第i个块\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n                 dropout, i, **kwargs):\n        super(DecoderBlock, self).__init__(**kwargs)\n        self.i = i\n        self.attention1 = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n        self.addnorm1 = AddNorm(norm_shape, dropout)\n        self.attention2 = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n        self.addnorm2 = AddNorm(norm_shape, dropout)\n        self.ffn = PositionWiseFFN(ffn_num_input, ffn_num_hiddens,\n                                   num_hiddens)\n        self.addnorm3 = AddNorm(norm_shape, dropout)\n\n    def forward(self, X, state):\n        enc_outputs, enc_valid_lens = state[0], state[1]\n        # 训练阶段，输出序列的所有词元都在同一时间处理，\n        # 因此state[2][self.i]初始化为None。\n        # 预测阶段，输出序列是通过词元一个接着一个解码的，\n        # 因此state[2][self.i]包含着直到当前时间步第i个块解码的输出表示\n        if state[2][self.i] is None:\n            key_values = X\n        else:\n            key_values = torch.cat((state[2][self.i], X), axis=1)\n        state[2][self.i] = key_values\n        if self.training:\n            batch_size, num_steps, _ = X.shape\n            # dec_valid_lens的开头:(batch_size,num_steps),\n            # 其中每一行是[1,2,...,num_steps]\n            dec_valid_lens = torch.arange(\n                1, num_steps + 1, device=X.device).repeat(batch_size, 1)\n        else:\n            dec_valid_lens = None\n\n        # 自注意力\n        X2 = self.attention1(X, key_values, key_values, dec_valid_lens)\n        Y = self.addnorm1(X, X2)\n        # 编码器－解码器注意力。\n        # enc_outputs的开头:(batch_size,num_steps,num_hiddens)\n        Y2 = self.attention2(Y, enc_outputs, enc_outputs, enc_valid_lens)\n        Z = self.addnorm2(Y, Y2)\n        return self.addnorm3(Z, self.ffn(Z)), state\n\nclass TransformerDecoder(AttentionDecoder):\n    def __init__(self, vocab_size, key_size, query_size, value_size,\n                 num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,\n                 num_heads, num_layers, dropout, **kwargs):\n        super(TransformerDecoder, self).__init__(**kwargs)\n        self.num_hiddens = num_hiddens\n        self.num_layers = num_layers\n        self.embedding = nn.Embedding(vocab_size, num_hiddens)\n        self.pos_encoding = PositionalEncoding(num_hiddens, dropout)\n        self.blks = nn.Sequential()\n        for i in range(num_layers):\n            self.blks.add_module(\"block\"+str(i),\n                DecoderBlock(key_size, query_size, value_size, num_hiddens,\n                             norm_shape, ffn_num_input, ffn_num_hiddens,\n                             num_heads, dropout, i))\n        self.dense = nn.Linear(num_hiddens, vocab_size)\n\n    def init_state(self, enc_outputs, enc_valid_lens, *args):\n        return [enc_outputs, enc_valid_lens, [None] * self.num_layers]\n\n    def forward(self, X, state):\n        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))\n        self._attention_weights = [[None] * len(self.blks) for _ in range (2)]\n        for i, blk in enumerate(self.blks):\n            X, state = blk(X, state)\n            # 解码器自注意力权重\n            self._attention_weights[0][i] = blk.attention1.attention.attention_weights\n            # “编码器－解码器”自注意力权重\n            self._attention_weights[1][i] = blk.attention2.attention.attention_weights\n        return self.dense(X), state\n\n    @property\n    def attention_weights(self):\n        return self._attention_weights\n",[2397,9672,9673,9690,9695,9723,9741,9757,9776,9788,9799,9804,9814,9825,9829,9839,9850,9855,9866,9870,9890,9909,9914,9919,9924,9929,9951,9960,9966,9993,10011,10020,10029,10034,10039,10049,10075,10081,10090,10094,10099,10111,10122,10127,10132,10144,10156,10171,10176,10191,10220,10239,10260,10280,10291,10304,10315,10326,10337,10350,10369,10375,10380,10386,10399,10404,10432,10451,10456,10477,10500,10546,10563,10574,10580,10598,10604,10620,10630,10635,10644,10658],{"__ignoreMap":2395},[59,9674,9675,9677,9680,9682,9684,9686,9688],{"class":2401,"line":2402},[59,9676,2630],{"class":2405},[59,9678,9679],{"class":2633}," DecoderBlock",[59,9681,798],{"class":2413},[59,9683,2640],{"class":2639},[59,9685,864],{"class":2413},[59,9687,2645],{"class":2639},[59,9689,2426],{"class":2413},[59,9691,9692],{"class":2401,"line":2429},[59,9693,9694],{"class":2432},"    \"\"\"解码器中第i个块\"\"\"\n",[59,9696,9697,9699,9701,9703,9705,9707,9709,9711,9713,9715,9717,9719,9721],{"class":2401,"line":2436},[59,9698,2657],{"class":2405},[59,9700,2660],{"class":2455},[59,9702,798],{"class":2413},[59,9704,2666],{"class":2665},[59,9706,2420],{"class":2413},[59,9708,4847],{"class":2416},[59,9710,2420],{"class":2413},[59,9712,4852],{"class":2416},[59,9714,2420],{"class":2413},[59,9716,4857],{"class":2416},[59,9718,2420],{"class":2413},[59,9720,4862],{"class":2416},[59,9722,3826],{"class":2413},[59,9724,9725,9727,9729,9731,9733,9735,9737,9739],{"class":2401,"line":2443},[59,9726,9108],{"class":2416},[59,9728,2420],{"class":2413},[59,9730,7675],{"class":2416},[59,9732,2420],{"class":2413},[59,9734,7680],{"class":2416},[59,9736,2420],{"class":2413},[59,9738,4595],{"class":2416},[59,9740,3826],{"class":2413},[59,9742,9743,9745,9747,9749,9751,9753,9755],{"class":2401,"line":2462},[59,9744,9127],{"class":2416},[59,9746,2420],{"class":2413},[59,9748,789],{"class":2416},[59,9750,2420],{"class":2413},[59,9752,2676],{"class":2405},[59,9754,2679],{"class":2416},[59,9756,2426],{"class":2413},[59,9758,9759,9761,9764,9766,9768,9770,9772,9774],{"class":2401,"line":2483},[59,9760,2687],{"class":2686},[59,9762,9763],{"class":2413},"(DecoderBlock, ",[59,9765,2666],{"class":2693},[59,9767,2696],{"class":2413},[59,9769,2699],{"class":2455},[59,9771,798],{"class":2413},[59,9773,2676],{"class":2405},[59,9775,2706],{"class":2413},[59,9777,9778,9780,9783,9785],{"class":2401,"line":2491},[59,9779,2711],{"class":2693},[59,9781,9782],{"class":2413},".i ",[59,9784,815],{"class":2405},[59,9786,9787],{"class":2413}," i\n",[59,9789,9790,9792,9795,9797],{"class":2401,"line":2502},[59,9791,2711],{"class":2693},[59,9793,9794],{"class":2413},".attention1 ",[59,9796,815],{"class":2405},[59,9798,9174],{"class":2413},[59,9800,9801],{"class":2401,"line":2519},[59,9802,9803],{"class":2413},"            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n",[59,9805,9806,9808,9810,9812],{"class":2401,"line":2535},[59,9807,2711],{"class":2693},[59,9809,9191],{"class":2413},[59,9811,815],{"class":2405},[59,9813,9196],{"class":2413},[59,9815,9816,9818,9821,9823],{"class":2401,"line":2543},[59,9817,2711],{"class":2693},[59,9819,9820],{"class":2413},".attention2 ",[59,9822,815],{"class":2405},[59,9824,9174],{"class":2413},[59,9826,9827],{"class":2401,"line":2560},[59,9828,9803],{"class":2413},[59,9830,9831,9833,9835,9837],{"class":2401,"line":2566},[59,9832,2711],{"class":2693},[59,9834,9220],{"class":2413},[59,9836,815],{"class":2405},[59,9838,9196],{"class":2413},[59,9840,9841,9843,9845,9847],{"class":2401,"line":2591},[59,9842,2711],{"class":2693},[59,9844,9203],{"class":2413},[59,9846,815],{"class":2405},[59,9848,9849],{"class":2413}," PositionWiseFFN(ffn_num_input, ffn_num_hiddens,\n",[59,9851,9852],{"class":2401,"line":2604},[59,9853,9854],{"class":2413},"                                   num_hiddens)\n",[59,9856,9857,9859,9862,9864],{"class":2401,"line":2845},[59,9858,2711],{"class":2693},[59,9860,9861],{"class":2413},".addnorm3 ",[59,9863,815],{"class":2405},[59,9865,9196],{"class":2413},[59,9867,9868],{"class":2401,"line":4781},[59,9869,2725],{"emptyLinePlaceholder":2724},[59,9871,9872,9874,9876,9878,9880,9882,9884,9886,9888],{"class":2401,"line":5057},[59,9873,2657],{"class":2405},[59,9875,2752],{"class":2409},[59,9877,798],{"class":2413},[59,9879,2666],{"class":2665},[59,9881,2420],{"class":2413},[59,9883,2417],{"class":2416},[59,9885,2420],{"class":2413},[59,9887,8862],{"class":2416},[59,9889,2426],{"class":2413},[59,9891,9892,9895,9897,9900,9902,9905,9907],{"class":2401,"line":5063},[59,9893,9894],{"class":2413},"        enc_outputs, enc_valid_lens ",[59,9896,815],{"class":2405},[59,9898,9899],{"class":2413}," state[",[59,9901,1754],{"class":2455},[59,9903,9904],{"class":2413},"], state[",[59,9906,83],{"class":2455},[59,9908,2799],{"class":2413},[59,9910,9911],{"class":2401,"line":5069},[59,9912,9913],{"class":2439},"        # 训练阶段，输出序列的所有词元都在同一时间处理，\n",[59,9915,9916],{"class":2401,"line":5075},[59,9917,9918],{"class":2439},"        # 因此state[2][self.i]初始化为None。\n",[59,9920,9921],{"class":2401,"line":5096},[59,9922,9923],{"class":2439},"        # 预测阶段，输出序列是通过词元一个接着一个解码的，\n",[59,9925,9926],{"class":2401,"line":5115},[59,9927,9928],{"class":2439},"        # 因此state[2][self.i]包含着直到当前时间步第i个块解码的输出表示\n",[59,9930,9931,9933,9935,9937,9940,9942,9945,9947,9949],{"class":2401,"line":5134},[59,9932,2505],{"class":2405},[59,9934,9899],{"class":2413},[59,9936,2821],{"class":2455},[59,9938,9939],{"class":2413},"][",[59,9941,2666],{"class":2693},[59,9943,9944],{"class":2413},".i] ",[59,9946,2452],{"class":2405},[59,9948,2456],{"class":2455},[59,9950,2459],{"class":2413},[59,9952,9953,9956,9958],{"class":2401,"line":5139},[59,9954,9955],{"class":2413},"            key_values ",[59,9957,815],{"class":2405},[59,9959,9660],{"class":2413},[59,9961,9962,9964],{"class":2401,"line":5155},[59,9963,2538],{"class":2405},[59,9965,2459],{"class":2413},[59,9967,9968,9970,9972,9975,9977,9979,9981,9984,9987,9989,9991],{"class":2401,"line":5161},[59,9969,9955],{"class":2413},[59,9971,815],{"class":2405},[59,9973,9974],{"class":2413}," torch.cat((state[",[59,9976,2821],{"class":2455},[59,9978,9939],{"class":2413},[59,9980,2666],{"class":2693},[59,9982,9983],{"class":2413},".i], X), ",[59,9985,9986],{"class":2471},"axis",[59,9988,815],{"class":2405},[59,9990,83],{"class":2455},[59,9992,2480],{"class":2413},[59,9994,9995,9998,10000,10002,10004,10006,10008],{"class":2401,"line":5167},[59,9996,9997],{"class":2413},"        state[",[59,9999,2821],{"class":2455},[59,10001,9939],{"class":2413},[59,10003,2666],{"class":2693},[59,10005,9944],{"class":2413},[59,10007,815],{"class":2405},[59,10009,10010],{"class":2413}," key_values\n",[59,10012,10013,10015,10017],{"class":2401,"line":5177},[59,10014,2505],{"class":2405},[59,10016,5220],{"class":2693},[59,10018,10019],{"class":2413},".training:\n",[59,10021,10022,10025,10027],{"class":2401,"line":5201},[59,10023,10024],{"class":2413},"            batch_size, num_steps, _ ",[59,10026,815],{"class":2405},[59,10028,2499],{"class":2413},[59,10030,10031],{"class":2401,"line":5206},[59,10032,10033],{"class":2439},"            # dec_valid_lens的开头:(batch_size,num_steps),\n",[59,10035,10036],{"class":2401,"line":5212},[59,10037,10038],{"class":2439},"            # 其中每一行是[1,2,...,num_steps]\n",[59,10040,10041,10044,10046],{"class":2401,"line":5226},[59,10042,10043],{"class":2413},"            dec_valid_lens ",[59,10045,815],{"class":2405},[59,10047,10048],{"class":2413}," torch.arange(\n",[59,10050,10051,10054,10057,10059,10061,10063,10066,10068,10071,10073],{"class":2401,"line":5231},[59,10052,10053],{"class":2455},"                1",[59,10055,10056],{"class":2413},", num_steps ",[59,10058,5777],{"class":2405},[59,10060,2514],{"class":2455},[59,10062,2420],{"class":2413},[59,10064,10065],{"class":2471},"device",[59,10067,815],{"class":2405},[59,10069,10070],{"class":2413},"X.device).repeat(batch_size, ",[59,10072,83],{"class":2455},[59,10074,2480],{"class":2413},[59,10076,10077,10079],{"class":2401,"line":5237},[59,10078,2538],{"class":2405},[59,10080,2459],{"class":2413},[59,10082,10083,10085,10087],{"class":2401,"line":5252},[59,10084,10043],{"class":2413},[59,10086,815],{"class":2405},[59,10088,10089],{"class":2455}," None\n",[59,10091,10092],{"class":2401,"line":9544},[59,10093,2725],{"emptyLinePlaceholder":2724},[59,10095,10096],{"class":2401,"line":9550},[59,10097,10098],{"class":2439},"        # 自注意力\n",[59,10100,10101,10104,10106,10108],{"class":2401,"line":9556},[59,10102,10103],{"class":2413},"        X2 ",[59,10105,815],{"class":2405},[59,10107,5220],{"class":2693},[59,10109,10110],{"class":2413},".attention1(X, key_values, key_values, dec_valid_lens)\n",[59,10112,10113,10115,10117,10119],{"class":2401,"line":9583},[59,10114,9253],{"class":2413},[59,10116,815],{"class":2405},[59,10118,5220],{"class":2693},[59,10120,10121],{"class":2413},".addnorm1(X, X2)\n",[59,10123,10124],{"class":2401,"line":9611},[59,10125,10126],{"class":2439},"        # 编码器－解码器注意力。\n",[59,10128,10129],{"class":2401,"line":9631},[59,10130,10131],{"class":2439},"        # enc_outputs的开头:(batch_size,num_steps,num_hiddens)\n",[59,10133,10134,10137,10139,10141],{"class":2401,"line":9642},[59,10135,10136],{"class":2413},"        Y2 ",[59,10138,815],{"class":2405},[59,10140,5220],{"class":2693},[59,10142,10143],{"class":2413},".attention2(Y, enc_outputs, enc_outputs, enc_valid_lens)\n",[59,10145,10146,10149,10151,10153],{"class":2401,"line":9655},[59,10147,10148],{"class":2413},"        Z ",[59,10150,815],{"class":2405},[59,10152,5220],{"class":2693},[59,10154,10155],{"class":2413},".addnorm2(Y, Y2)\n",[59,10157,10159,10161,10163,10166,10168],{"class":2401,"line":10158},45,[59,10160,2465],{"class":2405},[59,10162,5220],{"class":2693},[59,10164,10165],{"class":2413},".addnorm3(Z, ",[59,10167,2666],{"class":2693},[59,10169,10170],{"class":2413},".ffn(Z)), state\n",[59,10172,10174],{"class":2401,"line":10173},46,[59,10175,2725],{"emptyLinePlaceholder":2724},[59,10177,10179,10181,10184,10186,10189],{"class":2401,"line":10178},47,[59,10180,2630],{"class":2405},[59,10182,10183],{"class":2633}," TransformerDecoder",[59,10185,798],{"class":2413},[59,10187,10188],{"class":2639},"AttentionDecoder",[59,10190,2426],{"class":2413},[59,10192,10194,10196,10198,10200,10202,10204,10206,10208,10210,10212,10214,10216,10218],{"class":2401,"line":10193},48,[59,10195,2657],{"class":2405},[59,10197,2660],{"class":2455},[59,10199,798],{"class":2413},[59,10201,2666],{"class":2665},[59,10203,2420],{"class":2413},[59,10205,9317],{"class":2416},[59,10207,2420],{"class":2413},[59,10209,4847],{"class":2416},[59,10211,2420],{"class":2413},[59,10213,4852],{"class":2416},[59,10215,2420],{"class":2413},[59,10217,4857],{"class":2416},[59,10219,3826],{"class":2413},[59,10221,10223,10225,10227,10229,10231,10233,10235,10237],{"class":2401,"line":10222},49,[59,10224,9336],{"class":2416},[59,10226,2420],{"class":2413},[59,10228,9341],{"class":2416},[59,10230,2420],{"class":2413},[59,10232,7675],{"class":2416},[59,10234,2420],{"class":2413},[59,10236,7680],{"class":2416},[59,10238,3826],{"class":2413},[59,10240,10242,10244,10246,10248,10250,10252,10254,10256,10258],{"class":2401,"line":10241},50,[59,10243,4869],{"class":2416},[59,10245,2420],{"class":2413},[59,10247,9360],{"class":2416},[59,10249,2420],{"class":2413},[59,10251,2671],{"class":2416},[59,10253,2420],{"class":2413},[59,10255,2676],{"class":2405},[59,10257,2679],{"class":2416},[59,10259,2426],{"class":2413},[59,10261,10263,10265,10268,10270,10272,10274,10276,10278],{"class":2401,"line":10262},51,[59,10264,2687],{"class":2686},[59,10266,10267],{"class":2413},"(TransformerDecoder, ",[59,10269,2666],{"class":2693},[59,10271,2696],{"class":2413},[59,10273,2699],{"class":2455},[59,10275,798],{"class":2413},[59,10277,2676],{"class":2405},[59,10279,2706],{"class":2413},[59,10281,10283,10285,10287,10289],{"class":2401,"line":10282},52,[59,10284,2711],{"class":2693},[59,10286,9406],{"class":2413},[59,10288,815],{"class":2405},[59,10290,9411],{"class":2413},[59,10292,10294,10296,10299,10301],{"class":2401,"line":10293},53,[59,10295,2711],{"class":2693},[59,10297,10298],{"class":2413},".num_layers ",[59,10300,815],{"class":2405},[59,10302,10303],{"class":2413}," num_layers\n",[59,10305,10307,10309,10311,10313],{"class":2401,"line":10306},54,[59,10308,2711],{"class":2693},[59,10310,9418],{"class":2413},[59,10312,815],{"class":2405},[59,10314,9423],{"class":2413},[59,10316,10318,10320,10322,10324],{"class":2401,"line":10317},55,[59,10319,2711],{"class":2693},[59,10321,9430],{"class":2413},[59,10323,815],{"class":2405},[59,10325,9435],{"class":2413},[59,10327,10329,10331,10333,10335],{"class":2401,"line":10328},56,[59,10330,2711],{"class":2693},[59,10332,9442],{"class":2413},[59,10334,815],{"class":2405},[59,10336,9447],{"class":2413},[59,10338,10340,10342,10344,10346,10348],{"class":2401,"line":10339},57,[59,10341,9452],{"class":2405},[59,10343,9455],{"class":2413},[59,10345,9458],{"class":2405},[59,10347,9462],{"class":9461},[59,10349,9465],{"class":2413},[59,10351,10353,10355,10357,10359,10361,10363,10365,10367],{"class":2401,"line":10352},58,[59,10354,9470],{"class":2693},[59,10356,9473],{"class":2413},[59,10358,9477],{"class":9476},[59,10360,9481],{"class":9480},[59,10362,9477],{"class":9476},[59,10364,5777],{"class":2405},[59,10366,9488],{"class":2686},[59,10368,9491],{"class":2413},[59,10370,10372],{"class":2401,"line":10371},59,[59,10373,10374],{"class":2413},"                DecoderBlock(key_size, query_size, value_size, num_hiddens,\n",[59,10376,10378],{"class":2401,"line":10377},60,[59,10379,9501],{"class":2413},[59,10381,10383],{"class":2401,"line":10382},61,[59,10384,10385],{"class":2413},"                             num_heads, dropout, i))\n",[59,10387,10389,10391,10394,10396],{"class":2401,"line":10388},62,[59,10390,2711],{"class":2693},[59,10392,10393],{"class":2413},".dense ",[59,10395,815],{"class":2405},[59,10397,10398],{"class":2413}," nn.Linear(num_hiddens, vocab_size)\n",[59,10400,10402],{"class":2401,"line":10401},63,[59,10403,2725],{"emptyLinePlaceholder":2724},[59,10405,10407,10409,10411,10413,10415,10417,10419,10421,10424,10426,10428,10430],{"class":2401,"line":10406},64,[59,10408,2657],{"class":2405},[59,10410,8816],{"class":2409},[59,10412,798],{"class":2413},[59,10414,2666],{"class":2665},[59,10416,2420],{"class":2413},[59,10418,8825],{"class":2416},[59,10420,2420],{"class":2413},[59,10422,10423],{"class":2416},"enc_valid_lens",[59,10425,2420],{"class":2413},[59,10427,8729],{"class":2405},[59,10429,8732],{"class":2416},[59,10431,2426],{"class":2413},[59,10433,10435,10437,10440,10442,10444,10446,10448],{"class":2401,"line":10434},65,[59,10436,2465],{"class":2405},[59,10438,10439],{"class":2413}," [enc_outputs, enc_valid_lens, [",[59,10441,2780],{"class":2455},[59,10443,7200],{"class":2413},[59,10445,8729],{"class":2405},[59,10447,5220],{"class":2693},[59,10449,10450],{"class":2413},".num_layers]\n",[59,10452,10454],{"class":2401,"line":10453},66,[59,10455,2725],{"emptyLinePlaceholder":2724},[59,10457,10459,10461,10463,10465,10467,10469,10471,10473,10475],{"class":2401,"line":10458},67,[59,10460,2657],{"class":2405},[59,10462,2752],{"class":2409},[59,10464,798],{"class":2413},[59,10466,2666],{"class":2665},[59,10468,2420],{"class":2413},[59,10470,2417],{"class":2416},[59,10472,2420],{"class":2413},[59,10474,8862],{"class":2416},[59,10476,2426],{"class":2413},[59,10478,10480,10482,10484,10486,10488,10490,10492,10494,10496,10498],{"class":2401,"line":10479},68,[59,10481,2569],{"class":2413},[59,10483,815],{"class":2405},[59,10485,5220],{"class":2693},[59,10487,9565],{"class":2413},[59,10489,2666],{"class":2693},[59,10491,9570],{"class":2413},[59,10493,8729],{"class":2405},[59,10495,9575],{"class":2413},[59,10497,2666],{"class":2693},[59,10499,9580],{"class":2413},[59,10501,10503,10505,10508,10510,10513,10515,10517,10519,10521,10523,10525,10528,10531,10534,10536,10538,10541,10543],{"class":2401,"line":10502},69,[59,10504,2711],{"class":2693},[59,10506,10507],{"class":2413},"._attention_weights ",[59,10509,815],{"class":2405},[59,10511,10512],{"class":2413}," [[",[59,10514,2780],{"class":2455},[59,10516,7200],{"class":2413},[59,10518,8729],{"class":2405},[59,10520,9601],{"class":9461},[59,10522,798],{"class":2413},[59,10524,2666],{"class":2693},[59,10526,10527],{"class":2413},".blks) ",[59,10529,10530],{"class":2405},"for",[59,10532,10533],{"class":2413}," _ ",[59,10535,9458],{"class":2405},[59,10537,9462],{"class":9461},[59,10539,10540],{"class":2413}," (",[59,10542,2821],{"class":2455},[59,10544,10545],{"class":2413},")]\n",[59,10547,10549,10551,10553,10555,10557,10559,10561],{"class":2401,"line":10548},70,[59,10550,9452],{"class":2405},[59,10552,9616],{"class":2413},[59,10554,9458],{"class":2405},[59,10556,9621],{"class":9461},[59,10558,798],{"class":2413},[59,10560,2666],{"class":2693},[59,10562,9628],{"class":2413},[59,10564,10566,10569,10571],{"class":2401,"line":10565},71,[59,10567,10568],{"class":2413},"            X, state ",[59,10570,815],{"class":2405},[59,10572,10573],{"class":2413}," blk(X, state)\n",[59,10575,10577],{"class":2401,"line":10576},72,[59,10578,10579],{"class":2439},"            # 解码器自注意力权重\n",[59,10581,10583,10585,10588,10590,10593,10595],{"class":2401,"line":10582},73,[59,10584,9470],{"class":2693},[59,10586,10587],{"class":2413},"._attention_weights[",[59,10589,1754],{"class":2455},[59,10591,10592],{"class":2413},"][i] ",[59,10594,815],{"class":2405},[59,10596,10597],{"class":2413}," blk.attention1.attention.attention_weights\n",[59,10599,10601],{"class":2401,"line":10600},74,[59,10602,10603],{"class":2439},"            # “编码器－解码器”自注意力权重\n",[59,10605,10607,10609,10611,10613,10615,10617],{"class":2401,"line":10606},75,[59,10608,9470],{"class":2693},[59,10610,10587],{"class":2413},[59,10612,83],{"class":2455},[59,10614,10592],{"class":2413},[59,10616,815],{"class":2405},[59,10618,10619],{"class":2413}," blk.attention2.attention.attention_weights\n",[59,10621,10623,10625,10627],{"class":2401,"line":10622},76,[59,10624,2465],{"class":2405},[59,10626,5220],{"class":2693},[59,10628,10629],{"class":2413},".dense(X), state\n",[59,10631,10633],{"class":2401,"line":10632},77,[59,10634,2725],{"emptyLinePlaceholder":2724},[59,10636,10638,10641],{"class":2401,"line":10637},78,[59,10639,10640],{"class":2409},"    @",[59,10642,10643],{"class":2686},"property\n",[59,10645,10647,10649,10652,10654,10656],{"class":2401,"line":10646},79,[59,10648,2657],{"class":2405},[59,10650,10651],{"class":2409}," attention_weights",[59,10653,798],{"class":2413},[59,10655,2666],{"class":2665},[59,10657,2426],{"class":2413},[59,10659,10661,10663,10665],{"class":2401,"line":10660},80,[59,10662,2465],{"class":2405},[59,10664,5220],{"class":2693},[59,10666,10667],{"class":2413},"._attention_weights\n",[11,10669,10670],{"id":10670},"训练",[15,10672,10673],{},"同样，训练时需要屏蔽部分不需要的内容：",[2390,10675,10677],{"className":2392,"code":10676,"language":2394,"meta":2395,"style":2395},"def sequence_mask(X, valid_len, value=0):\n    \"\"\"在序列中屏蔽不相关的项\"\"\"\n    maxlen = X.size(1)\n    mask = torch.arange((maxlen), dtype=torch.float32,\n                        device=X.device)[None, :] \u003C valid_len[:, None]\n    X[~mask] = value\n    return X\n\nclass MaskedSoftmaxCELoss(nn.CrossEntropyLoss):\n    \"\"\"带遮蔽的softmax交叉熵损失函数\"\"\"\n    # pred的形状：(batch_size,num_steps,vocab_size)\n    # label的形状：(batch_size,num_steps)\n    # valid_len的形状：(batch_size,)\n    def forward(self, pred, label, valid_len):\n        weights = torch.ones_like(label)\n        weights = sequence_mask(weights, valid_len)\n        self.reduction='none'\n        unweighted_loss = super(MaskedSoftmaxCELoss, self).forward(\n            pred.permute(0, 2, 1), label)\n        weighted_loss = (unweighted_loss * weights).mean(dim=1)\n        return weighted_loss\n",[2397,10678,10679,10706,10711,10725,10743,10772,10788,10794,10798,10816,10821,10826,10831,10836,10862,10872,10881,10899,10917,10935,10958],{"__ignoreMap":2395},[59,10680,10681,10683,10686,10688,10690,10692,10695,10697,10700,10702,10704],{"class":2401,"line":2402},[59,10682,2406],{"class":2405},[59,10684,10685],{"class":2409}," sequence_mask",[59,10687,798],{"class":2413},[59,10689,2417],{"class":2416},[59,10691,2420],{"class":2413},[59,10693,10694],{"class":2416},"valid_len",[59,10696,2420],{"class":2413},[59,10698,10699],{"class":2416},"value",[59,10701,815],{"class":2405},[59,10703,1754],{"class":2455},[59,10705,2426],{"class":2413},[59,10707,10708],{"class":2401,"line":2429},[59,10709,10710],{"class":2432},"    \"\"\"在序列中屏蔽不相关的项\"\"\"\n",[59,10712,10713,10716,10718,10721,10723],{"class":2401,"line":2436},[59,10714,10715],{"class":2413},"    maxlen ",[59,10717,815],{"class":2405},[59,10719,10720],{"class":2413}," X.size(",[59,10722,83],{"class":2455},[59,10724,2480],{"class":2413},[59,10726,10727,10730,10732,10735,10738,10740],{"class":2401,"line":2443},[59,10728,10729],{"class":2413},"    mask ",[59,10731,815],{"class":2405},[59,10733,10734],{"class":2413}," torch.arange((maxlen), ",[59,10736,10737],{"class":2471},"dtype",[59,10739,815],{"class":2405},[59,10741,10742],{"class":2413},"torch.float32,\n",[59,10744,10745,10748,10750,10753,10755,10758,10761,10764,10766,10768,10770],{"class":2401,"line":2462},[59,10746,10747],{"class":2471},"                        device",[59,10749,815],{"class":2405},[59,10751,10752],{"class":2413},"X.device)[",[59,10754,2780],{"class":2455},[59,10756,10757],{"class":2413},", :] ",[59,10759,10760],{"class":2405},"\u003C",[59,10762,10763],{"class":2413}," valid_len[",[59,10765,7184],{"class":7183},[59,10767,2420],{"class":2413},[59,10769,2780],{"class":2455},[59,10771,2799],{"class":2413},[59,10773,10774,10777,10780,10783,10785],{"class":2401,"line":2483},[59,10775,10776],{"class":2413},"    X[",[59,10778,10779],{"class":2405},"~",[59,10781,10782],{"class":2413},"mask] ",[59,10784,815],{"class":2405},[59,10786,10787],{"class":2413}," value\n",[59,10789,10790,10792],{"class":2401,"line":2491},[59,10791,4687],{"class":2405},[59,10793,9660],{"class":2413},[59,10795,10796],{"class":2401,"line":2502},[59,10797,2725],{"emptyLinePlaceholder":2724},[59,10799,10800,10802,10805,10807,10809,10811,10814],{"class":2401,"line":2519},[59,10801,2630],{"class":2405},[59,10803,10804],{"class":2633}," MaskedSoftmaxCELoss",[59,10806,798],{"class":2413},[59,10808,2640],{"class":2639},[59,10810,864],{"class":2413},[59,10812,10813],{"class":2639},"CrossEntropyLoss",[59,10815,2426],{"class":2413},[59,10817,10818],{"class":2401,"line":2535},[59,10819,10820],{"class":2432},"    \"\"\"带遮蔽的softmax交叉熵损失函数\"\"\"\n",[59,10822,10823],{"class":2401,"line":2543},[59,10824,10825],{"class":2439},"    # pred的形状：(batch_size,num_steps,vocab_size)\n",[59,10827,10828],{"class":2401,"line":2560},[59,10829,10830],{"class":2439},"    # label的形状：(batch_size,num_steps)\n",[59,10832,10833],{"class":2401,"line":2566},[59,10834,10835],{"class":2439},"    # valid_len的形状：(batch_size,)\n",[59,10837,10838,10840,10842,10844,10846,10848,10851,10853,10856,10858,10860],{"class":2401,"line":2591},[59,10839,2657],{"class":2405},[59,10841,2752],{"class":2409},[59,10843,798],{"class":2413},[59,10845,2666],{"class":2665},[59,10847,2420],{"class":2413},[59,10849,10850],{"class":2416},"pred",[59,10852,2420],{"class":2413},[59,10854,10855],{"class":2416},"label",[59,10857,2420],{"class":2413},[59,10859,10694],{"class":2416},[59,10861,2426],{"class":2413},[59,10863,10864,10867,10869],{"class":2401,"line":2604},[59,10865,10866],{"class":2413},"        weights ",[59,10868,815],{"class":2405},[59,10870,10871],{"class":2413}," torch.ones_like(label)\n",[59,10873,10874,10876,10878],{"class":2401,"line":2845},[59,10875,10866],{"class":2413},[59,10877,815],{"class":2405},[59,10879,10880],{"class":2413}," sequence_mask(weights, valid_len)\n",[59,10882,10883,10885,10888,10890,10893,10896],{"class":2401,"line":4781},[59,10884,2711],{"class":2693},[59,10886,10887],{"class":2413},".reduction",[59,10889,815],{"class":2405},[59,10891,10892],{"class":9476},"'",[59,10894,10895],{"class":9480},"none",[59,10897,10898],{"class":9476},"'\n",[59,10900,10901,10904,10906,10909,10912,10914],{"class":2401,"line":5057},[59,10902,10903],{"class":2413},"        unweighted_loss ",[59,10905,815],{"class":2405},[59,10907,10908],{"class":2686}," super",[59,10910,10911],{"class":2413},"(MaskedSoftmaxCELoss, ",[59,10913,2666],{"class":2693},[59,10915,10916],{"class":2413},").forward(\n",[59,10918,10919,10922,10924,10926,10928,10930,10932],{"class":2401,"line":5063},[59,10920,10921],{"class":2413},"            pred.permute(",[59,10923,1754],{"class":2455},[59,10925,2420],{"class":2413},[59,10927,2821],{"class":2455},[59,10929,2420],{"class":2413},[59,10931,83],{"class":2455},[59,10933,10934],{"class":2413},"), label)\n",[59,10936,10937,10940,10942,10945,10947,10950,10952,10954,10956],{"class":2401,"line":5069},[59,10938,10939],{"class":2413},"        weighted_loss ",[59,10941,815],{"class":2405},[59,10943,10944],{"class":2413}," (unweighted_loss ",[59,10946,8729],{"class":2405},[59,10948,10949],{"class":2413}," weights).mean(",[59,10951,2472],{"class":2471},[59,10953,815],{"class":2405},[59,10955,83],{"class":2455},[59,10957,2480],{"class":2413},[59,10959,10960,10962],{"class":2401,"line":5075},[59,10961,2465],{"class":2405},[59,10963,10964],{"class":2413}," weighted_loss\n",[15,10966,10967],{},"梯度裁剪函数：",[2390,10969,10971],{"className":2392,"code":10970,"language":2394,"meta":2395,"style":2395},"def grad_clipping(net, theta):\n    \"\"\"裁剪梯度\"\"\"\n    if isinstance(net, nn.Module):\n        params = [p for p in net.parameters() if p.requires_grad]\n    else:\n        params = net.params\n    norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params))\n    if norm > theta:\n        for param in params:\n            param.grad[:] *= theta / norm\n",[2397,10972,10973,10992,10997,11007,11033,11039,11048,11080,11093,11105],{"__ignoreMap":2395},[59,10974,10975,10977,10980,10982,10985,10987,10990],{"class":2401,"line":2402},[59,10976,2406],{"class":2405},[59,10978,10979],{"class":2409}," grad_clipping",[59,10981,798],{"class":2413},[59,10983,10984],{"class":2416},"net",[59,10986,2420],{"class":2413},[59,10988,10989],{"class":2416},"theta",[59,10991,2426],{"class":2413},[59,10993,10994],{"class":2401,"line":2429},[59,10995,10996],{"class":2432},"    \"\"\"裁剪梯度\"\"\"\n",[59,10998,10999,11001,11004],{"class":2401,"line":2436},[59,11000,2446],{"class":2405},[59,11002,11003],{"class":9461}," isinstance",[59,11005,11006],{"class":2413},"(net, nn.Module):\n",[59,11008,11009,11012,11014,11017,11019,11022,11024,11027,11030],{"class":2401,"line":2443},[59,11010,11011],{"class":2413},"        params ",[59,11013,815],{"class":2405},[59,11015,11016],{"class":2413}," [p ",[59,11018,10530],{"class":2405},[59,11020,11021],{"class":2413}," p ",[59,11023,9458],{"class":2405},[59,11025,11026],{"class":2413}," net.parameters() ",[59,11028,11029],{"class":2405},"if",[59,11031,11032],{"class":2413}," p.requires_grad]\n",[59,11034,11035,11037],{"class":2401,"line":2462},[59,11036,2486],{"class":2405},[59,11038,2459],{"class":2413},[59,11040,11041,11043,11045],{"class":2401,"line":2483},[59,11042,11011],{"class":2413},[59,11044,815],{"class":2405},[59,11046,11047],{"class":2413}," net.params\n",[59,11049,11050,11053,11055,11058,11061,11064,11066,11069,11071,11073,11075,11077],{"class":2401,"line":2491},[59,11051,11052],{"class":2413},"    norm ",[59,11054,815],{"class":2405},[59,11056,11057],{"class":2413}," torch.sqrt(",[59,11059,11060],{"class":9461},"sum",[59,11062,11063],{"class":2413},"(torch.sum((p.grad ",[59,11065,2676],{"class":2405},[59,11067,11068],{"class":2455}," 2",[59,11070,2824],{"class":2413},[59,11072,10530],{"class":2405},[59,11074,11021],{"class":2413},[59,11076,9458],{"class":2405},[59,11078,11079],{"class":2413}," params))\n",[59,11081,11082,11084,11087,11090],{"class":2401,"line":2502},[59,11083,2446],{"class":2405},[59,11085,11086],{"class":2413}," norm ",[59,11088,11089],{"class":2405},">",[59,11091,11092],{"class":2413}," theta:\n",[59,11094,11095,11097,11100,11102],{"class":2401,"line":2519},[59,11096,9452],{"class":2405},[59,11098,11099],{"class":2413}," param ",[59,11101,9458],{"class":2405},[59,11103,11104],{"class":2413}," params:\n",[59,11106,11107,11110,11112,11114,11117,11120,11122],{"class":2401,"line":2535},[59,11108,11109],{"class":2413},"            param.grad[",[59,11111,7184],{"class":7183},[59,11113,7200],{"class":2413},[59,11115,11116],{"class":2405},"*=",[59,11118,11119],{"class":2413}," theta ",[59,11121,2827],{"class":2405},[59,11123,11124],{"class":2413}," norm\n",[15,11126,11127,11128],{},"之后，在训练时，特定的序列开始词元（“",[11129,11130,11131,11132],"bos",{},"”）和 原始的输出序列（不包括序列结束词元“",[11133,11134,11135],"eos",{},"”） 拼接在一起作为解码器的输入。 这被称为强制教学（teacher forcing）， 因为原始的输出序列（词元的标签）被送入解码器。 或者，将来自上一个时间步的预测得到的词元作为解码器的当前输入。",[2390,11137,11139],{"className":2392,"code":11138,"language":2394,"meta":2395,"style":2395},"def train_seq2seq(net, data_iter, lr, num_epochs, tgt_vocab, device):\n    \"\"\"训练序列到序列模型\"\"\"\n    def xavier_init_weights(m):\n        if type(m) == nn.Linear:\n            nn.init.xavier_uniform_(m.weight)\n        if type(m) == nn.GRU:\n            for param in m._flat_weights_names:\n                if \"weight\" in param:\n                    nn.init.xavier_uniform_(m._parameters[param])\n\n    net.apply(xavier_init_weights)\n    net.to(device)\n    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n    loss = MaskedSoftmaxCELoss()\n    net.train()\n    animator = d2l.Animator(xlabel='epoch', ylabel='loss',\n                     xlim=[10, num_epochs])\n    for epoch in range(num_epochs):\n        timer = d2l.Timer()\n        metric = d2l.Accumulator(2)  # 训练损失总和，词元数量\n        for batch in data_iter:\n            optimizer.zero_grad()\n            X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n            bos = torch.tensor([tgt_vocab['\u003Cbos>']] * Y.shape[0],\n                          device=device).reshape(-1, 1)\n            dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学\n            Y_hat, _ = net(X, dec_input, X_valid_len)\n            l = loss(Y_hat, Y, Y_valid_len)\n            l.sum().backward()      # 损失函数的标量进行“反向传播”\n            grad_clipping(net, 1)\n            num_tokens = Y_valid_len.sum()\n            optimizer.step()\n            with torch.no_grad():\n                metric.add(l.sum(), num_tokens)\n        if (epoch + 1) % 10 == 0:\n            animator.add(epoch + 1, (metric[0] / metric[1],))\n    print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} '\n        f'tokens/sec on {str(device)}')\n",[2397,11140,11141,11178,11183,11196,11211,11216,11234,11246,11265,11270,11274,11279,11284,11301,11311,11316,11354,11370,11385,11395,11413,11425,11430,11450,11480,11500,11530,11540,11550,11558,11567,11577,11582,11590,11595,11622,11648,11709],{"__ignoreMap":2395},[59,11142,11143,11145,11148,11150,11152,11154,11157,11159,11162,11164,11167,11169,11172,11174,11176],{"class":2401,"line":2402},[59,11144,2406],{"class":2405},[59,11146,11147],{"class":2409}," train_seq2seq",[59,11149,798],{"class":2413},[59,11151,10984],{"class":2416},[59,11153,2420],{"class":2413},[59,11155,11156],{"class":2416},"data_iter",[59,11158,2420],{"class":2413},[59,11160,11161],{"class":2416},"lr",[59,11163,2420],{"class":2413},[59,11165,11166],{"class":2416},"num_epochs",[59,11168,2420],{"class":2413},[59,11170,11171],{"class":2416},"tgt_vocab",[59,11173,2420],{"class":2413},[59,11175,10065],{"class":2416},[59,11177,2426],{"class":2413},[59,11179,11180],{"class":2401,"line":2429},[59,11181,11182],{"class":2432},"    \"\"\"训练序列到序列模型\"\"\"\n",[59,11184,11185,11187,11190,11192,11194],{"class":2401,"line":2436},[59,11186,2657],{"class":2405},[59,11188,11189],{"class":2409}," xavier_init_weights",[59,11191,798],{"class":2413},[59,11193,366],{"class":2416},[59,11195,2426],{"class":2413},[59,11197,11198,11200,11203,11206,11208],{"class":2401,"line":2443},[59,11199,2505],{"class":2405},[59,11201,11202],{"class":2686}," type",[59,11204,11205],{"class":2413},"(m) ",[59,11207,2511],{"class":2405},[59,11209,11210],{"class":2413}," nn.Linear:\n",[59,11212,11213],{"class":2401,"line":2462},[59,11214,11215],{"class":2413},"            nn.init.xavier_uniform_(m.weight)\n",[59,11217,11218,11220,11222,11224,11226,11229,11232],{"class":2401,"line":2483},[59,11219,2505],{"class":2405},[59,11221,11202],{"class":2686},[59,11223,11205],{"class":2413},[59,11225,2511],{"class":2405},[59,11227,11228],{"class":2413}," nn.",[59,11230,11231],{"class":2455},"GRU",[59,11233,2459],{"class":2413},[59,11235,11236,11239,11241,11243],{"class":2401,"line":2491},[59,11237,11238],{"class":2405},"            for",[59,11240,11099],{"class":2413},[59,11242,9458],{"class":2405},[59,11244,11245],{"class":2413}," m._flat_weights_names:\n",[59,11247,11248,11251,11254,11257,11259,11262],{"class":2401,"line":2502},[59,11249,11250],{"class":2405},"                if",[59,11252,11253],{"class":9476}," \"",[59,11255,11256],{"class":9480},"weight",[59,11258,9477],{"class":9476},[59,11260,11261],{"class":2405}," in",[59,11263,11264],{"class":2413}," param:\n",[59,11266,11267],{"class":2401,"line":2519},[59,11268,11269],{"class":2413},"                    nn.init.xavier_uniform_(m._parameters[param])\n",[59,11271,11272],{"class":2401,"line":2535},[59,11273,2725],{"emptyLinePlaceholder":2724},[59,11275,11276],{"class":2401,"line":2543},[59,11277,11278],{"class":2413},"    net.apply(xavier_init_weights)\n",[59,11280,11281],{"class":2401,"line":2560},[59,11282,11283],{"class":2413},"    net.to(device)\n",[59,11285,11286,11289,11291,11294,11296,11298],{"class":2401,"line":2566},[59,11287,11288],{"class":2413},"    optimizer ",[59,11290,815],{"class":2405},[59,11292,11293],{"class":2413}," torch.optim.Adam(net.parameters(), ",[59,11295,11161],{"class":2471},[59,11297,815],{"class":2405},[59,11299,11300],{"class":2413},"lr)\n",[59,11302,11303,11306,11308],{"class":2401,"line":2591},[59,11304,11305],{"class":2413},"    loss ",[59,11307,815],{"class":2405},[59,11309,11310],{"class":2413}," MaskedSoftmaxCELoss()\n",[59,11312,11313],{"class":2401,"line":2604},[59,11314,11315],{"class":2413},"    net.train()\n",[59,11317,11318,11321,11323,11326,11329,11331,11333,11336,11338,11340,11343,11345,11347,11350,11352],{"class":2401,"line":2845},[59,11319,11320],{"class":2413},"    animator ",[59,11322,815],{"class":2405},[59,11324,11325],{"class":2413}," d2l.Animator(",[59,11327,11328],{"class":2471},"xlabel",[59,11330,815],{"class":2405},[59,11332,10892],{"class":9476},[59,11334,11335],{"class":9480},"epoch",[59,11337,10892],{"class":9476},[59,11339,2420],{"class":2413},[59,11341,11342],{"class":2471},"ylabel",[59,11344,815],{"class":2405},[59,11346,10892],{"class":9476},[59,11348,11349],{"class":9480},"loss",[59,11351,10892],{"class":9476},[59,11353,3826],{"class":2413},[59,11355,11356,11359,11361,11364,11367],{"class":2401,"line":4781},[59,11357,11358],{"class":2471},"                     xlim",[59,11360,815],{"class":2405},[59,11362,11363],{"class":2413},"[",[59,11365,11366],{"class":2455},"10",[59,11368,11369],{"class":2413},", num_epochs])\n",[59,11371,11372,11375,11378,11380,11382],{"class":2401,"line":5057},[59,11373,11374],{"class":2405},"    for",[59,11376,11377],{"class":2413}," epoch ",[59,11379,9458],{"class":2405},[59,11381,9462],{"class":9461},[59,11383,11384],{"class":2413},"(num_epochs):\n",[59,11386,11387,11390,11392],{"class":2401,"line":5063},[59,11388,11389],{"class":2413},"        timer ",[59,11391,815],{"class":2405},[59,11393,11394],{"class":2413}," d2l.Timer()\n",[59,11396,11397,11400,11402,11405,11407,11410],{"class":2401,"line":5069},[59,11398,11399],{"class":2413},"        metric ",[59,11401,815],{"class":2405},[59,11403,11404],{"class":2413}," d2l.Accumulator(",[59,11406,2821],{"class":2455},[59,11408,11409],{"class":2413},")  ",[59,11411,11412],{"class":2439},"# 训练损失总和，词元数量\n",[59,11414,11415,11417,11420,11422],{"class":2401,"line":5075},[59,11416,9452],{"class":2405},[59,11418,11419],{"class":2413}," batch ",[59,11421,9458],{"class":2405},[59,11423,11424],{"class":2413}," data_iter:\n",[59,11426,11427],{"class":2401,"line":5096},[59,11428,11429],{"class":2413},"            optimizer.zero_grad()\n",[59,11431,11432,11435,11437,11440,11442,11445,11447],{"class":2401,"line":5115},[59,11433,11434],{"class":2413},"            X, X_valid_len, Y, Y_valid_len ",[59,11436,815],{"class":2405},[59,11438,11439],{"class":2413}," [x.to(device) ",[59,11441,10530],{"class":2405},[59,11443,11444],{"class":2413}," x ",[59,11446,9458],{"class":2405},[59,11448,11449],{"class":2413}," batch]\n",[59,11451,11452,11455,11457,11460,11462,11465,11467,11470,11472,11475,11477],{"class":2401,"line":5134},[59,11453,11454],{"class":2413},"            bos ",[59,11456,815],{"class":2405},[59,11458,11459],{"class":2413}," torch.tensor([tgt_vocab[",[59,11461,10892],{"class":9476},[59,11463,11464],{"class":9480},"\u003Cbos>",[59,11466,10892],{"class":9476},[59,11468,11469],{"class":2413},"]] ",[59,11471,8729],{"class":2405},[59,11473,11474],{"class":2413}," Y.shape[",[59,11476,1754],{"class":2455},[59,11478,11479],{"class":2413},"],\n",[59,11481,11482,11485,11487,11490,11492,11494,11496,11498],{"class":2401,"line":5139},[59,11483,11484],{"class":2471},"                          device",[59,11486,815],{"class":2405},[59,11488,11489],{"class":2413},"device).reshape(",[59,11491,2553],{"class":2405},[59,11493,83],{"class":2455},[59,11495,2420],{"class":2413},[59,11497,83],{"class":2455},[59,11499,2480],{"class":2413},[59,11501,11502,11505,11507,11510,11512,11514,11516,11518,11520,11523,11525,11527],{"class":2401,"line":5155},[59,11503,11504],{"class":2413},"            dec_input ",[59,11506,815],{"class":2405},[59,11508,11509],{"class":2413}," torch.cat([bos, Y[",[59,11511,7184],{"class":7183},[59,11513,2420],{"class":2413},[59,11515,7184],{"class":7183},[59,11517,2553],{"class":2405},[59,11519,83],{"class":2455},[59,11521,11522],{"class":2413},"]], ",[59,11524,83],{"class":2455},[59,11526,11409],{"class":2413},[59,11528,11529],{"class":2439},"# 强制教学\n",[59,11531,11532,11535,11537],{"class":2401,"line":5161},[59,11533,11534],{"class":2413},"            Y_hat, _ ",[59,11536,815],{"class":2405},[59,11538,11539],{"class":2413}," net(X, dec_input, X_valid_len)\n",[59,11541,11542,11545,11547],{"class":2401,"line":5167},[59,11543,11544],{"class":2413},"            l ",[59,11546,815],{"class":2405},[59,11548,11549],{"class":2413}," loss(Y_hat, Y, Y_valid_len)\n",[59,11551,11552,11555],{"class":2401,"line":5177},[59,11553,11554],{"class":2413},"            l.sum().backward()      ",[59,11556,11557],{"class":2439},"# 损失函数的标量进行“反向传播”\n",[59,11559,11560,11563,11565],{"class":2401,"line":5201},[59,11561,11562],{"class":2413},"            grad_clipping(net, ",[59,11564,83],{"class":2455},[59,11566,2480],{"class":2413},[59,11568,11569,11572,11574],{"class":2401,"line":5206},[59,11570,11571],{"class":2413},"            num_tokens ",[59,11573,815],{"class":2405},[59,11575,11576],{"class":2413}," Y_valid_len.sum()\n",[59,11578,11579],{"class":2401,"line":5212},[59,11580,11581],{"class":2413},"            optimizer.step()\n",[59,11583,11584,11587],{"class":2401,"line":5226},[59,11585,11586],{"class":2405},"            with",[59,11588,11589],{"class":2413}," torch.no_grad():\n",[59,11591,11592],{"class":2401,"line":5231},[59,11593,11594],{"class":2413},"                metric.add(l.sum(), num_tokens)\n",[59,11596,11597,11599,11602,11604,11606,11608,11611,11614,11617,11620],{"class":2401,"line":5237},[59,11598,2505],{"class":2405},[59,11600,11601],{"class":2413}," (epoch ",[59,11603,5777],{"class":2405},[59,11605,2514],{"class":2455},[59,11607,7146],{"class":2413},[59,11609,11610],{"class":2405},"%",[59,11612,11613],{"class":2455}," 10",[59,11615,11616],{"class":2405}," ==",[59,11618,11619],{"class":2455}," 0",[59,11621,2459],{"class":2413},[59,11623,11624,11627,11629,11631,11634,11636,11638,11640,11643,11645],{"class":2401,"line":5252},[59,11625,11626],{"class":2413},"            animator.add(epoch ",[59,11628,5777],{"class":2405},[59,11630,2514],{"class":2455},[59,11632,11633],{"class":2413},", (metric[",[59,11635,1754],{"class":2455},[59,11637,7200],{"class":2413},[59,11639,2827],{"class":2405},[59,11641,11642],{"class":2413}," metric[",[59,11644,83],{"class":2455},[59,11646,11647],{"class":2413},"],))\n",[59,11649,11650,11653,11655,11658,11661,11664,11667,11669,11671,11673,11675,11677,11680,11683,11686,11688,11690,11692,11694,11696,11698,11701,11704,11706],{"class":2401,"line":9544},[59,11651,11652],{"class":9461},"    print",[59,11654,798],{"class":2413},[59,11656,11657],{"class":2405},"f",[59,11659,11660],{"class":9480},"'loss ",[59,11662,11663],{"class":2455},"{",[59,11665,11666],{"class":2413},"metric[",[59,11668,1754],{"class":2455},[59,11670,7200],{"class":2413},[59,11672,2827],{"class":2405},[59,11674,11642],{"class":2413},[59,11676,83],{"class":2455},[59,11678,11679],{"class":2413},"]",[59,11681,11682],{"class":2405},":.3f",[59,11684,11685],{"class":2455},"}",[59,11687,2420],{"class":9480},[59,11689,11663],{"class":2455},[59,11691,11666],{"class":2413},[59,11693,83],{"class":2455},[59,11695,7200],{"class":2413},[59,11697,2827],{"class":2405},[59,11699,11700],{"class":2413}," timer.stop()",[59,11702,11703],{"class":2405},":.1f",[59,11705,11685],{"class":2455},[59,11707,11708],{"class":9480}," '\n",[59,11710,11711,11714,11717,11719,11721,11724,11726,11728],{"class":2401,"line":9550},[59,11712,11713],{"class":2405},"        f",[59,11715,11716],{"class":9480},"'tokens/sec on ",[59,11718,11663],{"class":2455},[59,11720,9488],{"class":2686},[59,11722,11723],{"class":2413},"(device)",[59,11725,11685],{"class":2455},[59,11727,10892],{"class":9480},[59,11729,2480],{"class":2413},[15,11731,11732],{},"进行数据集的数据处理",[2390,11734,11736],{"className":2392,"code":11735,"language":2394,"meta":2395,"style":2395},"def read_data_nmt():\n    \"\"\"载入“英语－法语”数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    data_dir = d2l.download_extract('fra-eng')\n    with open(os.path.join(data_dir, 'fra.txt'), 'r',\n             encoding='utf-8') as f:\n        return f.read()\n\ndef preprocess_nmt(text):\n    \"\"\"预处理“英语－法语”数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    def no_space(char, prev_char):\n        return char in set(',.!?') and prev_char != ' '\n\n    # 使用空格替换不间断空格\n    # 使用小写字母替换大写字母\n    text = text.replace('\\u202f', ' ').replace('\\xa0', ' ').lower()\n    # 在单词和标点符号之间插入空格\n    out = [' ' + char if i > 0 and no_space(char, text[i - 1]) else char\n           for i, char in enumerate(text)]\n    return ''.join(out)\n\ndef tokenize_nmt(text, num_examples=None):\n    \"\"\"词元化“英语－法语”数据数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    source, target = [], []\n    for i, line in enumerate(text.split('\\n')):\n        if num_examples and i > num_examples:\n            break\n        parts = line.split('\\t')\n        if len(parts) == 2:\n            source.append(parts[0].split(' '))\n            target.append(parts[1].split(' '))\n    return source, target\n\ndef load_data_nmt(batch_size, num_steps, num_examples=600):\n    \"\"\"返回翻译数据集的迭代器和词表\n\n    Defined in :numref:`subsec_mt_data_loading`\"\"\"\n    text = preprocess_nmt(read_data_nmt())\n    source, target = tokenize_nmt(text, num_examples)\n    src_vocab = d2l.Vocab(source, min_freq=2,\n                          reserved_tokens=['\u003Cpad>', '\u003Cbos>', '\u003Ceos>'])\n    tgt_vocab = d2l.Vocab(target, min_freq=2,\n                          reserved_tokens=['\u003Cpad>', '\u003Cbos>', '\u003Ceos>'])\n    src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)\n    tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps)\n    data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len)\n    data_iter = d2l.load_array(data_arrays, batch_size)\n    return data_iter, src_vocab, tgt_vocab\n",[2397,11737,11738,11748,11753,11757,11762,11781,11810,11832,11839,11843,11857,11862,11866,11870,11889,11926,11930,11935,11940,11983,11988,12033,12048,12058,12062,12084,12089,12093,12097,12107,12131,12147,12152,12171,12186,12203,12218,12225,12229,12257,12262,12266,12271,12280,12289,12308,12343,12361,12393,12403,12413,12423,12433],{"__ignoreMap":2395},[59,11739,11740,11742,11745],{"class":2401,"line":2402},[59,11741,2406],{"class":2405},[59,11743,11744],{"class":2409}," read_data_nmt",[59,11746,11747],{"class":2413},"():\n",[59,11749,11750],{"class":2401,"line":2429},[59,11751,11752],{"class":2432},"    \"\"\"载入“英语－法语”数据集\n",[59,11754,11755],{"class":2401,"line":2436},[59,11756,2725],{"emptyLinePlaceholder":2724},[59,11758,11759],{"class":2401,"line":2443},[59,11760,11761],{"class":2432},"    Defined in :numref:`sec_machine_translation`\"\"\"\n",[59,11763,11764,11767,11769,11772,11774,11777,11779],{"class":2401,"line":2462},[59,11765,11766],{"class":2413},"    data_dir ",[59,11768,815],{"class":2405},[59,11770,11771],{"class":2413}," d2l.download_extract(",[59,11773,10892],{"class":9476},[59,11775,11776],{"class":9480},"fra-eng",[59,11778,10892],{"class":9476},[59,11780,2480],{"class":2413},[59,11782,11783,11786,11789,11792,11794,11797,11799,11802,11804,11806,11808],{"class":2401,"line":2483},[59,11784,11785],{"class":2405},"    with",[59,11787,11788],{"class":9461}," open",[59,11790,11791],{"class":2413},"(os.path.join(data_dir, ",[59,11793,10892],{"class":9476},[59,11795,11796],{"class":9480},"fra.txt",[59,11798,10892],{"class":9476},[59,11800,11801],{"class":2413},"), ",[59,11803,10892],{"class":9476},[59,11805,7826],{"class":9480},[59,11807,10892],{"class":9476},[59,11809,3826],{"class":2413},[59,11811,11812,11815,11817,11819,11822,11824,11826,11829],{"class":2401,"line":2491},[59,11813,11814],{"class":2471},"             encoding",[59,11816,815],{"class":2405},[59,11818,10892],{"class":9476},[59,11820,11821],{"class":9480},"utf-8",[59,11823,10892],{"class":9476},[59,11825,7146],{"class":2413},[59,11827,11828],{"class":2405},"as",[59,11830,11831],{"class":2413}," f:\n",[59,11833,11834,11836],{"class":2401,"line":2502},[59,11835,2465],{"class":2405},[59,11837,11838],{"class":2413}," f.read()\n",[59,11840,11841],{"class":2401,"line":2519},[59,11842,2725],{"emptyLinePlaceholder":2724},[59,11844,11845,11847,11850,11852,11855],{"class":2401,"line":2535},[59,11846,2406],{"class":2405},[59,11848,11849],{"class":2409}," preprocess_nmt",[59,11851,798],{"class":2413},[59,11853,11854],{"class":2416},"text",[59,11856,2426],{"class":2413},[59,11858,11859],{"class":2401,"line":2543},[59,11860,11861],{"class":2432},"    \"\"\"预处理“英语－法语”数据集\n",[59,11863,11864],{"class":2401,"line":2560},[59,11865,2725],{"emptyLinePlaceholder":2724},[59,11867,11868],{"class":2401,"line":2566},[59,11869,11761],{"class":2432},[59,11871,11872,11874,11877,11879,11882,11884,11887],{"class":2401,"line":2591},[59,11873,2657],{"class":2405},[59,11875,11876],{"class":2409}," no_space",[59,11878,798],{"class":2413},[59,11880,11881],{"class":2416},"char",[59,11883,2420],{"class":2413},[59,11885,11886],{"class":2416},"prev_char",[59,11888,2426],{"class":2413},[59,11890,11891,11893,11896,11898,11901,11903,11905,11908,11910,11912,11915,11918,11921,11924],{"class":2401,"line":2604},[59,11892,2465],{"class":2405},[59,11894,11895],{"class":2413}," char ",[59,11897,9458],{"class":2405},[59,11899,11900],{"class":2686}," set",[59,11902,798],{"class":2413},[59,11904,10892],{"class":9476},[59,11906,11907],{"class":9480},",.!?",[59,11909,10892],{"class":9476},[59,11911,7146],{"class":2413},[59,11913,11914],{"class":2405},"and",[59,11916,11917],{"class":2413}," prev_char ",[59,11919,11920],{"class":2405},"!=",[59,11922,11923],{"class":9476}," '",[59,11925,11708],{"class":9476},[59,11927,11928],{"class":2401,"line":2845},[59,11929,2725],{"emptyLinePlaceholder":2724},[59,11931,11932],{"class":2401,"line":4781},[59,11933,11934],{"class":2439},"    # 使用空格替换不间断空格\n",[59,11936,11937],{"class":2401,"line":5057},[59,11938,11939],{"class":2439},"    # 使用小写字母替换大写字母\n",[59,11941,11942,11945,11947,11950,11952,11956,11958,11960,11962,11964,11967,11969,11972,11974,11976,11978,11980],{"class":2401,"line":5063},[59,11943,11944],{"class":2413},"    text ",[59,11946,815],{"class":2405},[59,11948,11949],{"class":2413}," text.replace(",[59,11951,10892],{"class":9476},[59,11953,11955],{"class":11954},"sRfyP","\\u202f",[59,11957,10892],{"class":9476},[59,11959,2420],{"class":2413},[59,11961,10892],{"class":9476},[59,11963,11923],{"class":9476},[59,11965,11966],{"class":2413},").replace(",[59,11968,10892],{"class":9476},[59,11970,11971],{"class":11954},"\\xa0",[59,11973,10892],{"class":9476},[59,11975,2420],{"class":2413},[59,11977,10892],{"class":9476},[59,11979,11923],{"class":9476},[59,11981,11982],{"class":2413},").lower()\n",[59,11984,11985],{"class":2401,"line":5069},[59,11986,11987],{"class":2439},"    # 在单词和标点符号之间插入空格\n",[59,11989,11990,11993,11995,11997,11999,12001,12004,12006,12008,12010,12012,12014,12017,12020,12022,12024,12027,12030],{"class":2401,"line":5075},[59,11991,11992],{"class":2413},"    out ",[59,11994,815],{"class":2405},[59,11996,9592],{"class":2413},[59,11998,10892],{"class":9476},[59,12000,11923],{"class":9476},[59,12002,12003],{"class":2405}," +",[59,12005,11895],{"class":2413},[59,12007,11029],{"class":2405},[59,12009,9455],{"class":2413},[59,12011,11089],{"class":2405},[59,12013,11619],{"class":2455},[59,12015,12016],{"class":2405}," and",[59,12018,12019],{"class":2413}," no_space(char, text[i ",[59,12021,2553],{"class":2405},[59,12023,2514],{"class":2455},[59,12025,12026],{"class":2413},"]) ",[59,12028,12029],{"class":2405},"else",[59,12031,12032],{"class":2413}," char\n",[59,12034,12035,12038,12041,12043,12045],{"class":2401,"line":5096},[59,12036,12037],{"class":2405},"           for",[59,12039,12040],{"class":2413}," i, char ",[59,12042,9458],{"class":2405},[59,12044,9621],{"class":9461},[59,12046,12047],{"class":2413},"(text)]\n",[59,12049,12050,12052,12055],{"class":2401,"line":5115},[59,12051,4687],{"class":2405},[59,12053,12054],{"class":9476}," ''",[59,12056,12057],{"class":2413},".join(out)\n",[59,12059,12060],{"class":2401,"line":5134},[59,12061,2725],{"emptyLinePlaceholder":2724},[59,12063,12064,12066,12069,12071,12073,12075,12078,12080,12082],{"class":2401,"line":5139},[59,12065,2406],{"class":2405},[59,12067,12068],{"class":2409}," tokenize_nmt",[59,12070,798],{"class":2413},[59,12072,11854],{"class":2416},[59,12074,2420],{"class":2413},[59,12076,12077],{"class":2416},"num_examples",[59,12079,815],{"class":2405},[59,12081,2780],{"class":2455},[59,12083,2426],{"class":2413},[59,12085,12086],{"class":2401,"line":5155},[59,12087,12088],{"class":2432},"    \"\"\"词元化“英语－法语”数据数据集\n",[59,12090,12091],{"class":2401,"line":5161},[59,12092,2725],{"emptyLinePlaceholder":2724},[59,12094,12095],{"class":2401,"line":5167},[59,12096,11761],{"class":2432},[59,12098,12099,12102,12104],{"class":2401,"line":5177},[59,12100,12101],{"class":2413},"    source, target ",[59,12103,815],{"class":2405},[59,12105,12106],{"class":2413}," [], []\n",[59,12108,12109,12111,12114,12116,12118,12121,12123,12126,12128],{"class":2401,"line":5201},[59,12110,11374],{"class":2405},[59,12112,12113],{"class":2413}," i, line ",[59,12115,9458],{"class":2405},[59,12117,9621],{"class":9461},[59,12119,12120],{"class":2413},"(text.split(",[59,12122,10892],{"class":9476},[59,12124,12125],{"class":11954},"\\n",[59,12127,10892],{"class":9476},[59,12129,12130],{"class":2413},")):\n",[59,12132,12133,12135,12138,12140,12142,12144],{"class":2401,"line":5206},[59,12134,2505],{"class":2405},[59,12136,12137],{"class":2413}," num_examples ",[59,12139,11914],{"class":2405},[59,12141,9455],{"class":2413},[59,12143,11089],{"class":2405},[59,12145,12146],{"class":2413}," num_examples:\n",[59,12148,12149],{"class":2401,"line":5212},[59,12150,12151],{"class":2405},"            break\n",[59,12153,12154,12157,12159,12162,12164,12167,12169],{"class":2401,"line":5226},[59,12155,12156],{"class":2413},"        parts ",[59,12158,815],{"class":2405},[59,12160,12161],{"class":2413}," line.split(",[59,12163,10892],{"class":9476},[59,12165,12166],{"class":11954},"\\t",[59,12168,10892],{"class":9476},[59,12170,2480],{"class":2413},[59,12172,12173,12175,12177,12180,12182,12184],{"class":2401,"line":5231},[59,12174,2505],{"class":2405},[59,12176,9601],{"class":9461},[59,12178,12179],{"class":2413},"(parts) ",[59,12181,2511],{"class":2405},[59,12183,11068],{"class":2455},[59,12185,2459],{"class":2413},[59,12187,12188,12191,12193,12196,12198,12200],{"class":2401,"line":5237},[59,12189,12190],{"class":2413},"            source.append(parts[",[59,12192,1754],{"class":2455},[59,12194,12195],{"class":2413},"].split(",[59,12197,10892],{"class":9476},[59,12199,11923],{"class":9476},[59,12201,12202],{"class":2413},"))\n",[59,12204,12205,12208,12210,12212,12214,12216],{"class":2401,"line":5252},[59,12206,12207],{"class":2413},"            target.append(parts[",[59,12209,83],{"class":2455},[59,12211,12195],{"class":2413},[59,12213,10892],{"class":9476},[59,12215,11923],{"class":9476},[59,12217,12202],{"class":2413},[59,12219,12220,12222],{"class":2401,"line":9544},[59,12221,4687],{"class":2405},[59,12223,12224],{"class":2413}," source, target\n",[59,12226,12227],{"class":2401,"line":9550},[59,12228,2725],{"emptyLinePlaceholder":2724},[59,12230,12231,12233,12236,12238,12241,12243,12246,12248,12250,12252,12255],{"class":2401,"line":9556},[59,12232,2406],{"class":2405},[59,12234,12235],{"class":2409}," load_data_nmt",[59,12237,798],{"class":2413},[59,12239,12240],{"class":2416},"batch_size",[59,12242,2420],{"class":2413},[59,12244,12245],{"class":2416},"num_steps",[59,12247,2420],{"class":2413},[59,12249,12077],{"class":2416},[59,12251,815],{"class":2405},[59,12253,12254],{"class":2455},"600",[59,12256,2426],{"class":2413},[59,12258,12259],{"class":2401,"line":9583},[59,12260,12261],{"class":2432},"    \"\"\"返回翻译数据集的迭代器和词表\n",[59,12263,12264],{"class":2401,"line":9611},[59,12265,2725],{"emptyLinePlaceholder":2724},[59,12267,12268],{"class":2401,"line":9631},[59,12269,12270],{"class":2432},"    Defined in :numref:`subsec_mt_data_loading`\"\"\"\n",[59,12272,12273,12275,12277],{"class":2401,"line":9642},[59,12274,11944],{"class":2413},[59,12276,815],{"class":2405},[59,12278,12279],{"class":2413}," preprocess_nmt(read_data_nmt())\n",[59,12281,12282,12284,12286],{"class":2401,"line":9655},[59,12283,12101],{"class":2413},[59,12285,815],{"class":2405},[59,12287,12288],{"class":2413}," tokenize_nmt(text, num_examples)\n",[59,12290,12291,12294,12296,12299,12302,12304,12306],{"class":2401,"line":10158},[59,12292,12293],{"class":2413},"    src_vocab ",[59,12295,815],{"class":2405},[59,12297,12298],{"class":2413}," d2l.Vocab(source, ",[59,12300,12301],{"class":2471},"min_freq",[59,12303,815],{"class":2405},[59,12305,2821],{"class":2455},[59,12307,3826],{"class":2413},[59,12309,12310,12313,12315,12317,12319,12322,12324,12326,12328,12330,12332,12334,12336,12339,12341],{"class":2401,"line":10173},[59,12311,12312],{"class":2471},"                          reserved_tokens",[59,12314,815],{"class":2405},[59,12316,11363],{"class":2413},[59,12318,10892],{"class":9476},[59,12320,12321],{"class":9480},"\u003Cpad>",[59,12323,10892],{"class":9476},[59,12325,2420],{"class":2413},[59,12327,10892],{"class":9476},[59,12329,11464],{"class":9480},[59,12331,10892],{"class":9476},[59,12333,2420],{"class":2413},[59,12335,10892],{"class":9476},[59,12337,12338],{"class":9480},"\u003Ceos>",[59,12340,10892],{"class":9476},[59,12342,2532],{"class":2413},[59,12344,12345,12348,12350,12353,12355,12357,12359],{"class":2401,"line":10178},[59,12346,12347],{"class":2413},"    tgt_vocab ",[59,12349,815],{"class":2405},[59,12351,12352],{"class":2413}," d2l.Vocab(target, ",[59,12354,12301],{"class":2471},[59,12356,815],{"class":2405},[59,12358,2821],{"class":2455},[59,12360,3826],{"class":2413},[59,12362,12363,12365,12367,12369,12371,12373,12375,12377,12379,12381,12383,12385,12387,12389,12391],{"class":2401,"line":10193},[59,12364,12312],{"class":2471},[59,12366,815],{"class":2405},[59,12368,11363],{"class":2413},[59,12370,10892],{"class":9476},[59,12372,12321],{"class":9480},[59,12374,10892],{"class":9476},[59,12376,2420],{"class":2413},[59,12378,10892],{"class":9476},[59,12380,11464],{"class":9480},[59,12382,10892],{"class":9476},[59,12384,2420],{"class":2413},[59,12386,10892],{"class":9476},[59,12388,12338],{"class":9480},[59,12390,10892],{"class":9476},[59,12392,2532],{"class":2413},[59,12394,12395,12398,12400],{"class":2401,"line":10222},[59,12396,12397],{"class":2413},"    src_array, src_valid_len ",[59,12399,815],{"class":2405},[59,12401,12402],{"class":2413}," build_array_nmt(source, src_vocab, num_steps)\n",[59,12404,12405,12408,12410],{"class":2401,"line":10241},[59,12406,12407],{"class":2413},"    tgt_array, tgt_valid_len ",[59,12409,815],{"class":2405},[59,12411,12412],{"class":2413}," build_array_nmt(target, tgt_vocab, num_steps)\n",[59,12414,12415,12418,12420],{"class":2401,"line":10262},[59,12416,12417],{"class":2413},"    data_arrays ",[59,12419,815],{"class":2405},[59,12421,12422],{"class":2413}," (src_array, src_valid_len, tgt_array, tgt_valid_len)\n",[59,12424,12425,12428,12430],{"class":2401,"line":10282},[59,12426,12427],{"class":2413},"    data_iter ",[59,12429,815],{"class":2405},[59,12431,12432],{"class":2413}," d2l.load_array(data_arrays, batch_size)\n",[59,12434,12435,12437],{"class":2401,"line":10293},[59,12436,4687],{"class":2405},[59,12438,12439],{"class":2413}," data_iter, src_vocab, tgt_vocab\n",[15,12441,12442],{},"之后按照 transformer 架构创建编码器-解码器架构，",[2390,12444,12446],{"className":2392,"code":12445,"language":2394,"meta":2395,"style":2395},"num_hiddens, num_layers, dropout, batch_size, num_steps = 32, 2, 0.1, 64, 10\nlr, num_epochs, device = 0.005, 200, torch.device('cuda:0') if torch.cuda.device_count() >= 1 else torch.device('cpu')\nffn_num_hiddens, num_heads = 64, 4\n\ntrain_iter, src_vocab, tgt_vocab = d2l.load_data_nmt(batch_size, num_steps)\n\nencoder = TransformerEncoder(\n    len(src_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n    dropout)\ndecoder = TransformerDecoder(\n    len(tgt_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n    dropout)\nnet = EncoderDecoder(encoder, decoder)\ntrain_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)\n",[2397,12447,12448,12477,12529,12544,12548,12558,12562,12572,12580,12585,12595,12602,12606,12616],{"__ignoreMap":2395},[59,12449,12450,12453,12455,12458,12460,12462,12464,12467,12469,12472,12474],{"class":2401,"line":2402},[59,12451,12452],{"class":2413},"num_hiddens, num_layers, dropout, batch_size, num_steps ",[59,12454,815],{"class":2405},[59,12456,12457],{"class":2455}," 32",[59,12459,2420],{"class":2413},[59,12461,2821],{"class":2455},[59,12463,2420],{"class":2413},[59,12465,12466],{"class":2455},"0.1",[59,12468,2420],{"class":2413},[59,12470,12471],{"class":2455},"64",[59,12473,2420],{"class":2413},[59,12475,12476],{"class":2455},"10\n",[59,12478,12479,12482,12484,12487,12489,12492,12495,12497,12500,12502,12504,12506,12509,12512,12514,12517,12520,12522,12525,12527],{"class":2401,"line":2429},[59,12480,12481],{"class":2413},"lr, num_epochs, device ",[59,12483,815],{"class":2405},[59,12485,12486],{"class":2455}," 0.005",[59,12488,2420],{"class":2413},[59,12490,12491],{"class":2455},"200",[59,12493,12494],{"class":2413},", torch.device(",[59,12496,10892],{"class":9476},[59,12498,12499],{"class":9480},"cuda:0",[59,12501,10892],{"class":9476},[59,12503,7146],{"class":2413},[59,12505,11029],{"class":2405},[59,12507,12508],{"class":2413}," torch.cuda.device_count() ",[59,12510,12511],{"class":2405},">=",[59,12513,2514],{"class":2455},[59,12515,12516],{"class":2405}," else",[59,12518,12519],{"class":2413}," torch.device(",[59,12521,10892],{"class":9476},[59,12523,12524],{"class":9480},"cpu",[59,12526,10892],{"class":9476},[59,12528,2480],{"class":2413},[59,12530,12531,12534,12536,12539,12541],{"class":2401,"line":2436},[59,12532,12533],{"class":2413},"ffn_num_hiddens, num_heads ",[59,12535,815],{"class":2405},[59,12537,12538],{"class":2455}," 64",[59,12540,2420],{"class":2413},[59,12542,12543],{"class":2455},"4\n",[59,12545,12546],{"class":2401,"line":2443},[59,12547,2725],{"emptyLinePlaceholder":2724},[59,12549,12550,12553,12555],{"class":2401,"line":2462},[59,12551,12552],{"class":2413},"train_iter, src_vocab, tgt_vocab ",[59,12554,815],{"class":2405},[59,12556,12557],{"class":2413}," d2l.load_data_nmt(batch_size, num_steps)\n",[59,12559,12560],{"class":2401,"line":2483},[59,12561,2725],{"emptyLinePlaceholder":2724},[59,12563,12564,12567,12569],{"class":2401,"line":2491},[59,12565,12566],{"class":2413},"encoder ",[59,12568,815],{"class":2405},[59,12570,12571],{"class":2413}," TransformerEncoder(\n",[59,12573,12574,12577],{"class":2401,"line":2502},[59,12575,12576],{"class":9461},"    len",[59,12578,12579],{"class":2413},"(src_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n",[59,12581,12582],{"class":2401,"line":2519},[59,12583,12584],{"class":2413},"    dropout)\n",[59,12586,12587,12590,12592],{"class":2401,"line":2535},[59,12588,12589],{"class":2413},"decoder ",[59,12591,815],{"class":2405},[59,12593,12594],{"class":2413}," TransformerDecoder(\n",[59,12596,12597,12599],{"class":2401,"line":2543},[59,12598,12576],{"class":9461},[59,12600,12601],{"class":2413},"(tgt_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n",[59,12603,12604],{"class":2401,"line":2560},[59,12605,12584],{"class":2413},[59,12607,12608,12611,12613],{"class":2401,"line":2566},[59,12609,12610],{"class":2413},"net ",[59,12612,815],{"class":2405},[59,12614,12615],{"class":2413}," EncoderDecoder(encoder, decoder)\n",[59,12617,12618],{"class":2401,"line":2591},[59,12619,12620],{"class":2413},"train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)\n",[11,12622,12623],{"id":12623},"参考文献",[15,12625,12626],{},[22,12627,33],{"href":31,"rel":12628},[26],[15,12630,12631],{},[22,12632,27],{"href":24,"rel":12633},[26],[12635,12636,12637],"style",{},"html pre.shiki code .szJfE, html code.shiki .szJfE{--shiki-default:#D73A49;--shiki-dark:#FF79C6}html pre.shiki code .sCLZk, html code.shiki .sCLZk{--shiki-default:#6F42C1;--shiki-dark:#50FA7B}html pre.shiki code .scbbO, html code.shiki .scbbO{--shiki-default:#24292E;--shiki-dark:#F8F8F2}html pre.shiki code .syNf4, html code.shiki .syNf4{--shiki-default:#24292E;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic}html pre.shiki code .seLWX, html code.shiki .seLWX{--shiki-default:#032F62;--shiki-dark:#6272A4}html pre.shiki code .sfgPZ, html code.shiki .sfgPZ{--shiki-default:#6A737D;--shiki-dark:#6272A4}html pre.shiki code .soDru, html code.shiki .soDru{--shiki-default:#005CC5;--shiki-dark:#BD93F9}html pre.shiki code .sQkXh, html code.shiki .sQkXh{--shiki-default:#E36209;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .skCyd, html code.shiki .skCyd{--shiki-default:#6F42C1;--shiki-dark:#8BE9FD}html pre.shiki code .sDP9b, html code.shiki .sDP9b{--shiki-default:#6F42C1;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .sD3jg, html code.shiki .sD3jg{--shiki-default:#24292E;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html pre.shiki code .sPGBF, html code.shiki .sPGBF{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .sJti5, html code.shiki .sJti5{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html pre.shiki code .sDoOe, html code.shiki .sDoOe{--shiki-default:#24292E;--shiki-dark:#FF79C6}html pre.shiki code .sDgm9, html code.shiki .sDgm9{--shiki-default:#005CC5;--shiki-dark:#8BE9FD}html pre.shiki code .sMWOi, html code.shiki .sMWOi{--shiki-default:#032F62;--shiki-dark:#E9F284}html pre.shiki code .sEzAm, html code.shiki .sEzAm{--shiki-default:#032F62;--shiki-dark:#F1FA8C}html pre.shiki code .sRfyP, html code.shiki .sRfyP{--shiki-default:#005CC5;--shiki-dark:#FF79C6}",{"title":2395,"searchDepth":2429,"depth":2429,"links":12639},[12640,12641,12642,12646,12647,12648,12649,12650,12651,12652,12653],{"id":13,"depth":2429,"text":13},{"id":37,"depth":2429,"text":37},{"id":47,"depth":2429,"text":47,"children":12643},[12644,12645],{"id":51,"depth":2436,"text":51},{"id":2870,"depth":2436,"text":2870},{"id":5262,"depth":2429,"text":5262},{"id":7297,"depth":2429,"text":7297},{"id":7793,"depth":2429,"text":7793},{"id":8130,"depth":2429,"text":8130},{"id":9044,"depth":2429,"text":9045},{"id":9663,"depth":2429,"text":9664},{"id":10670,"depth":2429,"text":10670},{"id":12623,"depth":2429,"text":12623},"距离首次学习 transformer 已经过去一年，内容忘的差不多了，决定复习一下。","md",{"date":12657,"image":12658,"alt":6,"tags":12659,"published":2724},"30st Jan 2026","/blogs-img/blog1.jpg",[12660,12661],"deep-learning","学习笔记","/blogs/transformer",{"title":6,"description":12654},"blogs/1. transformer复习笔记","59tIbWfqdpJGJRh5PHVoEHC170F6vnech8kVXdG8MGw",{"id":12667,"title":12668,"body":12669,"description":43939,"extension":12655,"meta":43940,"navigation":2724,"ogImage":43942,"path":43947,"seo":43948,"stem":43949,"__hash__":43950},"content/blogs/2. PPO学习笔记.md","零基础速通 PPO 学习笔记",{"type":8,"value":12670,"toc":43926},[12671,12674,12677,12680,13188,13491,13940,13943,13946,14008,14347,18339,18940,19459,19463,19538,20202,20446,20565,20893,20896,21582,22721,22823,22826,25458,25564,27436,27440,27443,27477,30259,30262,30265,30268,31755,31758,31762,31765,31768,31903,33567,33570,33574,34217,34408,38933,38936,39981,39984,39989,40025,41316,41320,41323,41932,43916,43919],[11,12672,12673],{"id":12673},"学习背景",[15,12675,12676],{},"最近陶瓷到了港科的一个 RA，组里在做 Agent 强化学习相关内容，听从学长的指导来学习 PPO 算法相关知识。",[11,12678,12679],{"id":12679},"符号定义",[15,12681,12682,12683,12755,12756,12785,12786,12857,12858,12886,12887,12916,12917,13054,13055,13126,13127,13155,13156,13187],{},"在强化学习中，",[59,12684,12686,12704],{"className":12685},[62],[59,12687,12689],{"className":12688},[66],[68,12690,12691],{"xmlns":70},[72,12692,12693,12701],{},[75,12694,12695],{},[88,12696,12697,12699],{},[91,12698,22],{},[91,12700,777],{},[98,12702,12703],{"encoding":100},"a_t",[59,12705,12707],{"className":12706,"ariaHidden":106},[105],[59,12708,12710,12714],{"className":12709},[110],[59,12711],{"className":12712,"style":12713},[114],"height:0.5806em;vertical-align:-0.15em;",[59,12715,12717,12720],{"className":12716},[119],[59,12718,22],{"className":12719},[119,192],[59,12721,12723],{"className":12722},[196],[59,12724,12726,12747],{"className":12725},[131,132],[59,12727,12729,12744],{"className":12728},[136],[59,12730,12733],{"className":12731,"style":12732},[140],"height:0.2806em;",[59,12734,12735,12738],{"style":1251},[59,12736],{"className":12737,"style":540},[148],[59,12739,12741],{"className":12740},[153,154,155,156],[59,12742,777],{"className":12743},[119,192,156],[59,12745,227],{"className":12746},[226],[59,12748,12750],{"className":12749},[136],[59,12751,12753],{"className":12752,"style":1270},[140],[59,12754],{}," 一般表示 ",[59,12757,12759,12772],{"className":12758},[62],[59,12760,12762],{"className":12761},[66],[68,12763,12764],{"xmlns":70},[72,12765,12766,12770],{},[75,12767,12768],{},[91,12769,777],{},[98,12771,777],{"encoding":100},[59,12773,12775],{"className":12774,"ariaHidden":106},[105],[59,12776,12778,12782],{"className":12777},[110],[59,12779],{"className":12780,"style":12781},[114],"height:0.6151em;",[59,12783,777],{"className":12784},[119,192]," 时刻 Agent 采取的动作（action）；",[59,12787,12789,12808],{"className":12788},[62],[59,12790,12792],{"className":12791},[66],[68,12793,12794],{"xmlns":70},[72,12795,12796,12805],{},[75,12797,12798],{},[88,12799,12800,12803],{},[91,12801,12802],{},"s",[91,12804,777],{},[98,12806,12807],{"encoding":100},"s_t",[59,12809,12811],{"className":12810,"ariaHidden":106},[105],[59,12812,12814,12817],{"className":12813},[110],[59,12815],{"className":12816,"style":12713},[114],[59,12818,12820,12823],{"className":12819},[119],[59,12821,12802],{"className":12822},[119,192],[59,12824,12826],{"className":12825},[196],[59,12827,12829,12849],{"className":12828},[131,132],[59,12830,12832,12846],{"className":12831},[136],[59,12833,12835],{"className":12834,"style":12732},[140],[59,12836,12837,12840],{"style":1251},[59,12838],{"className":12839,"style":540},[148],[59,12841,12843],{"className":12842},[153,154,155,156],[59,12844,777],{"className":12845},[119,192,156],[59,12847,227],{"className":12848},[226],[59,12850,12852],{"className":12851},[136],[59,12853,12855],{"className":12854,"style":1270},[140],[59,12856],{}," 表示 ",[59,12859,12861,12874],{"className":12860},[62],[59,12862,12864],{"className":12863},[66],[68,12865,12866],{"xmlns":70},[72,12867,12868,12872],{},[75,12869,12870],{},[91,12871,777],{},[98,12873,777],{"encoding":100},[59,12875,12877],{"className":12876,"ariaHidden":106},[105],[59,12878,12880,12883],{"className":12879},[110],[59,12881],{"className":12882,"style":12781},[114],[59,12884,777],{"className":12885},[119,192]," 时刻 Agent 所处的状态（state）；",[59,12888,12890,12904],{"className":12889},[62],[59,12891,12893],{"className":12892},[66],[68,12894,12895],{"xmlns":70},[72,12896,12897,12901],{},[75,12898,12899],{},[91,12900,6742],{},[98,12902,12903],{"encoding":100},"\\pi",[59,12905,12907],{"className":12906,"ariaHidden":106},[105],[59,12908,12910,12913],{"className":12909},[110],[59,12911],{"className":12912,"style":347},[114],[59,12914,6742],{"className":12915,"style":441},[119,192]," 表示 Agent的策略函数（policy），输入状态，输入采取每个动作的概率分布 ",[59,12918,12920,12953],{"className":12919},[62],[59,12921,12923],{"className":12922},[66],[68,12924,12925],{"xmlns":70},[72,12926,12927,12950],{},[75,12928,12929,12931,12933,12939,12942,12948],{},[91,12930,6742],{},[461,12932,798],{"stretchy":797},[88,12934,12935,12937],{},[91,12936,22],{},[91,12938,777],{},[91,12940,12941],{"mathvariant":818},"∣",[88,12943,12944,12946],{},[91,12945,12802],{},[91,12947,777],{},[461,12949,812],{"stretchy":797},[98,12951,12952],{"encoding":100},"\\pi(a_t|s_t)",[59,12954,12956],{"className":12955,"ariaHidden":106},[105],[59,12957,12959,12962,12965,12968,13008,13011,13051],{"className":12958},[110],[59,12960],{"className":12961,"style":877},[114],[59,12963,6742],{"className":12964,"style":441},[119,192],[59,12966,798],{"className":12967},[123],[59,12969,12971,12974],{"className":12970},[119],[59,12972,22],{"className":12973},[119,192],[59,12975,12977],{"className":12976},[196],[59,12978,12980,13000],{"className":12979},[131,132],[59,12981,12983,12997],{"className":12982},[136],[59,12984,12986],{"className":12985,"style":12732},[140],[59,12987,12988,12991],{"style":1251},[59,12989],{"className":12990,"style":540},[148],[59,12992,12994],{"className":12993},[153,154,155,156],[59,12995,777],{"className":12996},[119,192,156],[59,12998,227],{"className":12999},[226],[59,13001,13003],{"className":13002},[136],[59,13004,13006],{"className":13005,"style":1270},[140],[59,13007],{},[59,13009,12941],{"className":13010},[119],[59,13012,13014,13017],{"className":13013},[119],[59,13015,12802],{"className":13016},[119,192],[59,13018,13020],{"className":13019},[196],[59,13021,13023,13043],{"className":13022},[131,132],[59,13024,13026,13040],{"className":13025},[136],[59,13027,13029],{"className":13028,"style":12732},[140],[59,13030,13031,13034],{"style":1251},[59,13032],{"className":13033,"style":540},[148],[59,13035,13037],{"className":13036},[153,154,155,156],[59,13038,777],{"className":13039},[119,192,156],[59,13041,227],{"className":13042},[226],[59,13044,13046],{"className":13045},[136],[59,13047,13049],{"className":13048,"style":1270},[140],[59,13050],{},[59,13052,812],{"className":13053},[313],"；",[59,13056,13058,13076],{"className":13057},[62],[59,13059,13061],{"className":13060},[66],[68,13062,13063],{"xmlns":70},[72,13064,13065,13073],{},[75,13066,13067],{},[88,13068,13069,13071],{},[91,13070,7826],{},[91,13072,777],{},[98,13074,13075],{"encoding":100},"r_t",[59,13077,13079],{"className":13078,"ariaHidden":106},[105],[59,13080,13082,13085],{"className":13081},[110],[59,13083],{"className":13084,"style":12713},[114],[59,13086,13088,13091],{"className":13087},[119],[59,13089,7826],{"className":13090,"style":3210},[119,192],[59,13092,13094],{"className":13093},[196],[59,13095,13097,13118],{"className":13096},[131,132],[59,13098,13100,13115],{"className":13099},[136],[59,13101,13103],{"className":13102,"style":12732},[140],[59,13104,13106,13109],{"style":13105},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[59,13107],{"className":13108,"style":540},[148],[59,13110,13112],{"className":13111},[153,154,155,156],[59,13113,777],{"className":13114},[119,192,156],[59,13116,227],{"className":13117},[226],[59,13119,13121],{"className":13120},[136],[59,13122,13124],{"className":13123,"style":1270},[140],[59,13125],{}," 一边表示 ",[59,13128,13130,13143],{"className":13129},[62],[59,13131,13133],{"className":13132},[66],[68,13134,13135],{"xmlns":70},[72,13136,13137,13141],{},[75,13138,13139],{},[91,13140,777],{},[98,13142,777],{"encoding":100},[59,13144,13146],{"className":13145,"ariaHidden":106},[105],[59,13147,13149,13152],{"className":13148},[110],[59,13150],{"className":13151,"style":12781},[114],[59,13153,777],{"className":13154},[119,192]," 时刻 Agent 采取行动获得的奖励（Reward）；",[59,13157,13159,13174],{"className":13158},[62],[59,13160,13162],{"className":13161},[66],[68,13163,13164],{"xmlns":70},[72,13165,13166,13171],{},[75,13167,13168],{},[91,13169,13170],{},"τ",[98,13172,13173],{"encoding":100},"\\tau",[59,13175,13177],{"className":13176,"ariaHidden":106},[105],[59,13178,13180,13183],{"className":13179},[110],[59,13181],{"className":13182,"style":347},[114],[59,13184,13170],{"className":13185,"style":13186},[119,192],"margin-right:0.1132em;"," 表示轨迹（Trajectory），Episode 指一次完整的交互过程：从环境开始，到终止结束。Rollout 指按某个策略实际“跑出来”的一段轨迹数据。它不一定非得是完整的一局，也可以只是中间截取的一段。",[15,13189,13190],{},[59,13191,13193,13248],{"className":13192},[62],[59,13194,13196],{"className":13195},[66],[68,13197,13198],{"xmlns":70},[72,13199,13200,13245],{},[75,13201,13202,13204,13210,13212,13218,13220,13226,13228,13234,13236,13243],{},[461,13203,798],{"stretchy":797},[88,13205,13206,13208],{},[91,13207,12802],{},[81,13209,1754],{},[461,13211,803],{"separator":106},[88,13213,13214,13216],{},[91,13215,22],{},[81,13217,1754],{},[461,13219,803],{"separator":106},[88,13221,13222,13224],{},[91,13223,12802],{},[81,13225,83],{},[461,13227,803],{"separator":106},[88,13229,13230,13232],{},[91,13231,22],{},[81,13233,83],{},[461,13235,2970],{},[88,13237,13238,13240],{},[91,13239,12802],{},[91,13241,13242],{},"T",[461,13244,812],{"stretchy":797},[98,13246,13247],{"encoding":100},"(s_0,a_0,s_1,a_1 \\dots s_T)",[59,13249,13251],{"className":13250,"ariaHidden":106},[105],[59,13252,13254,13257,13260,13300,13303,13306,13346,13349,13352,13392,13395,13398,13438,13441,13444,13447,13488],{"className":13253},[110],[59,13255],{"className":13256,"style":877},[114],[59,13258,798],{"className":13259},[123],[59,13261,13263,13266],{"className":13262},[119],[59,13264,12802],{"className":13265},[119,192],[59,13267,13269],{"className":13268},[196],[59,13270,13272,13292],{"className":13271},[131,132],[59,13273,13275,13289],{"className":13274},[136],[59,13276,13278],{"className":13277,"style":3091},[140],[59,13279,13280,13283],{"style":1251},[59,13281],{"className":13282,"style":540},[148],[59,13284,13286],{"className":13285},[153,154,155,156],[59,13287,1754],{"className":13288},[119,156],[59,13290,227],{"className":13291},[226],[59,13293,13295],{"className":13294},[136],[59,13296,13298],{"className":13297,"style":1270},[140],[59,13299],{},[59,13301,803],{"className":13302},[912],[59,13304],{"className":13305,"style":916},[499],[59,13307,13309,13312],{"className":13308},[119],[59,13310,22],{"className":13311},[119,192],[59,13313,13315],{"className":13314},[196],[59,13316,13318,13338],{"className":13317},[131,132],[59,13319,13321,13335],{"className":13320},[136],[59,13322,13324],{"className":13323,"style":3091},[140],[59,13325,13326,13329],{"style":1251},[59,13327],{"className":13328,"style":540},[148],[59,13330,13332],{"className":13331},[153,154,155,156],[59,13333,1754],{"className":13334},[119,156],[59,13336,227],{"className":13337},[226],[59,13339,13341],{"className":13340},[136],[59,13342,13344],{"className":13343,"style":1270},[140],[59,13345],{},[59,13347,803],{"className":13348},[912],[59,13350],{"className":13351,"style":916},[499],[59,13353,13355,13358],{"className":13354},[119],[59,13356,12802],{"className":13357},[119,192],[59,13359,13361],{"className":13360},[196],[59,13362,13364,13384],{"className":13363},[131,132],[59,13365,13367,13381],{"className":13366},[136],[59,13368,13370],{"className":13369,"style":3091},[140],[59,13371,13372,13375],{"style":1251},[59,13373],{"className":13374,"style":540},[148],[59,13376,13378],{"className":13377},[153,154,155,156],[59,13379,83],{"className":13380},[119,156],[59,13382,227],{"className":13383},[226],[59,13385,13387],{"className":13386},[136],[59,13388,13390],{"className":13389,"style":1270},[140],[59,13391],{},[59,13393,803],{"className":13394},[912],[59,13396],{"className":13397,"style":916},[499],[59,13399,13401,13404],{"className":13400},[119],[59,13402,22],{"className":13403},[119,192],[59,13405,13407],{"className":13406},[196],[59,13408,13410,13430],{"className":13409},[131,132],[59,13411,13413,13427],{"className":13412},[136],[59,13414,13416],{"className":13415,"style":3091},[140],[59,13417,13418,13421],{"style":1251},[59,13419],{"className":13420,"style":540},[148],[59,13422,13424],{"className":13423},[153,154,155,156],[59,13425,83],{"className":13426},[119,156],[59,13428,227],{"className":13429},[226],[59,13431,13433],{"className":13432},[136],[59,13434,13436],{"className":13435,"style":1270},[140],[59,13437],{},[59,13439],{"className":13440,"style":916},[499],[59,13442,2970],{"className":13443},[962],[59,13445],{"className":13446,"style":916},[499],[59,13448,13450,13453],{"className":13449},[119],[59,13451,12802],{"className":13452},[119,192],[59,13454,13456],{"className":13455},[196],[59,13457,13459,13480],{"className":13458},[131,132],[59,13460,13462,13477],{"className":13461},[136],[59,13463,13466],{"className":13464,"style":13465},[140],"height:0.3283em;",[59,13467,13468,13471],{"style":1251},[59,13469],{"className":13470,"style":540},[148],[59,13472,13474],{"className":13473},[153,154,155,156],[59,13475,13242],{"className":13476,"style":3185},[119,192,156],[59,13478,227],{"className":13479},[226],[59,13481,13483],{"className":13482},[136],[59,13484,13486],{"className":13485,"style":1270},[140],[59,13487],{},[59,13489,812],{"className":13490},[313],[15,13492,13493,13712,13713,314],{},[59,13494,13496,13542],{"className":13495},[62],[59,13497,13499],{"className":13498},[66],[68,13500,13501],{"xmlns":70},[72,13502,13503,13539],{},[75,13504,13505,13517,13519,13521,13523,13529,13531,13537],{},[88,13506,13507,13509],{},[91,13508,12802],{},[75,13510,13511,13513,13515],{},[91,13512,777],{},[461,13514,5777],{},[81,13516,83],{},[461,13518,815],{},[91,13520,11657],{},[461,13522,798],{"stretchy":797},[88,13524,13525,13527],{},[91,13526,12802],{},[91,13528,777],{},[461,13530,803],{"separator":106},[88,13532,13533,13535],{},[91,13534,22],{},[91,13536,777],{},[461,13538,812],{"stretchy":797},[98,13540,13541],{"encoding":100}," s_{t+1} = f(s_t,a_t)",[59,13543,13545,13610],{"className":13544,"ariaHidden":106},[105],[59,13546,13548,13552,13601,13604,13607],{"className":13547},[110],[59,13549],{"className":13550,"style":13551},[114],"height:0.6389em;vertical-align:-0.2083em;",[59,13553,13555,13558],{"className":13554},[119],[59,13556,12802],{"className":13557},[119,192],[59,13559,13561],{"className":13560},[196],[59,13562,13564,13593],{"className":13563},[131,132],[59,13565,13567,13590],{"className":13566},[136],[59,13568,13570],{"className":13569,"style":3091},[140],[59,13571,13572,13575],{"style":1251},[59,13573],{"className":13574,"style":540},[148],[59,13576,13578],{"className":13577},[153,154,155,156],[59,13579,13581,13584,13587],{"className":13580},[119,156],[59,13582,777],{"className":13583},[119,192,156],[59,13585,5777],{"className":13586},[553,156],[59,13588,83],{"className":13589},[119,156],[59,13591,227],{"className":13592},[226],[59,13594,13596],{"className":13595},[136],[59,13597,13599],{"className":13598,"style":6938},[140],[59,13600],{},[59,13602],{"className":13603,"style":500},[499],[59,13605,815],{"className":13606},[504],[59,13608],{"className":13609,"style":500},[499],[59,13611,13613,13616,13620,13623,13663,13666,13669,13709],{"className":13612},[110],[59,13614],{"className":13615,"style":877},[114],[59,13617,11657],{"className":13618,"style":13619},[119,192],"margin-right:0.10764em;",[59,13621,798],{"className":13622},[123],[59,13624,13626,13629],{"className":13625},[119],[59,13627,12802],{"className":13628},[119,192],[59,13630,13632],{"className":13631},[196],[59,13633,13635,13655],{"className":13634},[131,132],[59,13636,13638,13652],{"className":13637},[136],[59,13639,13641],{"className":13640,"style":12732},[140],[59,13642,13643,13646],{"style":1251},[59,13644],{"className":13645,"style":540},[148],[59,13647,13649],{"className":13648},[153,154,155,156],[59,13650,777],{"className":13651},[119,192,156],[59,13653,227],{"className":13654},[226],[59,13656,13658],{"className":13657},[136],[59,13659,13661],{"className":13660,"style":1270},[140],[59,13662],{},[59,13664,803],{"className":13665},[912],[59,13667],{"className":13668,"style":916},[499],[59,13670,13672,13675],{"className":13671},[119],[59,13673,22],{"className":13674},[119,192],[59,13676,13678],{"className":13677},[196],[59,13679,13681,13701],{"className":13680},[131,132],[59,13682,13684,13698],{"className":13683},[136],[59,13685,13687],{"className":13686,"style":12732},[140],[59,13688,13689,13692],{"style":1251},[59,13690],{"className":13691,"style":540},[148],[59,13693,13695],{"className":13694},[153,154,155,156],[59,13696,777],{"className":13697},[119,192,156],[59,13699,227],{"className":13700},[226],[59,13702,13704],{"className":13703},[136],[59,13705,13707],{"className":13706,"style":1270},[140],[59,13708],{},[59,13710,812],{"className":13711},[313]," 或者 ",[59,13714,13716,13766],{"className":13715},[62],[59,13717,13719],{"className":13718},[66],[68,13720,13721],{"xmlns":70},[72,13722,13723,13763],{},[75,13724,13725,13737,13739,13741,13743,13745,13747,13753,13755,13761],{},[88,13726,13727,13729],{},[91,13728,12802],{},[75,13730,13731,13733,13735],{},[91,13732,777],{},[461,13734,5777],{},[81,13736,83],{},[461,13738,815],{},[91,13740,5604],{},[461,13742,798],{"stretchy":797},[461,13744,1816],{},[91,13746,12941],{"mathvariant":818},[88,13748,13749,13751],{},[91,13750,12802],{},[91,13752,777],{},[461,13754,803],{"separator":106},[88,13756,13757,13759],{},[91,13758,22],{},[91,13760,777],{},[461,13762,812],{"stretchy":797},[98,13764,13765],{"encoding":100},"s_{t+1} = P(\\cdot|s_t,a_t)",[59,13767,13769,13833],{"className":13768,"ariaHidden":106},[105],[59,13770,13772,13775,13824,13827,13830],{"className":13771},[110],[59,13773],{"className":13774,"style":13551},[114],[59,13776,13778,13781],{"className":13777},[119],[59,13779,12802],{"className":13780},[119,192],[59,13782,13784],{"className":13783},[196],[59,13785,13787,13816],{"className":13786},[131,132],[59,13788,13790,13813],{"className":13789},[136],[59,13791,13793],{"className":13792,"style":3091},[140],[59,13794,13795,13798],{"style":1251},[59,13796],{"className":13797,"style":540},[148],[59,13799,13801],{"className":13800},[153,154,155,156],[59,13802,13804,13807,13810],{"className":13803},[119,156],[59,13805,777],{"className":13806},[119,192,156],[59,13808,5777],{"className":13809},[553,156],[59,13811,83],{"className":13812},[119,156],[59,13814,227],{"className":13815},[226],[59,13817,13819],{"className":13818},[136],[59,13820,13822],{"className":13821,"style":6938},[140],[59,13823],{},[59,13825],{"className":13826,"style":500},[499],[59,13828,815],{"className":13829},[504],[59,13831],{"className":13832,"style":500},[499],[59,13834,13836,13839,13842,13845,13848,13851,13891,13894,13897,13937],{"className":13835},[110],[59,13837],{"className":13838,"style":877},[114],[59,13840,5604],{"className":13841,"style":3185},[119,192],[59,13843,798],{"className":13844},[123],[59,13846,1816],{"className":13847},[119],[59,13849,12941],{"className":13850},[119],[59,13852,13854,13857],{"className":13853},[119],[59,13855,12802],{"className":13856},[119,192],[59,13858,13860],{"className":13859},[196],[59,13861,13863,13883],{"className":13862},[131,132],[59,13864,13866,13880],{"className":13865},[136],[59,13867,13869],{"className":13868,"style":12732},[140],[59,13870,13871,13874],{"style":1251},[59,13872],{"className":13873,"style":540},[148],[59,13875,13877],{"className":13876},[153,154,155,156],[59,13878,777],{"className":13879},[119,192,156],[59,13881,227],{"className":13882},[226],[59,13884,13886],{"className":13885},[136],[59,13887,13889],{"className":13888,"style":1270},[140],[59,13890],{},[59,13892,803],{"className":13893},[912],[59,13895],{"className":13896,"style":916},[499],[59,13898,13900,13903],{"className":13899},[119],[59,13901,22],{"className":13902},[119,192],[59,13904,13906],{"className":13905},[196],[59,13907,13909,13929],{"className":13908},[131,132],[59,13910,13912,13926],{"className":13911},[136],[59,13913,13915],{"className":13914,"style":12732},[140],[59,13916,13917,13920],{"style":1251},[59,13918],{"className":13919,"style":540},[148],[59,13921,13923],{"className":13922},[153,154,155,156],[59,13924,777],{"className":13925},[119,192,156],[59,13927,227],{"className":13928},[226],[59,13930,13932],{"className":13931},[136],[59,13933,13935],{"className":13934,"style":1270},[140],[59,13936],{},[59,13938,812],{"className":13939},[313],[15,13941,13942],{},"Return 一般表示回报，指从当前时间点到游戏结束时奖励的累积和或者加权累积和。",[11,13944,13945],{"id":13945},"训练目标",[15,13947,13948,13949,13977,13978,14007],{},"训练一个 Policy 神经网络 ",[59,13950,13952,13965],{"className":13951},[62],[59,13953,13955],{"className":13954},[66],[68,13956,13957],{"xmlns":70},[72,13958,13959,13963],{},[75,13960,13961],{},[91,13962,6742],{},[98,13964,12903],{"encoding":100},[59,13966,13968],{"className":13967,"ariaHidden":106},[105],[59,13969,13971,13974],{"className":13970},[110],[59,13972],{"className":13973,"style":347},[114],[59,13975,6742],{"className":13976,"style":441},[119,192],"，在所有状态 ",[59,13979,13981,13994],{"className":13980},[62],[59,13982,13984],{"className":13983},[66],[68,13985,13986],{"xmlns":70},[72,13987,13988,13992],{},[75,13989,13990],{},[91,13991,7845],{},[98,13993,7845],{"encoding":100},[59,13995,13997],{"className":13996,"ariaHidden":106},[105],[59,13998,14000,14004],{"className":13999},[110],[59,14001],{"className":14002,"style":14003},[114],"height:0.6833em;",[59,14005,7845],{"className":14006,"style":6050},[119,192]," 下，给出相应的 Action，得到的 Return 的期望最大。",[15,14009,14010],{},[59,14011,14013,14090],{"className":14012},[62],[59,14014,14016],{"className":14015},[66],[68,14017,14018],{"xmlns":70},[72,14019,14020,14087],{},[75,14021,14022,14024,14026,14028,14030,14032,14034,14058,14060,14067,14069,14071,14073,14075,14081,14083,14085],{},[91,14023,5607],{},[461,14025,798],{"stretchy":797},[91,14027,470],{},[461,14029,798],{"stretchy":797},[91,14031,13170],{},[461,14033,812],{"stretchy":797},[88,14035,14036,14038],{},[461,14037,812],{"stretchy":797},[75,14039,14040,14042,14045,14052,14054,14056],{},[91,14041,13170],{},[461,14043,14044],{},"∼",[88,14046,14047,14049],{},[91,14048,5604],{},[91,14050,14051],{},"θ",[461,14053,798],{"stretchy":797},[91,14055,13170],{},[461,14057,812],{"stretchy":797},[461,14059,815],{},[14061,14062,14063,14065],"munder",{},[461,14064,1826],{},[91,14066,13170],{},[91,14068,470],{},[461,14070,798],{"stretchy":797},[91,14072,13170],{},[461,14074,812],{"stretchy":797},[88,14076,14077,14079],{},[91,14078,5604],{},[91,14080,14051],{},[461,14082,798],{"stretchy":797},[91,14084,13170],{},[461,14086,812],{"stretchy":797},[98,14088,14089],{"encoding":100}," E (R(\\tau))_{\\tau \\sim P_{\\theta}(\\tau)} = \\sum\\limits_\\tau R(\\tau)P_\\theta(\\tau)",[59,14091,14093,14228],{"className":14092,"ariaHidden":106},[105],[59,14094,14096,14100,14103,14106,14110,14113,14116,14119,14219,14222,14225],{"className":14095},[110],[59,14097],{"className":14098,"style":14099},[114],"height:1.1052em;vertical-align:-0.3552em;",[59,14101,5607],{"className":14102,"style":6050},[119,192],[59,14104,798],{"className":14105},[123],[59,14107,470],{"className":14108,"style":14109},[119,192],"margin-right:0.00773em;",[59,14111,798],{"className":14112},[123],[59,14114,13170],{"className":14115,"style":13186},[119,192],[59,14117,812],{"className":14118},[313],[59,14120,14122,14125],{"className":14121},[313],[59,14123,812],{"className":14124},[313],[59,14126,14128],{"className":14127},[196],[59,14129,14131,14211],{"className":14130},[131,132],[59,14132,14134,14208],{"className":14133},[136],[59,14135,14137],{"className":14136,"style":206},[140],[59,14138,14140,14143],{"style":14139},"top:-2.5198em;margin-left:0em;margin-right:0.05em;",[59,14141],{"className":14142,"style":540},[148],[59,14144,14146],{"className":14145},[153,154,155,156],[59,14147,14149,14152,14155,14199,14202,14205],{"className":14148},[119,156],[59,14150,13170],{"className":14151,"style":13186},[119,192,156],[59,14153,14044],{"className":14154},[504,156],[59,14156,14158,14161],{"className":14157},[119,156],[59,14159,5604],{"className":14160,"style":3185},[119,192,156],[59,14162,14164],{"className":14163},[196],[59,14165,14167,14191],{"className":14166},[131,132],[59,14168,14170,14188],{"className":14169},[136],[59,14171,14173],{"className":14172,"style":206},[140],[59,14174,14176,14179],{"style":14175},"top:-2.3488em;margin-left:-0.1389em;margin-right:0.0714em;",[59,14177],{"className":14178,"style":213},[148],[59,14180,14182],{"className":14181},[153,217,218,156],[59,14183,14185],{"className":14184},[119,156],[59,14186,14051],{"className":14187,"style":3210},[119,192,156],[59,14189,227],{"className":14190},[226],[59,14192,14194],{"className":14193},[136],[59,14195,14197],{"className":14196,"style":234},[140],[59,14198],{},[59,14200,798],{"className":14201},[123,156],[59,14203,13170],{"className":14204,"style":13186},[119,192,156],[59,14206,812],{"className":14207},[313,156],[59,14209,227],{"className":14210},[226],[59,14212,14214],{"className":14213},[136],[59,14215,14217],{"className":14216,"style":6102},[140],[59,14218],{},[59,14220],{"className":14221,"style":500},[499],[59,14223,815],{"className":14224},[504],[59,14226],{"className":14227,"style":500},[499],[59,14229,14231,14235,14283,14286,14289,14292,14295,14298,14338,14341,14344],{"className":14230},[110],[59,14232],{"className":14233,"style":14234},[114],"height:1.7em;vertical-align:-0.95em;",[59,14236,14239],{"className":14237},[951,14238],"op-limits",[59,14240,14242,14274],{"className":14241},[131,132],[59,14243,14245,14271],{"className":14244},[136],[59,14246,14249,14261],{"className":14247,"style":14248},[140],"height:0.75em;",[59,14250,14252,14255],{"style":14251},"top:-2.15em;margin-left:0em;",[59,14253],{"className":14254,"style":149},[148],[59,14256,14258],{"className":14257},[153,154,155,156],[59,14259,13170],{"className":14260,"style":13186},[119,192,156],[59,14262,14263,14266],{"style":178},[59,14264],{"className":14265,"style":149},[148],[59,14267,14268],{},[59,14269,1826],{"className":14270},[951,1910,1911],[59,14272,227],{"className":14273},[226],[59,14275,14277],{"className":14276},[136],[59,14278,14281],{"className":14279,"style":14280},[140],"height:0.95em;",[59,14282],{},[59,14284],{"className":14285,"style":916},[499],[59,14287,470],{"className":14288,"style":14109},[119,192],[59,14290,798],{"className":14291},[123],[59,14293,13170],{"className":14294,"style":13186},[119,192],[59,14296,812],{"className":14297},[313],[59,14299,14301,14304],{"className":14300},[119],[59,14302,5604],{"className":14303,"style":3185},[119,192],[59,14305,14307],{"className":14306},[196],[59,14308,14310,14330],{"className":14309},[131,132],[59,14311,14313,14327],{"className":14312},[136],[59,14314,14316],{"className":14315,"style":1248},[140],[59,14317,14318,14321],{"style":7460},[59,14319],{"className":14320,"style":540},[148],[59,14322,14324],{"className":14323},[153,154,155,156],[59,14325,14051],{"className":14326,"style":3210},[119,192,156],[59,14328,227],{"className":14329},[226],[59,14331,14333],{"className":14332},[136],[59,14334,14336],{"className":14335,"style":1270},[140],[59,14337],{},[59,14339,798],{"className":14340},[123],[59,14342,13170],{"className":14343,"style":13186},[119,192],[59,14345,812],{"className":14346},[313],[59,14348,14351],{"className":14349},[14350],"katex-display",[59,14352,14354,15134],{"className":14353},[62],[59,14355,14357],{"className":14356},[66],[68,14358,14359],{"xmlns":70,"display":9481},[72,14360,14361,15131],{},[14362,14363,14367,14453,14499,14579,14657,14751,14824,14927,15029],"mtable",{"rowspacing":14364,"columnalign":14365,"columnspacing":14366},"0.25em","right left","0em",[14368,14369,14370,14415],"mtr",{},[14371,14372,14373],"mtd",{},[14374,14375,14376],"mstyle",{"scriptlevel":1754,"displaystyle":106},[75,14377,14378,14381,14383,14385,14387,14389,14391,14393],{},[91,14379,14380],{"mathvariant":818},"∇",[91,14382,5607],{},[461,14384,798],{"stretchy":797},[91,14386,470],{},[461,14388,798],{"stretchy":797},[91,14390,13170],{},[461,14392,812],{"stretchy":797},[88,14394,14395,14397],{},[461,14396,812],{"stretchy":797},[75,14398,14399,14401,14403,14409,14411,14413],{},[91,14400,13170],{},[461,14402,14044],{},[88,14404,14405,14407],{},[91,14406,5604],{},[91,14408,14051],{},[461,14410,798],{"stretchy":797},[91,14412,13170],{},[461,14414,812],{"stretchy":797},[14371,14416,14417],{},[14374,14418,14419],{"scriptlevel":1754,"displaystyle":106},[75,14420,14421,14423,14425,14427,14433,14435,14437,14439,14441,14447,14449,14451],{},[75,14422],{},[461,14424,815],{},[91,14426,14380],{"mathvariant":818},[14061,14428,14429,14431],{},[461,14430,1826],{},[91,14432,13170],{},[91,14434,470],{},[461,14436,798],{"stretchy":797},[91,14438,13170],{},[461,14440,812],{"stretchy":797},[88,14442,14443,14445],{},[91,14444,5604],{},[91,14446,14051],{},[461,14448,798],{"stretchy":797},[91,14450,13170],{},[461,14452,812],{"stretchy":797},[14368,14454,14455,14461],{},[14371,14456,14457],{},[14374,14458,14459],{"scriptlevel":1754,"displaystyle":106},[75,14460],{},[14371,14462,14463],{},[14374,14464,14465],{"scriptlevel":1754,"displaystyle":106},[75,14466,14467,14469,14471,14477,14479,14481,14483,14485,14487,14493,14495,14497],{},[75,14468],{},[461,14470,815],{},[14061,14472,14473,14475],{},[461,14474,1826],{},[91,14476,13170],{},[91,14478,470],{},[461,14480,798],{"stretchy":797},[91,14482,13170],{},[461,14484,812],{"stretchy":797},[91,14486,14380],{"mathvariant":818},[88,14488,14489,14491],{},[91,14490,5604],{},[91,14492,14051],{},[461,14494,798],{"stretchy":797},[91,14496,13170],{},[461,14498,812],{"stretchy":797},[14368,14500,14501,14507],{},[14371,14502,14503],{},[14374,14504,14505],{"scriptlevel":1754,"displaystyle":106},[75,14506],{},[14371,14508,14509],{},[14374,14510,14511],{"scriptlevel":1754,"displaystyle":106},[75,14512,14513,14515,14517,14523,14525,14527,14529,14531,14533,14539,14541,14543,14545,14547],{},[75,14514],{},[461,14516,815],{},[14061,14518,14519,14521],{},[461,14520,1826],{},[91,14522,13170],{},[91,14524,470],{},[461,14526,798],{"stretchy":797},[91,14528,13170],{},[461,14530,812],{"stretchy":797},[91,14532,14380],{"mathvariant":818},[88,14534,14535,14537],{},[91,14536,5604],{},[91,14538,14051],{},[461,14540,798],{"stretchy":797},[91,14542,13170],{},[461,14544,812],{"stretchy":797},[461,14546,1816],{},[14374,14548,14549],{"scriptlevel":1754,"displaystyle":106},[78,14550,14551,14565],{},[75,14552,14553,14559,14561,14563],{},[88,14554,14555,14557],{},[91,14556,5604],{},[91,14558,14051],{},[461,14560,798],{"stretchy":797},[91,14562,13170],{},[461,14564,812],{"stretchy":797},[75,14566,14567,14573,14575,14577],{},[88,14568,14569,14571],{},[91,14570,5604],{},[91,14572,14051],{},[461,14574,798],{"stretchy":797},[91,14576,13170],{},[461,14578,812],{"stretchy":797},[14368,14580,14581,14587],{},[14371,14582,14583],{},[14374,14584,14585],{"scriptlevel":1754,"displaystyle":106},[75,14586],{},[14371,14588,14589],{},[14374,14590,14591],{"scriptlevel":1754,"displaystyle":106},[75,14592,14593,14595,14597,14603,14609,14611,14613,14615,14617,14619,14621,14623],{},[75,14594],{},[461,14596,815],{},[14061,14598,14599,14601],{},[461,14600,1826],{},[91,14602,13170],{},[88,14604,14605,14607],{},[91,14606,5604],{},[91,14608,14051],{},[461,14610,798],{"stretchy":797},[91,14612,13170],{},[461,14614,812],{"stretchy":797},[91,14616,470],{},[461,14618,798],{"stretchy":797},[91,14620,13170],{},[461,14622,812],{"stretchy":797},[14374,14624,14625],{"scriptlevel":1754,"displaystyle":106},[78,14626,14627,14643],{},[75,14628,14629,14631,14637,14639,14641],{},[91,14630,14380],{"mathvariant":818},[88,14632,14633,14635],{},[91,14634,5604],{},[91,14636,14051],{},[461,14638,798],{"stretchy":797},[91,14640,13170],{},[461,14642,812],{"stretchy":797},[75,14644,14645,14651,14653,14655],{},[88,14646,14647,14649],{},[91,14648,5604],{},[91,14650,14051],{},[461,14652,798],{"stretchy":797},[91,14654,13170],{},[461,14656,812],{"stretchy":797},[14368,14658,14659,14665],{},[14371,14660,14661],{},[14374,14662,14663],{"scriptlevel":1754,"displaystyle":106},[75,14664],{},[14371,14666,14667],{},[14374,14668,14669],{"scriptlevel":1754,"displaystyle":106},[75,14670,14671,14673,14676,14682,14697,14699,14701,14707,14709],{},[75,14672],{},[461,14674,14675],{},"≈",[78,14677,14678,14680],{},[81,14679,83],{},[91,14681,7322],{},[14683,14684,14685,14687,14695],"munderover",{},[461,14686,1826],{},[75,14688,14689,14691,14693],{},[91,14690,335],{},[461,14692,815],{},[81,14694,83],{},[91,14696,7322],{},[91,14698,470],{},[461,14700,798],{"stretchy":797},[465,14702,14703,14705],{},[91,14704,13170],{},[91,14706,335],{},[461,14708,812],{"stretchy":797},[14374,14710,14711],{"scriptlevel":1754,"displaystyle":106},[78,14712,14713,14733],{},[75,14714,14715,14717,14723,14725,14731],{},[91,14716,14380],{"mathvariant":818},[88,14718,14719,14721],{},[91,14720,15],{},[91,14722,14051],{},[461,14724,798],{"stretchy":797},[465,14726,14727,14729],{},[91,14728,13170],{},[91,14730,335],{},[461,14732,812],{"stretchy":797},[75,14734,14735,14741,14743,14749],{},[88,14736,14737,14739],{},[91,14738,5604],{},[91,14740,14051],{},[461,14742,798],{"stretchy":797},[465,14744,14745,14747],{},[91,14746,13170],{},[91,14748,335],{},[461,14750,812],{"stretchy":797},[14368,14752,14753,14759],{},[14371,14754,14755],{},[14374,14756,14757],{"scriptlevel":1754,"displaystyle":106},[75,14758],{},[14371,14760,14761],{},[14374,14762,14763],{"scriptlevel":1754,"displaystyle":106},[75,14764,14765,14767,14769,14775,14789,14791,14793,14799,14801,14803,14806,14808,14814,14816,14822],{},[75,14766],{},[461,14768,815],{},[78,14770,14771,14773],{},[81,14772,83],{},[91,14774,7322],{},[14683,14776,14777,14779,14787],{},[461,14778,1826],{},[75,14780,14781,14783,14785],{},[91,14782,335],{},[461,14784,815],{},[81,14786,83],{},[91,14788,7322],{},[91,14790,470],{},[461,14792,798],{"stretchy":797},[465,14794,14795,14797],{},[91,14796,13170],{},[91,14798,335],{},[461,14800,812],{"stretchy":797},[91,14802,14380],{"mathvariant":818},[91,14804,14805],{},"log",[461,14807,822],{},[88,14809,14810,14812],{},[91,14811,5604],{},[91,14813,14051],{},[461,14815,798],{"stretchy":797},[465,14817,14818,14820],{},[91,14819,13170],{},[91,14821,335],{},[461,14823,812],{"stretchy":797},[14368,14825,14826,14832],{},[14371,14827,14828],{},[14374,14829,14830],{"scriptlevel":1754,"displaystyle":106},[75,14831],{},[14371,14833,14834],{},[14374,14835,14836],{"scriptlevel":1754,"displaystyle":106},[75,14837,14838,14840,14842,14848,14862,14864,14866,14872,14874,14876,14878,14880,14899,14905,14907,14915,14917,14925],{},[75,14839],{},[461,14841,815],{},[78,14843,14844,14846],{},[81,14845,83],{},[91,14847,7322],{},[14683,14849,14850,14852,14860],{},[461,14851,1826],{},[75,14853,14854,14856,14858],{},[91,14855,335],{},[461,14857,815],{},[81,14859,83],{},[91,14861,7322],{},[91,14863,470],{},[461,14865,798],{"stretchy":797},[465,14867,14868,14870],{},[91,14869,13170],{},[91,14871,335],{},[461,14873,812],{"stretchy":797},[91,14875,14380],{"mathvariant":818},[91,14877,14805],{},[461,14879,822],{},[14683,14881,14882,14885,14893],{},[461,14883,14884],{},"∏",[75,14886,14887,14889,14891],{},[91,14888,777],{},[461,14890,815],{},[81,14892,83],{},[88,14894,14895,14897],{},[91,14896,13242],{},[91,14898,335],{},[88,14900,14901,14903],{},[91,14902,5604],{},[91,14904,14051],{},[461,14906,798],{"stretchy":797},[1822,14908,14909,14911,14913],{},[91,14910,22],{},[91,14912,335],{},[91,14914,777],{},[91,14916,12941],{"mathvariant":818},[1822,14918,14919,14921,14923],{},[91,14920,12802],{},[91,14922,335],{},[91,14924,777],{},[461,14926,812],{"stretchy":797},[14368,14928,14929,14935],{},[14371,14930,14931],{},[14374,14932,14933],{"scriptlevel":1754,"displaystyle":106},[75,14934],{},[14371,14936,14937],{},[14374,14938,14939],{"scriptlevel":1754,"displaystyle":106},[75,14940,14941,14943,14945,14951,14965,14967,14969,14975,14977,14995,14997,14999,15001,15007,15009,15017,15019,15027],{},[75,14942],{},[461,14944,815],{},[78,14946,14947,14949],{},[81,14948,83],{},[91,14950,7322],{},[14683,14952,14953,14955,14963],{},[461,14954,1826],{},[75,14956,14957,14959,14961],{},[91,14958,335],{},[461,14960,815],{},[81,14962,83],{},[91,14964,7322],{},[91,14966,470],{},[461,14968,798],{"stretchy":797},[465,14970,14971,14973],{},[91,14972,13170],{},[91,14974,335],{},[461,14976,812],{"stretchy":797},[14683,14978,14979,14981,14989],{},[461,14980,1826],{},[75,14982,14983,14985,14987],{},[91,14984,777],{},[461,14986,815],{},[81,14988,83],{},[88,14990,14991,14993],{},[91,14992,13242],{},[91,14994,335],{},[91,14996,14380],{"mathvariant":818},[91,14998,14805],{},[461,15000,822],{},[88,15002,15003,15005],{},[91,15004,5604],{},[91,15006,14051],{},[461,15008,798],{"stretchy":797},[1822,15010,15011,15013,15015],{},[91,15012,22],{},[91,15014,335],{},[91,15016,777],{},[91,15018,12941],{"mathvariant":818},[1822,15020,15021,15023,15025],{},[91,15022,12802],{},[91,15024,335],{},[91,15026,777],{},[461,15028,812],{"stretchy":797},[14368,15030,15031,15037],{},[14371,15032,15033],{},[14374,15034,15035],{"scriptlevel":1754,"displaystyle":106},[75,15036],{},[14371,15038,15039],{},[14374,15040,15041],{"scriptlevel":1754,"displaystyle":106},[75,15042,15043,15045,15047,15053,15067,15085,15087,15089,15095,15097,15099,15101,15103,15109,15111,15119,15121,15129],{},[75,15044],{},[461,15046,815],{},[78,15048,15049,15051],{},[81,15050,83],{},[91,15052,7322],{},[14683,15054,15055,15057,15065],{},[461,15056,1826],{},[75,15058,15059,15061,15063],{},[91,15060,335],{},[461,15062,815],{},[81,15064,83],{},[91,15066,7322],{},[14683,15068,15069,15071,15079],{},[461,15070,1826],{},[75,15072,15073,15075,15077],{},[91,15074,777],{},[461,15076,815],{},[81,15078,83],{},[88,15080,15081,15083],{},[91,15082,13242],{},[91,15084,335],{},[91,15086,470],{},[461,15088,798],{"stretchy":797},[465,15090,15091,15093],{},[91,15092,13170],{},[91,15094,335],{},[461,15096,812],{"stretchy":797},[91,15098,14380],{"mathvariant":818},[91,15100,14805],{},[461,15102,822],{},[88,15104,15105,15107],{},[91,15106,5604],{},[91,15108,14051],{},[461,15110,798],{"stretchy":797},[1822,15112,15113,15115,15117],{},[91,15114,22],{},[91,15116,335],{},[91,15118,777],{},[91,15120,12941],{"mathvariant":818},[1822,15122,15123,15125,15127],{},[91,15124,12802],{},[91,15126,335],{},[91,15128,777],{},[461,15130,812],{"stretchy":797},[98,15132,15133],{"encoding":100},"\\begin{aligned}\n\\nabla E(R(\\tau))_{\\tau \\sim P_\\theta(\\tau)} &= \\nabla \\sum_\\tau R(\\tau)P_\\theta(\\tau)\\\\\n&= \\sum_\\tau R(\\tau)\\nabla P_\\theta(\\tau)\\\\\n&= \\sum_\\tau R(\\tau)\\nabla P_\\theta(\\tau) \\cdot \\dfrac{P_\\theta(\\tau)}{P_\\theta(\\tau)}\\\\\n&= \\sum_\\tau P_\\theta(\\tau)R(\\tau) \\dfrac{\\nabla P_\\theta(\\tau)}{P_\\theta(\\tau)}\\\\\n&\\approx \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\dfrac{\\nabla p_\\theta(\\tau^n)}{P_\\theta(\\tau^n)}\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\nabla \\log P_\\theta(\\tau^n)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\nabla \\log \\prod\\limits_{t=1}^{T_n}P_\\theta(a_n^t|s_n^t)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\sum\\limits_{t=1}^{T_n}\\nabla \\log P_\\theta(a_n^t|s_n^t)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} R(\\tau^n)\\nabla \\log P_\\theta(a_n^t|s_n^t)\\\\\n\\end{aligned}",[59,15135,15137],{"className":15136,"ariaHidden":106},[105],[59,15138,15140,15144],{"className":15139},[110],[59,15141],{"className":15142,"style":15143},[114],"height:28.1646em;vertical-align:-13.8323em;",[59,15145,15147],{"className":15146},[119],[59,15148,15150,15374],{"className":15149},[14362],[59,15151,15154],{"className":15152},[15153],"col-align-r",[59,15155,15157,15365],{"className":15156},[131,132],[59,15158,15160,15362],{"className":15159},[136],[59,15161,15164,15290,15299,15308,15317,15326,15335,15344,15353],{"className":15162,"style":15163},[140],"height:14.3323em;",[59,15165,15167,15171],{"style":15166},"top:-17.1217em;",[59,15168],{"className":15169,"style":15170},[148],"height:3.8394em;",[59,15172,15174,15177,15180,15183,15186,15189,15192,15195],{"className":15173},[119],[59,15175,14380],{"className":15176},[119],[59,15178,5607],{"className":15179,"style":6050},[119,192],[59,15181,798],{"className":15182},[123],[59,15184,470],{"className":15185,"style":14109},[119,192],[59,15187,798],{"className":15188},[123],[59,15190,13170],{"className":15191,"style":13186},[119,192],[59,15193,812],{"className":15194},[313],[59,15196,15198,15201],{"className":15197},[313],[59,15199,812],{"className":15200},[313],[59,15202,15204],{"className":15203},[196],[59,15205,15207,15282],{"className":15206},[131,132],[59,15208,15210,15279],{"className":15209},[136],[59,15211,15213],{"className":15212,"style":206},[140],[59,15214,15215,15218],{"style":14139},[59,15216],{"className":15217,"style":540},[148],[59,15219,15221],{"className":15220},[153,154,155,156],[59,15222,15224,15227,15230,15270,15273,15276],{"className":15223},[119,156],[59,15225,13170],{"className":15226,"style":13186},[119,192,156],[59,15228,14044],{"className":15229},[504,156],[59,15231,15233,15236],{"className":15232},[119,156],[59,15234,5604],{"className":15235,"style":3185},[119,192,156],[59,15237,15239],{"className":15238},[196],[59,15240,15242,15262],{"className":15241},[131,132],[59,15243,15245,15259],{"className":15244},[136],[59,15246,15248],{"className":15247,"style":206},[140],[59,15249,15250,15253],{"style":14175},[59,15251],{"className":15252,"style":213},[148],[59,15254,15256],{"className":15255},[153,217,218,156],[59,15257,14051],{"className":15258,"style":3210},[119,192,156],[59,15260,227],{"className":15261},[226],[59,15263,15265],{"className":15264},[136],[59,15266,15268],{"className":15267,"style":234},[140],[59,15269],{},[59,15271,798],{"className":15272},[123,156],[59,15274,13170],{"className":15275,"style":13186},[119,192,156],[59,15277,812],{"className":15278},[313,156],[59,15280,227],{"className":15281},[226],[59,15283,15285],{"className":15284},[136],[59,15286,15288],{"className":15287,"style":6102},[140],[59,15289],{},[59,15291,15293,15296],{"style":15292},"top:-14.5217em;",[59,15294],{"className":15295,"style":15170},[148],[59,15297],{"className":15298},[119],[59,15300,15302,15305],{"style":15301},"top:-11.5447em;",[59,15303],{"className":15304,"style":15170},[148],[59,15306],{"className":15307},[119],[59,15309,15311,15314],{"style":15310},"top:-8.5677em;",[59,15312],{"className":15313,"style":15170},[148],[59,15315],{"className":15316},[119],[59,15318,15320,15323],{"style":15319},"top:-5.1894em;",[59,15321],{"className":15322,"style":15170},[148],[59,15324],{"className":15325},[119],[59,15327,15329,15332],{"style":15328},"top:-1.7939em;",[59,15330],{"className":15331,"style":15170},[148],[59,15333],{"className":15334},[119],[59,15336,15338,15341],{"style":15337},"top:1.6126em;",[59,15339],{"className":15340,"style":15170},[148],[59,15342],{"className":15343},[119],[59,15345,15347,15350],{"style":15346},"top:5.0192em;",[59,15348],{"className":15349,"style":15170},[148],[59,15351],{"className":15352},[119],[59,15354,15356,15359],{"style":15355},"top:8.4257em;",[59,15357],{"className":15358,"style":15170},[148],[59,15360],{"className":15361},[119],[59,15363,227],{"className":15364},[226],[59,15366,15368],{"className":15367},[136],[59,15369,15372],{"className":15370,"style":15371},[140],"height:13.8323em;",[59,15373],{},[59,15375,15378],{"className":15376},[15377],"col-align-l",[59,15379,15381,18331],{"className":15380},[131,132],[59,15382,15384,18328],{"className":15383},[136],[59,15385,15387,15527,15658,15956,16241,16651,16933,17401,17866],{"className":15386,"style":15163},[140],[59,15388,15389,15392],{"style":15166},[59,15390],{"className":15391,"style":15170},[148],[59,15393,15395,15398,15401,15404,15407,15410,15413,15463,15466,15469,15472,15475,15478,15518,15521,15524],{"className":15394},[119],[59,15396],{"className":15397},[119],[59,15399],{"className":15400,"style":500},[499],[59,15402,815],{"className":15403},[504],[59,15405],{"className":15406,"style":500},[499],[59,15408,14380],{"className":15409},[119],[59,15411],{"className":15412,"style":916},[499],[59,15414,15416],{"className":15415},[951,14238],[59,15417,15419,15454],{"className":15418},[131,132],[59,15420,15422,15451],{"className":15421},[136],[59,15423,15426,15439],{"className":15424,"style":15425},[140],"height:1.05em;",[59,15427,15429,15433],{"style":15428},"top:-1.9em;margin-left:0em;",[59,15430],{"className":15431,"style":15432},[148],"height:3.05em;",[59,15434,15436],{"className":15435},[153,154,155,156],[59,15437,13170],{"className":15438,"style":13186},[119,192,156],[59,15440,15442,15445],{"style":15441},"top:-3.05em;",[59,15443],{"className":15444,"style":15432},[148],[59,15446,15447],{},[59,15448,1826],{"className":15449},[951,1910,15450],"large-op",[59,15452,227],{"className":15453},[226],[59,15455,15457],{"className":15456},[136],[59,15458,15461],{"className":15459,"style":15460},[140],"height:1.25em;",[59,15462],{},[59,15464],{"className":15465,"style":916},[499],[59,15467,470],{"className":15468,"style":14109},[119,192],[59,15470,798],{"className":15471},[123],[59,15473,13170],{"className":15474,"style":13186},[119,192],[59,15476,812],{"className":15477},[313],[59,15479,15481,15484],{"className":15480},[119],[59,15482,5604],{"className":15483,"style":3185},[119,192],[59,15485,15487],{"className":15486},[196],[59,15488,15490,15510],{"className":15489},[131,132],[59,15491,15493,15507],{"className":15492},[136],[59,15494,15496],{"className":15495,"style":1248},[140],[59,15497,15498,15501],{"style":7460},[59,15499],{"className":15500,"style":540},[148],[59,15502,15504],{"className":15503},[153,154,155,156],[59,15505,14051],{"className":15506,"style":3210},[119,192,156],[59,15508,227],{"className":15509},[226],[59,15511,15513],{"className":15512},[136],[59,15514,15516],{"className":15515,"style":1270},[140],[59,15517],{},[59,15519,798],{"className":15520},[123],[59,15522,13170],{"className":15523,"style":13186},[119,192],[59,15525,812],{"className":15526},[313],[59,15528,15529,15532],{"style":15292},[59,15530],{"className":15531,"style":15170},[148],[59,15533,15535,15538,15541,15544,15547,15591,15594,15597,15600,15603,15606,15609,15649,15652,15655],{"className":15534},[119],[59,15536],{"className":15537},[119],[59,15539],{"className":15540,"style":500},[499],[59,15542,815],{"className":15543},[504],[59,15545],{"className":15546,"style":500},[499],[59,15548,15550],{"className":15549},[951,14238],[59,15551,15553,15583],{"className":15552},[131,132],[59,15554,15556,15580],{"className":15555},[136],[59,15557,15559,15570],{"className":15558,"style":15425},[140],[59,15560,15561,15564],{"style":15428},[59,15562],{"className":15563,"style":15432},[148],[59,15565,15567],{"className":15566},[153,154,155,156],[59,15568,13170],{"className":15569,"style":13186},[119,192,156],[59,15571,15572,15575],{"style":15441},[59,15573],{"className":15574,"style":15432},[148],[59,15576,15577],{},[59,15578,1826],{"className":15579},[951,1910,15450],[59,15581,227],{"className":15582},[226],[59,15584,15586],{"className":15585},[136],[59,15587,15589],{"className":15588,"style":15460},[140],[59,15590],{},[59,15592],{"className":15593,"style":916},[499],[59,15595,470],{"className":15596,"style":14109},[119,192],[59,15598,798],{"className":15599},[123],[59,15601,13170],{"className":15602,"style":13186},[119,192],[59,15604,812],{"className":15605},[313],[59,15607,14380],{"className":15608},[119],[59,15610,15612,15615],{"className":15611},[119],[59,15613,5604],{"className":15614,"style":3185},[119,192],[59,15616,15618],{"className":15617},[196],[59,15619,15621,15641],{"className":15620},[131,132],[59,15622,15624,15638],{"className":15623},[136],[59,15625,15627],{"className":15626,"style":1248},[140],[59,15628,15629,15632],{"style":7460},[59,15630],{"className":15631,"style":540},[148],[59,15633,15635],{"className":15634},[153,154,155,156],[59,15636,14051],{"className":15637,"style":3210},[119,192,156],[59,15639,227],{"className":15640},[226],[59,15642,15644],{"className":15643},[136],[59,15645,15647],{"className":15646,"style":1270},[140],[59,15648],{},[59,15650,798],{"className":15651},[123],[59,15653,13170],{"className":15654,"style":13186},[119,192],[59,15656,812],{"className":15657},[313],[59,15659,15660,15663],{"style":15301},[59,15661],{"className":15662,"style":15170},[148],[59,15664,15666,15669,15672,15675,15678,15722,15725,15728,15731,15734,15737,15740,15780,15783,15786,15789,15792,15795,15798],{"className":15665},[119],[59,15667],{"className":15668},[119],[59,15670],{"className":15671,"style":500},[499],[59,15673,815],{"className":15674},[504],[59,15676],{"className":15677,"style":500},[499],[59,15679,15681],{"className":15680},[951,14238],[59,15682,15684,15714],{"className":15683},[131,132],[59,15685,15687,15711],{"className":15686},[136],[59,15688,15690,15701],{"className":15689,"style":15425},[140],[59,15691,15692,15695],{"style":15428},[59,15693],{"className":15694,"style":15432},[148],[59,15696,15698],{"className":15697},[153,154,155,156],[59,15699,13170],{"className":15700,"style":13186},[119,192,156],[59,15702,15703,15706],{"style":15441},[59,15704],{"className":15705,"style":15432},[148],[59,15707,15708],{},[59,15709,1826],{"className":15710},[951,1910,15450],[59,15712,227],{"className":15713},[226],[59,15715,15717],{"className":15716},[136],[59,15718,15720],{"className":15719,"style":15460},[140],[59,15721],{},[59,15723],{"className":15724,"style":916},[499],[59,15726,470],{"className":15727,"style":14109},[119,192],[59,15729,798],{"className":15730},[123],[59,15732,13170],{"className":15733,"style":13186},[119,192],[59,15735,812],{"className":15736},[313],[59,15738,14380],{"className":15739},[119],[59,15741,15743,15746],{"className":15742},[119],[59,15744,5604],{"className":15745,"style":3185},[119,192],[59,15747,15749],{"className":15748},[196],[59,15750,15752,15772],{"className":15751},[131,132],[59,15753,15755,15769],{"className":15754},[136],[59,15756,15758],{"className":15757,"style":1248},[140],[59,15759,15760,15763],{"style":7460},[59,15761],{"className":15762,"style":540},[148],[59,15764,15766],{"className":15765},[153,154,155,156],[59,15767,14051],{"className":15768,"style":3210},[119,192,156],[59,15770,227],{"className":15771},[226],[59,15773,15775],{"className":15774},[136],[59,15776,15778],{"className":15777,"style":1270},[140],[59,15779],{},[59,15781,798],{"className":15782},[123],[59,15784,13170],{"className":15785,"style":13186},[119,192],[59,15787,812],{"className":15788},[313],[59,15790],{"className":15791,"style":1872},[499],[59,15793,1816],{"className":15794},[553],[59,15796],{"className":15797,"style":1872},[499],[59,15799,15801,15804,15953],{"className":15800},[119],[59,15802],{"className":15803},[123,124],[59,15805,15807],{"className":15806},[78],[59,15808,15810,15944],{"className":15809},[131,132],[59,15811,15813,15941],{"className":15812},[136],[59,15814,15817,15875,15883],{"className":15815,"style":15816},[140],"height:1.427em;",[59,15818,15820,15823],{"style":15819},"top:-2.314em;",[59,15821],{"className":15822,"style":149},[148],[59,15824,15826,15866,15869,15872],{"className":15825},[119],[59,15827,15829,15832],{"className":15828},[119],[59,15830,5604],{"className":15831,"style":3185},[119,192],[59,15833,15835],{"className":15834},[196],[59,15836,15838,15858],{"className":15837},[131,132],[59,15839,15841,15855],{"className":15840},[136],[59,15842,15844],{"className":15843,"style":1248},[140],[59,15845,15846,15849],{"style":7460},[59,15847],{"className":15848,"style":540},[148],[59,15850,15852],{"className":15851},[153,154,155,156],[59,15853,14051],{"className":15854,"style":3210},[119,192,156],[59,15856,227],{"className":15857},[226],[59,15859,15861],{"className":15860},[136],[59,15862,15864],{"className":15863,"style":1270},[140],[59,15865],{},[59,15867,798],{"className":15868},[123],[59,15870,13170],{"className":15871,"style":13186},[119,192],[59,15873,812],{"className":15874},[313],[59,15876,15877,15880],{"style":274},[59,15878],{"className":15879,"style":149},[148],[59,15881],{"className":15882,"style":282},[281],[59,15884,15886,15889],{"style":15885},"top:-3.677em;",[59,15887],{"className":15888,"style":149},[148],[59,15890,15892,15932,15935,15938],{"className":15891},[119],[59,15893,15895,15898],{"className":15894},[119],[59,15896,5604],{"className":15897,"style":3185},[119,192],[59,15899,15901],{"className":15900},[196],[59,15902,15904,15924],{"className":15903},[131,132],[59,15905,15907,15921],{"className":15906},[136],[59,15908,15910],{"className":15909,"style":1248},[140],[59,15911,15912,15915],{"style":7460},[59,15913],{"className":15914,"style":540},[148],[59,15916,15918],{"className":15917},[153,154,155,156],[59,15919,14051],{"className":15920,"style":3210},[119,192,156],[59,15922,227],{"className":15923},[226],[59,15925,15927],{"className":15926},[136],[59,15928,15930],{"className":15929,"style":1270},[140],[59,15931],{},[59,15933,798],{"className":15934},[123],[59,15936,13170],{"className":15937,"style":13186},[119,192],[59,15939,812],{"className":15940},[313],[59,15942,227],{"className":15943},[226],[59,15945,15947],{"className":15946},[136],[59,15948,15951],{"className":15949,"style":15950},[140],"height:0.936em;",[59,15952],{},[59,15954],{"className":15955},[313,124],[59,15957,15958,15961],{"style":15310},[59,15959],{"className":15960,"style":15170},[148],[59,15962,15964,15967,15970,15973,15976,16020,16023,16063,16066,16069,16072,16075,16078,16081,16084],{"className":15963},[119],[59,15965],{"className":15966},[119],[59,15968],{"className":15969,"style":500},[499],[59,15971,815],{"className":15972},[504],[59,15974],{"className":15975,"style":500},[499],[59,15977,15979],{"className":15978},[951,14238],[59,15980,15982,16012],{"className":15981},[131,132],[59,15983,15985,16009],{"className":15984},[136],[59,15986,15988,15999],{"className":15987,"style":15425},[140],[59,15989,15990,15993],{"style":15428},[59,15991],{"className":15992,"style":15432},[148],[59,15994,15996],{"className":15995},[153,154,155,156],[59,15997,13170],{"className":15998,"style":13186},[119,192,156],[59,16000,16001,16004],{"style":15441},[59,16002],{"className":16003,"style":15432},[148],[59,16005,16006],{},[59,16007,1826],{"className":16008},[951,1910,15450],[59,16010,227],{"className":16011},[226],[59,16013,16015],{"className":16014},[136],[59,16016,16018],{"className":16017,"style":15460},[140],[59,16019],{},[59,16021],{"className":16022,"style":916},[499],[59,16024,16026,16029],{"className":16025},[119],[59,16027,5604],{"className":16028,"style":3185},[119,192],[59,16030,16032],{"className":16031},[196],[59,16033,16035,16055],{"className":16034},[131,132],[59,16036,16038,16052],{"className":16037},[136],[59,16039,16041],{"className":16040,"style":1248},[140],[59,16042,16043,16046],{"style":7460},[59,16044],{"className":16045,"style":540},[148],[59,16047,16049],{"className":16048},[153,154,155,156],[59,16050,14051],{"className":16051,"style":3210},[119,192,156],[59,16053,227],{"className":16054},[226],[59,16056,16058],{"className":16057},[136],[59,16059,16061],{"className":16060,"style":1270},[140],[59,16062],{},[59,16064,798],{"className":16065},[123],[59,16067,13170],{"className":16068,"style":13186},[119,192],[59,16070,812],{"className":16071},[313],[59,16073,470],{"className":16074,"style":14109},[119,192],[59,16076,798],{"className":16077},[123],[59,16079,13170],{"className":16080,"style":13186},[119,192],[59,16082,812],{"className":16083},[313],[59,16085,16087,16090,16238],{"className":16086},[119],[59,16088],{"className":16089},[123,124],[59,16091,16093],{"className":16092},[78],[59,16094,16096,16230],{"className":16095},[131,132],[59,16097,16099,16227],{"className":16098},[136],[59,16100,16102,16159,16167],{"className":16101,"style":15816},[140],[59,16103,16104,16107],{"style":15819},[59,16105],{"className":16106,"style":149},[148],[59,16108,16110,16150,16153,16156],{"className":16109},[119],[59,16111,16113,16116],{"className":16112},[119],[59,16114,5604],{"className":16115,"style":3185},[119,192],[59,16117,16119],{"className":16118},[196],[59,16120,16122,16142],{"className":16121},[131,132],[59,16123,16125,16139],{"className":16124},[136],[59,16126,16128],{"className":16127,"style":1248},[140],[59,16129,16130,16133],{"style":7460},[59,16131],{"className":16132,"style":540},[148],[59,16134,16136],{"className":16135},[153,154,155,156],[59,16137,14051],{"className":16138,"style":3210},[119,192,156],[59,16140,227],{"className":16141},[226],[59,16143,16145],{"className":16144},[136],[59,16146,16148],{"className":16147,"style":1270},[140],[59,16149],{},[59,16151,798],{"className":16152},[123],[59,16154,13170],{"className":16155,"style":13186},[119,192],[59,16157,812],{"className":16158},[313],[59,16160,16161,16164],{"style":274},[59,16162],{"className":16163,"style":149},[148],[59,16165],{"className":16166,"style":282},[281],[59,16168,16169,16172],{"style":15885},[59,16170],{"className":16171,"style":149},[148],[59,16173,16175,16178,16218,16221,16224],{"className":16174},[119],[59,16176,14380],{"className":16177},[119],[59,16179,16181,16184],{"className":16180},[119],[59,16182,5604],{"className":16183,"style":3185},[119,192],[59,16185,16187],{"className":16186},[196],[59,16188,16190,16210],{"className":16189},[131,132],[59,16191,16193,16207],{"className":16192},[136],[59,16194,16196],{"className":16195,"style":1248},[140],[59,16197,16198,16201],{"style":7460},[59,16199],{"className":16200,"style":540},[148],[59,16202,16204],{"className":16203},[153,154,155,156],[59,16205,14051],{"className":16206,"style":3210},[119,192,156],[59,16208,227],{"className":16209},[226],[59,16211,16213],{"className":16212},[136],[59,16214,16216],{"className":16215,"style":1270},[140],[59,16217],{},[59,16219,798],{"className":16220},[123],[59,16222,13170],{"className":16223,"style":13186},[119,192],[59,16225,812],{"className":16226},[313],[59,16228,227],{"className":16229},[226],[59,16231,16233],{"className":16232},[136],[59,16234,16236],{"className":16235,"style":15950},[140],[59,16237],{},[59,16239],{"className":16240},[313,124],[59,16242,16243,16246],{"style":15319},[59,16244],{"className":16245,"style":15170},[148],[59,16247,16249,16252,16255,16258,16261,16325,16328,16396,16399,16402,16405,16436,16439],{"className":16248},[119],[59,16250],{"className":16251},[119],[59,16253],{"className":16254,"style":500},[499],[59,16256,14675],{"className":16257},[504],[59,16259],{"className":16260,"style":500},[499],[59,16262,16264,16267,16322],{"className":16263},[119],[59,16265],{"className":16266},[123,124],[59,16268,16270],{"className":16269},[78],[59,16271,16273,16313],{"className":16272},[131,132],[59,16274,16276,16310],{"className":16275},[136],[59,16277,16280,16291,16299],{"className":16278,"style":16279},[140],"height:1.3214em;",[59,16281,16282,16285],{"style":15819},[59,16283],{"className":16284,"style":149},[148],[59,16286,16288],{"className":16287},[119],[59,16289,7322],{"className":16290,"style":7397},[119,192],[59,16292,16293,16296],{"style":274},[59,16294],{"className":16295,"style":149},[148],[59,16297],{"className":16298,"style":282},[281],[59,16300,16301,16304],{"style":15885},[59,16302],{"className":16303,"style":149},[148],[59,16305,16307],{"className":16306},[119],[59,16308,83],{"className":16309},[119],[59,16311,227],{"className":16312},[226],[59,16314,16316],{"className":16315},[136],[59,16317,16320],{"className":16318,"style":16319},[140],"height:0.686em;",[59,16321],{},[59,16323],{"className":16324},[313,124],[59,16326],{"className":16327,"style":916},[499],[59,16329,16331],{"className":16330},[951,14238],[59,16332,16334,16387],{"className":16333},[131,132],[59,16335,16337,16384],{"className":16336},[136],[59,16338,16341,16362,16372],{"className":16339,"style":16340},[140],"height:1.8283em;",[59,16342,16344,16347],{"style":16343},"top:-1.8829em;margin-left:0em;",[59,16345],{"className":16346,"style":15432},[148],[59,16348,16350],{"className":16349},[153,154,155,156],[59,16351,16353,16356,16359],{"className":16352},[119,156],[59,16354,335],{"className":16355},[119,192,156],[59,16357,815],{"className":16358},[504,156],[59,16360,83],{"className":16361},[119,156],[59,16363,16364,16367],{"style":15441},[59,16365],{"className":16366,"style":15432},[148],[59,16368,16369],{},[59,16370,1826],{"className":16371},[951,1910,15450],[59,16373,16375,16378],{"style":16374},"top:-4.3em;margin-left:0em;",[59,16376],{"className":16377,"style":15432},[148],[59,16379,16381],{"className":16380},[153,154,155,156],[59,16382,7322],{"className":16383,"style":7397},[119,192,156],[59,16385,227],{"className":16386},[226],[59,16388,16390],{"className":16389},[136],[59,16391,16394],{"className":16392,"style":16393},[140],"height:1.2671em;",[59,16395],{},[59,16397],{"className":16398,"style":916},[499],[59,16400,470],{"className":16401,"style":14109},[119,192],[59,16403,798],{"className":16404},[123],[59,16406,16408,16411],{"className":16407},[119],[59,16409,13170],{"className":16410,"style":13186},[119,192],[59,16412,16414],{"className":16413},[196],[59,16415,16417],{"className":16416},[131],[59,16418,16420],{"className":16419},[136],[59,16421,16424],{"className":16422,"style":16423},[140],"height:0.7144em;",[59,16425,16427,16430],{"style":16426},"top:-3.113em;margin-right:0.05em;",[59,16428],{"className":16429,"style":540},[148],[59,16431,16433],{"className":16432},[153,154,155,156],[59,16434,335],{"className":16435},[119,192,156],[59,16437,812],{"className":16438},[313],[59,16440,16442,16445,16648],{"className":16441},[119],[59,16443],{"className":16444},[123,124],[59,16446,16448],{"className":16447},[78],[59,16449,16451,16640],{"className":16450},[131,132],[59,16452,16454,16637],{"className":16453},[136],[59,16455,16457,16542,16550],{"className":16456,"style":15816},[140],[59,16458,16459,16462],{"style":15819},[59,16460],{"className":16461,"style":149},[148],[59,16463,16465,16505,16508,16539],{"className":16464},[119],[59,16466,16468,16471],{"className":16467},[119],[59,16469,5604],{"className":16470,"style":3185},[119,192],[59,16472,16474],{"className":16473},[196],[59,16475,16477,16497],{"className":16476},[131,132],[59,16478,16480,16494],{"className":16479},[136],[59,16481,16483],{"className":16482,"style":1248},[140],[59,16484,16485,16488],{"style":7460},[59,16486],{"className":16487,"style":540},[148],[59,16489,16491],{"className":16490},[153,154,155,156],[59,16492,14051],{"className":16493,"style":3210},[119,192,156],[59,16495,227],{"className":16496},[226],[59,16498,16500],{"className":16499},[136],[59,16501,16503],{"className":16502,"style":1270},[140],[59,16504],{},[59,16506,798],{"className":16507},[123],[59,16509,16511,16514],{"className":16510},[119],[59,16512,13170],{"className":16513,"style":13186},[119,192],[59,16515,16517],{"className":16516},[196],[59,16518,16520],{"className":16519},[131],[59,16521,16523],{"className":16522},[136],[59,16524,16527],{"className":16525,"style":16526},[140],"height:0.5904em;",[59,16528,16530,16533],{"style":16529},"top:-2.989em;margin-right:0.05em;",[59,16531],{"className":16532,"style":540},[148],[59,16534,16536],{"className":16535},[153,154,155,156],[59,16537,335],{"className":16538},[119,192,156],[59,16540,812],{"className":16541},[313],[59,16543,16544,16547],{"style":274},[59,16545],{"className":16546,"style":149},[148],[59,16548],{"className":16549,"style":282},[281],[59,16551,16552,16555],{"style":15885},[59,16553],{"className":16554,"style":149},[148],[59,16556,16558,16561,16601,16604,16634],{"className":16557},[119],[59,16559,14380],{"className":16560},[119],[59,16562,16564,16567],{"className":16563},[119],[59,16565,15],{"className":16566},[119,192],[59,16568,16570],{"className":16569},[196],[59,16571,16573,16593],{"className":16572},[131,132],[59,16574,16576,16590],{"className":16575},[136],[59,16577,16579],{"className":16578,"style":1248},[140],[59,16580,16581,16584],{"style":1251},[59,16582],{"className":16583,"style":540},[148],[59,16585,16587],{"className":16586},[153,154,155,156],[59,16588,14051],{"className":16589,"style":3210},[119,192,156],[59,16591,227],{"className":16592},[226],[59,16594,16596],{"className":16595},[136],[59,16597,16599],{"className":16598,"style":1270},[140],[59,16600],{},[59,16602,798],{"className":16603},[123],[59,16605,16607,16610],{"className":16606},[119],[59,16608,13170],{"className":16609,"style":13186},[119,192],[59,16611,16613],{"className":16612},[196],[59,16614,16616],{"className":16615},[131],[59,16617,16619],{"className":16618},[136],[59,16620,16623],{"className":16621,"style":16622},[140],"height:0.6644em;",[59,16624,16625,16628],{"style":536},[59,16626],{"className":16627,"style":540},[148],[59,16629,16631],{"className":16630},[153,154,155,156],[59,16632,335],{"className":16633},[119,192,156],[59,16635,812],{"className":16636},[313],[59,16638,227],{"className":16639},[226],[59,16641,16643],{"className":16642},[136],[59,16644,16646],{"className":16645,"style":15950},[140],[59,16647],{},[59,16649],{"className":16650},[313,124],[59,16652,16653,16656],{"style":15328},[59,16654],{"className":16655,"style":15170},[148],[59,16657,16659,16662,16665,16668,16671,16733,16736,16800,16803,16806,16809,16838,16841,16844,16847,16855,16858,16898,16901,16930],{"className":16658},[119],[59,16660],{"className":16661},[119],[59,16663],{"className":16664,"style":500},[499],[59,16666,815],{"className":16667},[504],[59,16669],{"className":16670,"style":500},[499],[59,16672,16674,16677,16730],{"className":16673},[119],[59,16675],{"className":16676},[123,124],[59,16678,16680],{"className":16679},[78],[59,16681,16683,16722],{"className":16682},[131,132],[59,16684,16686,16719],{"className":16685},[136],[59,16687,16689,16700,16708],{"className":16688,"style":16279},[140],[59,16690,16691,16694],{"style":15819},[59,16692],{"className":16693,"style":149},[148],[59,16695,16697],{"className":16696},[119],[59,16698,7322],{"className":16699,"style":7397},[119,192],[59,16701,16702,16705],{"style":274},[59,16703],{"className":16704,"style":149},[148],[59,16706],{"className":16707,"style":282},[281],[59,16709,16710,16713],{"style":15885},[59,16711],{"className":16712,"style":149},[148],[59,16714,16716],{"className":16715},[119],[59,16717,83],{"className":16718},[119],[59,16720,227],{"className":16721},[226],[59,16723,16725],{"className":16724},[136],[59,16726,16728],{"className":16727,"style":16319},[140],[59,16729],{},[59,16731],{"className":16732},[313,124],[59,16734],{"className":16735,"style":916},[499],[59,16737,16739],{"className":16738},[951,14238],[59,16740,16742,16792],{"className":16741},[131,132],[59,16743,16745,16789],{"className":16744},[136],[59,16746,16748,16768,16778],{"className":16747,"style":16340},[140],[59,16749,16750,16753],{"style":16343},[59,16751],{"className":16752,"style":15432},[148],[59,16754,16756],{"className":16755},[153,154,155,156],[59,16757,16759,16762,16765],{"className":16758},[119,156],[59,16760,335],{"className":16761},[119,192,156],[59,16763,815],{"className":16764},[504,156],[59,16766,83],{"className":16767},[119,156],[59,16769,16770,16773],{"style":15441},[59,16771],{"className":16772,"style":15432},[148],[59,16774,16775],{},[59,16776,1826],{"className":16777},[951,1910,15450],[59,16779,16780,16783],{"style":16374},[59,16781],{"className":16782,"style":15432},[148],[59,16784,16786],{"className":16785},[153,154,155,156],[59,16787,7322],{"className":16788,"style":7397},[119,192,156],[59,16790,227],{"className":16791},[226],[59,16793,16795],{"className":16794},[136],[59,16796,16798],{"className":16797,"style":16393},[140],[59,16799],{},[59,16801],{"className":16802,"style":916},[499],[59,16804,470],{"className":16805,"style":14109},[119,192],[59,16807,798],{"className":16808},[123],[59,16810,16812,16815],{"className":16811},[119],[59,16813,13170],{"className":16814,"style":13186},[119,192],[59,16816,16818],{"className":16817},[196],[59,16819,16821],{"className":16820},[131],[59,16822,16824],{"className":16823},[136],[59,16825,16827],{"className":16826,"style":16423},[140],[59,16828,16829,16832],{"style":16426},[59,16830],{"className":16831,"style":540},[148],[59,16833,16835],{"className":16834},[153,154,155,156],[59,16836,335],{"className":16837},[119,192,156],[59,16839,812],{"className":16840},[313],[59,16842,14380],{"className":16843},[119],[59,16845],{"className":16846,"style":916},[499],[59,16848,16850,16851],{"className":16849},[951],"lo",[59,16852,16854],{"style":16853},"margin-right:0.01389em;","g",[59,16856],{"className":16857,"style":916},[499],[59,16859,16861,16864],{"className":16860},[119],[59,16862,5604],{"className":16863,"style":3185},[119,192],[59,16865,16867],{"className":16866},[196],[59,16868,16870,16890],{"className":16869},[131,132],[59,16871,16873,16887],{"className":16872},[136],[59,16874,16876],{"className":16875,"style":1248},[140],[59,16877,16878,16881],{"style":7460},[59,16879],{"className":16880,"style":540},[148],[59,16882,16884],{"className":16883},[153,154,155,156],[59,16885,14051],{"className":16886,"style":3210},[119,192,156],[59,16888,227],{"className":16889},[226],[59,16891,16893],{"className":16892},[136],[59,16894,16896],{"className":16895,"style":1270},[140],[59,16897],{},[59,16899,798],{"className":16900},[123],[59,16902,16904,16907],{"className":16903},[119],[59,16905,13170],{"className":16906,"style":13186},[119,192],[59,16908,16910],{"className":16909},[196],[59,16911,16913],{"className":16912},[131],[59,16914,16916],{"className":16915},[136],[59,16917,16919],{"className":16918,"style":16423},[140],[59,16920,16921,16924],{"style":16426},[59,16922],{"className":16923,"style":540},[148],[59,16925,16927],{"className":16926},[153,154,155,156],[59,16928,335],{"className":16929},[119,192,156],[59,16931,812],{"className":16932},[313],[59,16934,16935,16938],{"style":15337},[59,16936],{"className":16937,"style":15170},[148],[59,16939,16941,16944,16947,16950,16953,17015,17018,17082,17085,17088,17091,17120,17123,17126,17129,17134,17137,17244,17247,17287,17290,17344,17347,17398],{"className":16940},[119],[59,16942],{"className":16943},[119],[59,16945],{"className":16946,"style":500},[499],[59,16948,815],{"className":16949},[504],[59,16951],{"className":16952,"style":500},[499],[59,16954,16956,16959,17012],{"className":16955},[119],[59,16957],{"className":16958},[123,124],[59,16960,16962],{"className":16961},[78],[59,16963,16965,17004],{"className":16964},[131,132],[59,16966,16968,17001],{"className":16967},[136],[59,16969,16971,16982,16990],{"className":16970,"style":16279},[140],[59,16972,16973,16976],{"style":15819},[59,16974],{"className":16975,"style":149},[148],[59,16977,16979],{"className":16978},[119],[59,16980,7322],{"className":16981,"style":7397},[119,192],[59,16983,16984,16987],{"style":274},[59,16985],{"className":16986,"style":149},[148],[59,16988],{"className":16989,"style":282},[281],[59,16991,16992,16995],{"style":15885},[59,16993],{"className":16994,"style":149},[148],[59,16996,16998],{"className":16997},[119],[59,16999,83],{"className":17000},[119],[59,17002,227],{"className":17003},[226],[59,17005,17007],{"className":17006},[136],[59,17008,17010],{"className":17009,"style":16319},[140],[59,17011],{},[59,17013],{"className":17014},[313,124],[59,17016],{"className":17017,"style":916},[499],[59,17019,17021],{"className":17020},[951,14238],[59,17022,17024,17074],{"className":17023},[131,132],[59,17025,17027,17071],{"className":17026},[136],[59,17028,17030,17050,17060],{"className":17029,"style":16340},[140],[59,17031,17032,17035],{"style":16343},[59,17033],{"className":17034,"style":15432},[148],[59,17036,17038],{"className":17037},[153,154,155,156],[59,17039,17041,17044,17047],{"className":17040},[119,156],[59,17042,335],{"className":17043},[119,192,156],[59,17045,815],{"className":17046},[504,156],[59,17048,83],{"className":17049},[119,156],[59,17051,17052,17055],{"style":15441},[59,17053],{"className":17054,"style":15432},[148],[59,17056,17057],{},[59,17058,1826],{"className":17059},[951,1910,15450],[59,17061,17062,17065],{"style":16374},[59,17063],{"className":17064,"style":15432},[148],[59,17066,17068],{"className":17067},[153,154,155,156],[59,17069,7322],{"className":17070,"style":7397},[119,192,156],[59,17072,227],{"className":17073},[226],[59,17075,17077],{"className":17076},[136],[59,17078,17080],{"className":17079,"style":16393},[140],[59,17081],{},[59,17083],{"className":17084,"style":916},[499],[59,17086,470],{"className":17087,"style":14109},[119,192],[59,17089,798],{"className":17090},[123],[59,17092,17094,17097],{"className":17093},[119],[59,17095,13170],{"className":17096,"style":13186},[119,192],[59,17098,17100],{"className":17099},[196],[59,17101,17103],{"className":17102},[131],[59,17104,17106],{"className":17105},[136],[59,17107,17109],{"className":17108,"style":16423},[140],[59,17110,17111,17114],{"style":16426},[59,17112],{"className":17113,"style":540},[148],[59,17115,17117],{"className":17116},[153,154,155,156],[59,17118,335],{"className":17119},[119,192,156],[59,17121,812],{"className":17122},[313],[59,17124,14380],{"className":17125},[119],[59,17127],{"className":17128,"style":916},[499],[59,17130,16850,17132],{"className":17131},[951],[59,17133,16854],{"style":16853},[59,17135],{"className":17136,"style":916},[499],[59,17138,17140],{"className":17139},[951,14238],[59,17141,17143,17236],{"className":17142},[131,132],[59,17144,17146,17233],{"className":17145},[136],[59,17147,17150,17170,17180],{"className":17148,"style":17149},[140],"height:1.8394em;",[59,17151,17152,17155],{"style":16343},[59,17153],{"className":17154,"style":15432},[148],[59,17156,17158],{"className":17157},[153,154,155,156],[59,17159,17161,17164,17167],{"className":17160},[119,156],[59,17162,777],{"className":17163},[119,192,156],[59,17165,815],{"className":17166},[504,156],[59,17168,83],{"className":17169},[119,156],[59,17171,17172,17175],{"style":15441},[59,17173],{"className":17174,"style":15432},[148],[59,17176,17177],{},[59,17178,14884],{"className":17179},[951,1910,15450],[59,17181,17183,17186],{"style":17182},"top:-4.3111em;margin-left:0em;",[59,17184],{"className":17185,"style":15432},[148],[59,17187,17189],{"className":17188},[153,154,155,156],[59,17190,17192],{"className":17191},[119,156],[59,17193,17195,17198],{"className":17194},[119,156],[59,17196,13242],{"className":17197,"style":3185},[119,192,156],[59,17199,17201],{"className":17200},[196],[59,17202,17204,17225],{"className":17203},[131,132],[59,17205,17207,17222],{"className":17206},[136],[59,17208,17210],{"className":17209,"style":4302},[140],[59,17211,17213,17216],{"style":17212},"top:-2.357em;margin-left:-0.1389em;margin-right:0.0714em;",[59,17214],{"className":17215,"style":213},[148],[59,17217,17219],{"className":17218},[153,217,218,156],[59,17220,335],{"className":17221},[119,192,156],[59,17223,227],{"className":17224},[226],[59,17226,17228],{"className":17227},[136],[59,17229,17231],{"className":17230,"style":4324},[140],[59,17232],{},[59,17234,227],{"className":17235},[226],[59,17237,17239],{"className":17238},[136],[59,17240,17242],{"className":17241,"style":16393},[140],[59,17243],{},[59,17245],{"className":17246,"style":916},[499],[59,17248,17250,17253],{"className":17249},[119],[59,17251,5604],{"className":17252,"style":3185},[119,192],[59,17254,17256],{"className":17255},[196],[59,17257,17259,17279],{"className":17258},[131,132],[59,17260,17262,17276],{"className":17261},[136],[59,17263,17265],{"className":17264,"style":1248},[140],[59,17266,17267,17270],{"style":7460},[59,17268],{"className":17269,"style":540},[148],[59,17271,17273],{"className":17272},[153,154,155,156],[59,17274,14051],{"className":17275,"style":3210},[119,192,156],[59,17277,227],{"className":17278},[226],[59,17280,17282],{"className":17281},[136],[59,17283,17285],{"className":17284,"style":1270},[140],[59,17286],{},[59,17288,798],{"className":17289},[123],[59,17291,17293,17296],{"className":17292},[119],[59,17294,22],{"className":17295},[119,192],[59,17297,17299],{"className":17298},[196],[59,17300,17302,17335],{"className":17301},[131,132],[59,17303,17305,17332],{"className":17304},[136],[59,17306,17309,17321],{"className":17307,"style":17308},[140],"height:0.8436em;",[59,17310,17312,17315],{"style":17311},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[59,17313],{"className":17314,"style":540},[148],[59,17316,17318],{"className":17317},[153,154,155,156],[59,17319,335],{"className":17320},[119,192,156],[59,17322,17323,17326],{"style":16426},[59,17324],{"className":17325,"style":540},[148],[59,17327,17329],{"className":17328},[153,154,155,156],[59,17330,777],{"className":17331},[119,192,156],[59,17333,227],{"className":17334},[226],[59,17336,17338],{"className":17337},[136],[59,17339,17342],{"className":17340,"style":17341},[140],"height:0.247em;",[59,17343],{},[59,17345,12941],{"className":17346},[119],[59,17348,17350,17353],{"className":17349},[119],[59,17351,12802],{"className":17352},[119,192],[59,17354,17356],{"className":17355},[196],[59,17357,17359,17390],{"className":17358},[131,132],[59,17360,17362,17387],{"className":17361},[136],[59,17363,17365,17376],{"className":17364,"style":17308},[140],[59,17366,17367,17370],{"style":17311},[59,17368],{"className":17369,"style":540},[148],[59,17371,17373],{"className":17372},[153,154,155,156],[59,17374,335],{"className":17375},[119,192,156],[59,17377,17378,17381],{"style":16426},[59,17379],{"className":17380,"style":540},[148],[59,17382,17384],{"className":17383},[153,154,155,156],[59,17385,777],{"className":17386},[119,192,156],[59,17388,227],{"className":17389},[226],[59,17391,17393],{"className":17392},[136],[59,17394,17396],{"className":17395,"style":17341},[140],[59,17397],{},[59,17399,812],{"className":17400},[313],[59,17402,17403,17406],{"style":15346},[59,17404],{"className":17405,"style":15170},[148],[59,17407,17409,17412,17415,17418,17421,17483,17486,17550,17553,17556,17559,17588,17591,17594,17698,17701,17704,17707,17712,17715,17755,17758,17809,17812,17863],{"className":17408},[119],[59,17410],{"className":17411},[119],[59,17413],{"className":17414,"style":500},[499],[59,17416,815],{"className":17417},[504],[59,17419],{"className":17420,"style":500},[499],[59,17422,17424,17427,17480],{"className":17423},[119],[59,17425],{"className":17426},[123,124],[59,17428,17430],{"className":17429},[78],[59,17431,17433,17472],{"className":17432},[131,132],[59,17434,17436,17469],{"className":17435},[136],[59,17437,17439,17450,17458],{"className":17438,"style":16279},[140],[59,17440,17441,17444],{"style":15819},[59,17442],{"className":17443,"style":149},[148],[59,17445,17447],{"className":17446},[119],[59,17448,7322],{"className":17449,"style":7397},[119,192],[59,17451,17452,17455],{"style":274},[59,17453],{"className":17454,"style":149},[148],[59,17456],{"className":17457,"style":282},[281],[59,17459,17460,17463],{"style":15885},[59,17461],{"className":17462,"style":149},[148],[59,17464,17466],{"className":17465},[119],[59,17467,83],{"className":17468},[119],[59,17470,227],{"className":17471},[226],[59,17473,17475],{"className":17474},[136],[59,17476,17478],{"className":17477,"style":16319},[140],[59,17479],{},[59,17481],{"className":17482},[313,124],[59,17484],{"className":17485,"style":916},[499],[59,17487,17489],{"className":17488},[951,14238],[59,17490,17492,17542],{"className":17491},[131,132],[59,17493,17495,17539],{"className":17494},[136],[59,17496,17498,17518,17528],{"className":17497,"style":16340},[140],[59,17499,17500,17503],{"style":16343},[59,17501],{"className":17502,"style":15432},[148],[59,17504,17506],{"className":17505},[153,154,155,156],[59,17507,17509,17512,17515],{"className":17508},[119,156],[59,17510,335],{"className":17511},[119,192,156],[59,17513,815],{"className":17514},[504,156],[59,17516,83],{"className":17517},[119,156],[59,17519,17520,17523],{"style":15441},[59,17521],{"className":17522,"style":15432},[148],[59,17524,17525],{},[59,17526,1826],{"className":17527},[951,1910,15450],[59,17529,17530,17533],{"style":16374},[59,17531],{"className":17532,"style":15432},[148],[59,17534,17536],{"className":17535},[153,154,155,156],[59,17537,7322],{"className":17538,"style":7397},[119,192,156],[59,17540,227],{"className":17541},[226],[59,17543,17545],{"className":17544},[136],[59,17546,17548],{"className":17547,"style":16393},[140],[59,17549],{},[59,17551],{"className":17552,"style":916},[499],[59,17554,470],{"className":17555,"style":14109},[119,192],[59,17557,798],{"className":17558},[123],[59,17560,17562,17565],{"className":17561},[119],[59,17563,13170],{"className":17564,"style":13186},[119,192],[59,17566,17568],{"className":17567},[196],[59,17569,17571],{"className":17570},[131],[59,17572,17574],{"className":17573},[136],[59,17575,17577],{"className":17576,"style":16423},[140],[59,17578,17579,17582],{"style":16426},[59,17580],{"className":17581,"style":540},[148],[59,17583,17585],{"className":17584},[153,154,155,156],[59,17586,335],{"className":17587},[119,192,156],[59,17589,812],{"className":17590},[313],[59,17592],{"className":17593,"style":916},[499],[59,17595,17597],{"className":17596},[951,14238],[59,17598,17600,17690],{"className":17599},[131,132],[59,17601,17603,17687],{"className":17602},[136],[59,17604,17606,17626,17636],{"className":17605,"style":17149},[140],[59,17607,17608,17611],{"style":16343},[59,17609],{"className":17610,"style":15432},[148],[59,17612,17614],{"className":17613},[153,154,155,156],[59,17615,17617,17620,17623],{"className":17616},[119,156],[59,17618,777],{"className":17619},[119,192,156],[59,17621,815],{"className":17622},[504,156],[59,17624,83],{"className":17625},[119,156],[59,17627,17628,17631],{"style":15441},[59,17629],{"className":17630,"style":15432},[148],[59,17632,17633],{},[59,17634,1826],{"className":17635},[951,1910,15450],[59,17637,17638,17641],{"style":17182},[59,17639],{"className":17640,"style":15432},[148],[59,17642,17644],{"className":17643},[153,154,155,156],[59,17645,17647],{"className":17646},[119,156],[59,17648,17650,17653],{"className":17649},[119,156],[59,17651,13242],{"className":17652,"style":3185},[119,192,156],[59,17654,17656],{"className":17655},[196],[59,17657,17659,17679],{"className":17658},[131,132],[59,17660,17662,17676],{"className":17661},[136],[59,17663,17665],{"className":17664,"style":4302},[140],[59,17666,17667,17670],{"style":17212},[59,17668],{"className":17669,"style":213},[148],[59,17671,17673],{"className":17672},[153,217,218,156],[59,17674,335],{"className":17675},[119,192,156],[59,17677,227],{"className":17678},[226],[59,17680,17682],{"className":17681},[136],[59,17683,17685],{"className":17684,"style":4324},[140],[59,17686],{},[59,17688,227],{"className":17689},[226],[59,17691,17693],{"className":17692},[136],[59,17694,17696],{"className":17695,"style":16393},[140],[59,17697],{},[59,17699],{"className":17700,"style":916},[499],[59,17702,14380],{"className":17703},[119],[59,17705],{"className":17706,"style":916},[499],[59,17708,16850,17710],{"className":17709},[951],[59,17711,16854],{"style":16853},[59,17713],{"className":17714,"style":916},[499],[59,17716,17718,17721],{"className":17717},[119],[59,17719,5604],{"className":17720,"style":3185},[119,192],[59,17722,17724],{"className":17723},[196],[59,17725,17727,17747],{"className":17726},[131,132],[59,17728,17730,17744],{"className":17729},[136],[59,17731,17733],{"className":17732,"style":1248},[140],[59,17734,17735,17738],{"style":7460},[59,17736],{"className":17737,"style":540},[148],[59,17739,17741],{"className":17740},[153,154,155,156],[59,17742,14051],{"className":17743,"style":3210},[119,192,156],[59,17745,227],{"className":17746},[226],[59,17748,17750],{"className":17749},[136],[59,17751,17753],{"className":17752,"style":1270},[140],[59,17754],{},[59,17756,798],{"className":17757},[123],[59,17759,17761,17764],{"className":17760},[119],[59,17762,22],{"className":17763},[119,192],[59,17765,17767],{"className":17766},[196],[59,17768,17770,17801],{"className":17769},[131,132],[59,17771,17773,17798],{"className":17772},[136],[59,17774,17776,17787],{"className":17775,"style":17308},[140],[59,17777,17778,17781],{"style":17311},[59,17779],{"className":17780,"style":540},[148],[59,17782,17784],{"className":17783},[153,154,155,156],[59,17785,335],{"className":17786},[119,192,156],[59,17788,17789,17792],{"style":16426},[59,17790],{"className":17791,"style":540},[148],[59,17793,17795],{"className":17794},[153,154,155,156],[59,17796,777],{"className":17797},[119,192,156],[59,17799,227],{"className":17800},[226],[59,17802,17804],{"className":17803},[136],[59,17805,17807],{"className":17806,"style":17341},[140],[59,17808],{},[59,17810,12941],{"className":17811},[119],[59,17813,17815,17818],{"className":17814},[119],[59,17816,12802],{"className":17817},[119,192],[59,17819,17821],{"className":17820},[196],[59,17822,17824,17855],{"className":17823},[131,132],[59,17825,17827,17852],{"className":17826},[136],[59,17828,17830,17841],{"className":17829,"style":17308},[140],[59,17831,17832,17835],{"style":17311},[59,17833],{"className":17834,"style":540},[148],[59,17836,17838],{"className":17837},[153,154,155,156],[59,17839,335],{"className":17840},[119,192,156],[59,17842,17843,17846],{"style":16426},[59,17844],{"className":17845,"style":540},[148],[59,17847,17849],{"className":17848},[153,154,155,156],[59,17850,777],{"className":17851},[119,192,156],[59,17853,227],{"className":17854},[226],[59,17856,17858],{"className":17857},[136],[59,17859,17861],{"className":17860,"style":17341},[140],[59,17862],{},[59,17864,812],{"className":17865},[313],[59,17867,17868,17871],{"style":15355},[59,17869],{"className":17870,"style":15170},[148],[59,17872,17874,17877,17880,17883,17886,17948,17951,18015,18018,18122,18125,18128,18131,18160,18163,18166,18169,18174,18177,18217,18220,18271,18274,18325],{"className":17873},[119],[59,17875],{"className":17876},[119],[59,17878],{"className":17879,"style":500},[499],[59,17881,815],{"className":17882},[504],[59,17884],{"className":17885,"style":500},[499],[59,17887,17889,17892,17945],{"className":17888},[119],[59,17890],{"className":17891},[123,124],[59,17893,17895],{"className":17894},[78],[59,17896,17898,17937],{"className":17897},[131,132],[59,17899,17901,17934],{"className":17900},[136],[59,17902,17904,17915,17923],{"className":17903,"style":16279},[140],[59,17905,17906,17909],{"style":15819},[59,17907],{"className":17908,"style":149},[148],[59,17910,17912],{"className":17911},[119],[59,17913,7322],{"className":17914,"style":7397},[119,192],[59,17916,17917,17920],{"style":274},[59,17918],{"className":17919,"style":149},[148],[59,17921],{"className":17922,"style":282},[281],[59,17924,17925,17928],{"style":15885},[59,17926],{"className":17927,"style":149},[148],[59,17929,17931],{"className":17930},[119],[59,17932,83],{"className":17933},[119],[59,17935,227],{"className":17936},[226],[59,17938,17940],{"className":17939},[136],[59,17941,17943],{"className":17942,"style":16319},[140],[59,17944],{},[59,17946],{"className":17947},[313,124],[59,17949],{"className":17950,"style":916},[499],[59,17952,17954],{"className":17953},[951,14238],[59,17955,17957,18007],{"className":17956},[131,132],[59,17958,17960,18004],{"className":17959},[136],[59,17961,17963,17983,17993],{"className":17962,"style":16340},[140],[59,17964,17965,17968],{"style":16343},[59,17966],{"className":17967,"style":15432},[148],[59,17969,17971],{"className":17970},[153,154,155,156],[59,17972,17974,17977,17980],{"className":17973},[119,156],[59,17975,335],{"className":17976},[119,192,156],[59,17978,815],{"className":17979},[504,156],[59,17981,83],{"className":17982},[119,156],[59,17984,17985,17988],{"style":15441},[59,17986],{"className":17987,"style":15432},[148],[59,17989,17990],{},[59,17991,1826],{"className":17992},[951,1910,15450],[59,17994,17995,17998],{"style":16374},[59,17996],{"className":17997,"style":15432},[148],[59,17999,18001],{"className":18000},[153,154,155,156],[59,18002,7322],{"className":18003,"style":7397},[119,192,156],[59,18005,227],{"className":18006},[226],[59,18008,18010],{"className":18009},[136],[59,18011,18013],{"className":18012,"style":16393},[140],[59,18014],{},[59,18016],{"className":18017,"style":916},[499],[59,18019,18021],{"className":18020},[951,14238],[59,18022,18024,18114],{"className":18023},[131,132],[59,18025,18027,18111],{"className":18026},[136],[59,18028,18030,18050,18060],{"className":18029,"style":17149},[140],[59,18031,18032,18035],{"style":16343},[59,18033],{"className":18034,"style":15432},[148],[59,18036,18038],{"className":18037},[153,154,155,156],[59,18039,18041,18044,18047],{"className":18040},[119,156],[59,18042,777],{"className":18043},[119,192,156],[59,18045,815],{"className":18046},[504,156],[59,18048,83],{"className":18049},[119,156],[59,18051,18052,18055],{"style":15441},[59,18053],{"className":18054,"style":15432},[148],[59,18056,18057],{},[59,18058,1826],{"className":18059},[951,1910,15450],[59,18061,18062,18065],{"style":17182},[59,18063],{"className":18064,"style":15432},[148],[59,18066,18068],{"className":18067},[153,154,155,156],[59,18069,18071],{"className":18070},[119,156],[59,18072,18074,18077],{"className":18073},[119,156],[59,18075,13242],{"className":18076,"style":3185},[119,192,156],[59,18078,18080],{"className":18079},[196],[59,18081,18083,18103],{"className":18082},[131,132],[59,18084,18086,18100],{"className":18085},[136],[59,18087,18089],{"className":18088,"style":4302},[140],[59,18090,18091,18094],{"style":17212},[59,18092],{"className":18093,"style":213},[148],[59,18095,18097],{"className":18096},[153,217,218,156],[59,18098,335],{"className":18099},[119,192,156],[59,18101,227],{"className":18102},[226],[59,18104,18106],{"className":18105},[136],[59,18107,18109],{"className":18108,"style":4324},[140],[59,18110],{},[59,18112,227],{"className":18113},[226],[59,18115,18117],{"className":18116},[136],[59,18118,18120],{"className":18119,"style":16393},[140],[59,18121],{},[59,18123],{"className":18124,"style":916},[499],[59,18126,470],{"className":18127,"style":14109},[119,192],[59,18129,798],{"className":18130},[123],[59,18132,18134,18137],{"className":18133},[119],[59,18135,13170],{"className":18136,"style":13186},[119,192],[59,18138,18140],{"className":18139},[196],[59,18141,18143],{"className":18142},[131],[59,18144,18146],{"className":18145},[136],[59,18147,18149],{"className":18148,"style":16423},[140],[59,18150,18151,18154],{"style":16426},[59,18152],{"className":18153,"style":540},[148],[59,18155,18157],{"className":18156},[153,154,155,156],[59,18158,335],{"className":18159},[119,192,156],[59,18161,812],{"className":18162},[313],[59,18164,14380],{"className":18165},[119],[59,18167],{"className":18168,"style":916},[499],[59,18170,16850,18172],{"className":18171},[951],[59,18173,16854],{"style":16853},[59,18175],{"className":18176,"style":916},[499],[59,18178,18180,18183],{"className":18179},[119],[59,18181,5604],{"className":18182,"style":3185},[119,192],[59,18184,18186],{"className":18185},[196],[59,18187,18189,18209],{"className":18188},[131,132],[59,18190,18192,18206],{"className":18191},[136],[59,18193,18195],{"className":18194,"style":1248},[140],[59,18196,18197,18200],{"style":7460},[59,18198],{"className":18199,"style":540},[148],[59,18201,18203],{"className":18202},[153,154,155,156],[59,18204,14051],{"className":18205,"style":3210},[119,192,156],[59,18207,227],{"className":18208},[226],[59,18210,18212],{"className":18211},[136],[59,18213,18215],{"className":18214,"style":1270},[140],[59,18216],{},[59,18218,798],{"className":18219},[123],[59,18221,18223,18226],{"className":18222},[119],[59,18224,22],{"className":18225},[119,192],[59,18227,18229],{"className":18228},[196],[59,18230,18232,18263],{"className":18231},[131,132],[59,18233,18235,18260],{"className":18234},[136],[59,18236,18238,18249],{"className":18237,"style":17308},[140],[59,18239,18240,18243],{"style":17311},[59,18241],{"className":18242,"style":540},[148],[59,18244,18246],{"className":18245},[153,154,155,156],[59,18247,335],{"className":18248},[119,192,156],[59,18250,18251,18254],{"style":16426},[59,18252],{"className":18253,"style":540},[148],[59,18255,18257],{"className":18256},[153,154,155,156],[59,18258,777],{"className":18259},[119,192,156],[59,18261,227],{"className":18262},[226],[59,18264,18266],{"className":18265},[136],[59,18267,18269],{"className":18268,"style":17341},[140],[59,18270],{},[59,18272,12941],{"className":18273},[119],[59,18275,18277,18280],{"className":18276},[119],[59,18278,12802],{"className":18279},[119,192],[59,18281,18283],{"className":18282},[196],[59,18284,18286,18317],{"className":18285},[131,132],[59,18287,18289,18314],{"className":18288},[136],[59,18290,18292,18303],{"className":18291,"style":17308},[140],[59,18293,18294,18297],{"style":17311},[59,18295],{"className":18296,"style":540},[148],[59,18298,18300],{"className":18299},[153,154,155,156],[59,18301,335],{"className":18302},[119,192,156],[59,18304,18305,18308],{"style":16426},[59,18306],{"className":18307,"style":540},[148],[59,18309,18311],{"className":18310},[153,154,155,156],[59,18312,777],{"className":18313},[119,192,156],[59,18315,227],{"className":18316},[226],[59,18318,18320],{"className":18319},[136],[59,18321,18323],{"className":18322,"style":17341},[140],[59,18324],{},[59,18326,812],{"className":18327},[313],[59,18329,227],{"className":18330},[226],[59,18332,18334],{"className":18333},[136],[59,18335,18337],{"className":18336,"style":15371},[140],[59,18338],{},[15,18340,18341],{},[59,18342,18344,18451],{"className":18343},[62],[59,18345,18347],{"className":18346},[66],[68,18348,18349],{"xmlns":70},[72,18350,18351,18448],{},[75,18352,18353,18355,18357,18359,18361,18363,18366,18372,18386,18404,18406,18408,18414,18416,18418,18420,18426,18428,18436,18438,18446],{},[91,18354,7816],{},[91,18356,792],{},[91,18358,12802],{},[91,18360,12802],{},[461,18362,815],{},[461,18364,18365],{},"−",[78,18367,18368,18370],{},[81,18369,83],{},[91,18371,7322],{},[1822,18373,18374,18376,18384],{},[461,18375,1826],{},[75,18377,18378,18380,18382],{},[91,18379,335],{},[461,18381,815],{},[81,18383,83],{},[91,18385,7322],{},[1822,18387,18388,18390,18398],{},[461,18389,1826],{},[75,18391,18392,18394,18396],{},[91,18393,777],{},[461,18395,815],{},[81,18397,83],{},[88,18399,18400,18402],{},[91,18401,13242],{},[91,18403,335],{},[91,18405,470],{},[461,18407,798],{"stretchy":797},[465,18409,18410,18412],{},[91,18411,13170],{},[91,18413,335],{},[461,18415,812],{"stretchy":797},[91,18417,14805],{},[461,18419,822],{},[88,18421,18422,18424],{},[91,18423,5604],{},[91,18425,14051],{},[461,18427,798],{"stretchy":797},[1822,18429,18430,18432,18434],{},[91,18431,22],{},[91,18433,335],{},[91,18435,777],{},[91,18437,12941],{"mathvariant":818},[1822,18439,18440,18442,18444],{},[91,18441,12802],{},[91,18443,335],{},[91,18445,777],{},[461,18447,812],{"stretchy":797},[98,18449,18450],{"encoding":100},"Loss = -\\frac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} R(\\tau^n) \\log P_\\theta(a_n^t|s_n^t)",[59,18452,18454,18476],{"className":18453,"ariaHidden":106},[105],[59,18455,18457,18460,18463,18467,18470,18473],{"className":18456},[110],[59,18458],{"className":18459,"style":14003},[114],[59,18461,7816],{"className":18462},[119,192],[59,18464,18466],{"className":18465},[119,192],"oss",[59,18468],{"className":18469,"style":500},[499],[59,18471,815],{"className":18472},[504],[59,18474],{"className":18475,"style":500},[499],[59,18477,18479,18483,18486,18556,18559,18627,18630,18736,18739,18742,18745,18774,18777,18780,18785,18788,18828,18831,18883,18886,18937],{"className":18478},[110],[59,18480],{"className":18481,"style":18482},[114],"height:2.5065em;vertical-align:-0.9671em;",[59,18484,18365],{"className":18485},[119],[59,18487,18489,18492,18553],{"className":18488},[119],[59,18490],{"className":18491},[123,124],[59,18493,18495],{"className":18494},[78],[59,18496,18498,18544],{"className":18497},[131,132],[59,18499,18501,18541],{"className":18500},[136],[59,18502,18504,18519,18527],{"className":18503,"style":141},[140],[59,18505,18507,18510],{"style":18506},"top:-2.655em;",[59,18508],{"className":18509,"style":149},[148],[59,18511,18513],{"className":18512},[153,154,155,156],[59,18514,18516],{"className":18515},[119,156],[59,18517,7322],{"className":18518,"style":7397},[119,192,156],[59,18520,18521,18524],{"style":274},[59,18522],{"className":18523,"style":149},[148],[59,18525],{"className":18526,"style":282},[281],[59,18528,18529,18532],{"style":285},[59,18530],{"className":18531,"style":149},[148],[59,18533,18535],{"className":18534},[153,154,155,156],[59,18536,18538],{"className":18537},[119,156],[59,18539,83],{"className":18540},[119,156],[59,18542,227],{"className":18543},[226],[59,18545,18547],{"className":18546},[136],[59,18548,18551],{"className":18549,"style":18550},[140],"height:0.345em;",[59,18552],{},[59,18554],{"className":18555},[313,124],[59,18557],{"className":18558,"style":916},[499],[59,18560,18562],{"className":18561},[951,14238],[59,18563,18565,18618],{"className":18564},[131,132],[59,18566,18568,18615],{"className":18567},[136],[59,18569,18572,18593,18603],{"className":18570,"style":18571},[140],"height:1.5283em;",[59,18573,18575,18578],{"style":18574},"top:-2.1329em;margin-left:0em;",[59,18576],{"className":18577,"style":149},[148],[59,18579,18581],{"className":18580},[153,154,155,156],[59,18582,18584,18587,18590],{"className":18583},[119,156],[59,18585,335],{"className":18586},[119,192,156],[59,18588,815],{"className":18589},[504,156],[59,18591,83],{"className":18592},[119,156],[59,18594,18595,18598],{"style":178},[59,18596],{"className":18597,"style":149},[148],[59,18599,18600],{},[59,18601,1826],{"className":18602},[951,1910,1911],[59,18604,18606,18609],{"style":18605},"top:-3.95em;margin-left:0em;",[59,18607],{"className":18608,"style":149},[148],[59,18610,18612],{"className":18611},[153,154,155,156],[59,18613,7322],{"className":18614,"style":7397},[119,192,156],[59,18616,227],{"className":18617},[226],[59,18619,18621],{"className":18620},[136],[59,18622,18625],{"className":18623,"style":18624},[140],"height:0.9671em;",[59,18626],{},[59,18628],{"className":18629,"style":916},[499],[59,18631,18633],{"className":18632},[951,14238],[59,18634,18636,18728],{"className":18635},[131,132],[59,18637,18639,18725],{"className":18638},[136],[59,18640,18643,18663,18673],{"className":18641,"style":18642},[140],"height:1.5394em;",[59,18644,18645,18648],{"style":18574},[59,18646],{"className":18647,"style":149},[148],[59,18649,18651],{"className":18650},[153,154,155,156],[59,18652,18654,18657,18660],{"className":18653},[119,156],[59,18655,777],{"className":18656},[119,192,156],[59,18658,815],{"className":18659},[504,156],[59,18661,83],{"className":18662},[119,156],[59,18664,18665,18668],{"style":178},[59,18666],{"className":18667,"style":149},[148],[59,18669,18670],{},[59,18671,1826],{"className":18672},[951,1910,1911],[59,18674,18676,18679],{"style":18675},"top:-3.9611em;margin-left:0em;",[59,18677],{"className":18678,"style":149},[148],[59,18680,18682],{"className":18681},[153,154,155,156],[59,18683,18685],{"className":18684},[119,156],[59,18686,18688,18691],{"className":18687},[119,156],[59,18689,13242],{"className":18690,"style":3185},[119,192,156],[59,18692,18694],{"className":18693},[196],[59,18695,18697,18717],{"className":18696},[131,132],[59,18698,18700,18714],{"className":18699},[136],[59,18701,18703],{"className":18702,"style":4302},[140],[59,18704,18705,18708],{"style":17212},[59,18706],{"className":18707,"style":213},[148],[59,18709,18711],{"className":18710},[153,217,218,156],[59,18712,335],{"className":18713},[119,192,156],[59,18715,227],{"className":18716},[226],[59,18718,18720],{"className":18719},[136],[59,18721,18723],{"className":18722,"style":4324},[140],[59,18724],{},[59,18726,227],{"className":18727},[226],[59,18729,18731],{"className":18730},[136],[59,18732,18734],{"className":18733,"style":18624},[140],[59,18735],{},[59,18737],{"className":18738,"style":916},[499],[59,18740,470],{"className":18741,"style":14109},[119,192],[59,18743,798],{"className":18744},[123],[59,18746,18748,18751],{"className":18747},[119],[59,18749,13170],{"className":18750,"style":13186},[119,192],[59,18752,18754],{"className":18753},[196],[59,18755,18757],{"className":18756},[131],[59,18758,18760],{"className":18759},[136],[59,18761,18763],{"className":18762,"style":16622},[140],[59,18764,18765,18768],{"style":536},[59,18766],{"className":18767,"style":540},[148],[59,18769,18771],{"className":18770},[153,154,155,156],[59,18772,335],{"className":18773},[119,192,156],[59,18775,812],{"className":18776},[313],[59,18778],{"className":18779,"style":916},[499],[59,18781,16850,18783],{"className":18782},[951],[59,18784,16854],{"style":16853},[59,18786],{"className":18787,"style":916},[499],[59,18789,18791,18794],{"className":18790},[119],[59,18792,5604],{"className":18793,"style":3185},[119,192],[59,18795,18797],{"className":18796},[196],[59,18798,18800,18820],{"className":18799},[131,132],[59,18801,18803,18817],{"className":18802},[136],[59,18804,18806],{"className":18805,"style":1248},[140],[59,18807,18808,18811],{"style":7460},[59,18809],{"className":18810,"style":540},[148],[59,18812,18814],{"className":18813},[153,154,155,156],[59,18815,14051],{"className":18816,"style":3210},[119,192,156],[59,18818,227],{"className":18819},[226],[59,18821,18823],{"className":18822},[136],[59,18824,18826],{"className":18825,"style":1270},[140],[59,18827],{},[59,18829,798],{"className":18830},[123],[59,18832,18834,18837],{"className":18833},[119],[59,18835,22],{"className":18836},[119,192],[59,18838,18840],{"className":18839},[196],[59,18841,18843,18875],{"className":18842},[131,132],[59,18844,18846,18872],{"className":18845},[136],[59,18847,18850,18861],{"className":18848,"style":18849},[140],"height:0.7936em;",[59,18851,18852,18855],{"style":17311},[59,18853],{"className":18854,"style":540},[148],[59,18856,18858],{"className":18857},[153,154,155,156],[59,18859,335],{"className":18860},[119,192,156],[59,18862,18863,18866],{"style":536},[59,18864],{"className":18865,"style":540},[148],[59,18867,18869],{"className":18868},[153,154,155,156],[59,18870,777],{"className":18871},[119,192,156],[59,18873,227],{"className":18874},[226],[59,18876,18878],{"className":18877},[136],[59,18879,18881],{"className":18880,"style":17341},[140],[59,18882],{},[59,18884,12941],{"className":18885},[119],[59,18887,18889,18892],{"className":18888},[119],[59,18890,12802],{"className":18891},[119,192],[59,18893,18895],{"className":18894},[196],[59,18896,18898,18929],{"className":18897},[131,132],[59,18899,18901,18926],{"className":18900},[136],[59,18902,18904,18915],{"className":18903,"style":18849},[140],[59,18905,18906,18909],{"style":17311},[59,18907],{"className":18908,"style":540},[148],[59,18910,18912],{"className":18911},[153,154,155,156],[59,18913,335],{"className":18914},[119,192,156],[59,18916,18917,18920],{"style":536},[59,18918],{"className":18919,"style":540},[148],[59,18921,18923],{"className":18922},[153,154,155,156],[59,18924,777],{"className":18925},[119,192,156],[59,18927,227],{"className":18928},[226],[59,18930,18932],{"className":18931},[136],[59,18933,18935],{"className":18934,"style":17341},[140],[59,18936],{},[59,18938,812],{"className":18939},[313],[15,18941,18942],{},[59,18943,18945,19033],{"className":18944},[62],[59,18946,18948],{"className":18947},[66],[68,18949,18950],{"xmlns":70},[72,18951,18952,19030],{},[75,18953,18954,18956,18958,18964,18966,18968,18991,19008,19020,19022],{},[91,18955,470],{},[461,18957,798],{"stretchy":797},[465,18959,18960,18962],{},[91,18961,13170],{},[91,18963,335],{},[461,18965,812],{"stretchy":797},[461,18967,815],{},[1822,18969,18970,18972,18985],{},[461,18971,1826],{},[75,18973,18974,18976,18978],{},[91,18975,777],{},[461,18977,815],{},[465,18979,18980,18982],{},[91,18981,777],{},[461,18983,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},"′",[88,18986,18987,18989],{},[91,18988,13242],{},[91,18990,335],{},[465,18992,18993,18996],{},[91,18994,18995],{},"γ",[75,18997,18998,19000,19002],{},[91,18999,777],{},[461,19001,18365],{},[465,19003,19004,19006],{},[91,19005,777],{},[461,19007,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[1822,19009,19010,19012,19018],{},[91,19011,7826],{},[465,19013,19014,19016],{},[91,19015,777],{},[461,19017,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[91,19019,335],{},[461,19021,815],{},[1822,19023,19024,19026,19028],{},[91,19025,470],{},[91,19027,777],{},[91,19029,335],{},[98,19031,19032],{"encoding":100},"R(\\tau^n) = \\sum\\limits_{t=t'}^{T_n} \\gamma^{t-t'} r_{t'}^n = R_t^n",[59,19034,19036,19089,19400],{"className":19035,"ariaHidden":106},[105],[59,19037,19039,19042,19045,19048,19077,19080,19083,19086],{"className":19038},[110],[59,19040],{"className":19041,"style":877},[114],[59,19043,470],{"className":19044,"style":14109},[119,192],[59,19046,798],{"className":19047},[123],[59,19049,19051,19054],{"className":19050},[119],[59,19052,13170],{"className":19053,"style":13186},[119,192],[59,19055,19057],{"className":19056},[196],[59,19058,19060],{"className":19059},[131],[59,19061,19063],{"className":19062},[136],[59,19064,19066],{"className":19065,"style":16622},[140],[59,19067,19068,19071],{"style":536},[59,19069],{"className":19070,"style":540},[148],[59,19072,19074],{"className":19073},[153,154,155,156],[59,19075,335],{"className":19076},[119,192,156],[59,19078,812],{"className":19079},[313],[59,19081],{"className":19082,"style":500},[499],[59,19084,815],{"className":19085},[504],[59,19087],{"className":19088,"style":500},[499],[59,19090,19092,19096,19233,19236,19306,19391,19394,19397],{"className":19091},[110],[59,19093],{"className":19094,"style":19095},[114],"height:2.5334em;vertical-align:-0.994em;",[59,19097,19099],{"className":19098},[951,14238],[59,19100,19102,19224],{"className":19101},[131,132],[59,19103,19105,19221],{"className":19104},[136],[59,19106,19108,19160,19170],{"className":19107,"style":18642},[140],[59,19109,19111,19114],{"style":19110},"top:-2.106em;margin-left:0em;",[59,19112],{"className":19113,"style":149},[148],[59,19115,19117],{"className":19116},[153,154,155,156],[59,19118,19120,19123,19126],{"className":19119},[119,156],[59,19121,777],{"className":19122},[119,192,156],[59,19124,815],{"className":19125},[504,156],[59,19127,19129,19132],{"className":19128},[119,156],[59,19130,777],{"className":19131},[119,192,156],[59,19133,19135],{"className":19134},[196],[59,19136,19138],{"className":19137},[131],[59,19139,19141],{"className":19140},[136],[59,19142,19145],{"className":19143,"style":19144},[140],"height:0.6828em;",[59,19146,19148,19151],{"style":19147},"top:-2.786em;margin-right:0.0714em;",[59,19149],{"className":19150,"style":213},[148],[59,19152,19154],{"className":19153},[153,217,218,156],[59,19155,19157],{"className":19156},[119,156],[59,19158,18984],{"className":19159},[119,156],[59,19161,19162,19165],{"style":178},[59,19163],{"className":19164,"style":149},[148],[59,19166,19167],{},[59,19168,1826],{"className":19169},[951,1910,1911],[59,19171,19172,19175],{"style":18675},[59,19173],{"className":19174,"style":149},[148],[59,19176,19178],{"className":19177},[153,154,155,156],[59,19179,19181],{"className":19180},[119,156],[59,19182,19184,19187],{"className":19183},[119,156],[59,19185,13242],{"className":19186,"style":3185},[119,192,156],[59,19188,19190],{"className":19189},[196],[59,19191,19193,19213],{"className":19192},[131,132],[59,19194,19196,19210],{"className":19195},[136],[59,19197,19199],{"className":19198,"style":4302},[140],[59,19200,19201,19204],{"style":17212},[59,19202],{"className":19203,"style":213},[148],[59,19205,19207],{"className":19206},[153,217,218,156],[59,19208,335],{"className":19209},[119,192,156],[59,19211,227],{"className":19212},[226],[59,19214,19216],{"className":19215},[136],[59,19217,19219],{"className":19218,"style":4324},[140],[59,19220],{},[59,19222,227],{"className":19223},[226],[59,19225,19227],{"className":19226},[136],[59,19228,19231],{"className":19229,"style":19230},[140],"height:0.994em;",[59,19232],{},[59,19234],{"className":19235,"style":916},[499],[59,19237,19239,19243],{"className":19238},[119],[59,19240,18995],{"className":19241,"style":19242},[119,192],"margin-right:0.05556em;",[59,19244,19246],{"className":19245},[196],[59,19247,19249],{"className":19248},[131],[59,19250,19252],{"className":19251},[136],[59,19253,19256],{"className":19254,"style":19255},[140],"height:0.9425em;",[59,19257,19258,19261],{"style":536},[59,19259],{"className":19260,"style":540},[148],[59,19262,19264],{"className":19263},[153,154,155,156],[59,19265,19267,19270,19273],{"className":19266},[119,156],[59,19268,777],{"className":19269},[119,192,156],[59,19271,18365],{"className":19272},[553,156],[59,19274,19276,19279],{"className":19275},[119,156],[59,19277,777],{"className":19278},[119,192,156],[59,19280,19282],{"className":19281},[196],[59,19283,19285],{"className":19284},[131],[59,19286,19288],{"className":19287},[136],[59,19289,19292],{"className":19290,"style":19291},[140],"height:0.8278em;",[59,19293,19294,19297],{"style":1098},[59,19295],{"className":19296,"style":213},[148],[59,19298,19300],{"className":19299},[153,217,218,156],[59,19301,19303],{"className":19302},[119,156],[59,19304,18984],{"className":19305},[119,156],[59,19307,19309,19312],{"className":19308},[119],[59,19310,7826],{"className":19311,"style":3210},[119,192],[59,19313,19315],{"className":19314},[196],[59,19316,19318,19382],{"className":19317},[131,132],[59,19319,19321,19379],{"className":19320},[136],[59,19322,19324,19368],{"className":19323,"style":16622},[140],[59,19325,19327,19330],{"style":19326},"top:-2.425em;margin-left:-0.0278em;margin-right:0.05em;",[59,19328],{"className":19329,"style":540},[148],[59,19331,19333],{"className":19332},[153,154,155,156],[59,19334,19336],{"className":19335},[119,156],[59,19337,19339,19342],{"className":19338},[119,156],[59,19340,777],{"className":19341},[119,192,156],[59,19343,19345],{"className":19344},[196],[59,19346,19348],{"className":19347},[131],[59,19349,19351],{"className":19350},[136],[59,19352,19354],{"className":19353,"style":19144},[140],[59,19355,19356,19359],{"style":19147},[59,19357],{"className":19358,"style":213},[148],[59,19360,19362],{"className":19361},[153,217,218,156],[59,19363,19365],{"className":19364},[119,156],[59,19366,18984],{"className":19367},[119,156],[59,19369,19370,19373],{"style":536},[59,19371],{"className":19372,"style":540},[148],[59,19374,19376],{"className":19375},[153,154,155,156],[59,19377,335],{"className":19378},[119,192,156],[59,19380,227],{"className":19381},[226],[59,19383,19385],{"className":19384},[136],[59,19386,19389],{"className":19387,"style":19388},[140],"height:0.275em;",[59,19390],{},[59,19392],{"className":19393,"style":500},[499],[59,19395,815],{"className":19396},[504],[59,19398],{"className":19399,"style":500},[499],[59,19401,19403,19407],{"className":19402},[110],[59,19404],{"className":19405,"style":19406},[114],"height:0.9303em;vertical-align:-0.247em;",[59,19408,19410,19413],{"className":19409},[119],[59,19411,470],{"className":19412,"style":14109},[119,192],[59,19414,19416],{"className":19415},[196],[59,19417,19419,19451],{"className":19418},[131,132],[59,19420,19422,19448],{"className":19421},[136],[59,19423,19425,19437],{"className":19424,"style":16622},[140],[59,19426,19428,19431],{"style":19427},"top:-2.453em;margin-left:-0.0077em;margin-right:0.05em;",[59,19429],{"className":19430,"style":540},[148],[59,19432,19434],{"className":19433},[153,154,155,156],[59,19435,777],{"className":19436},[119,192,156],[59,19438,19439,19442],{"style":536},[59,19440],{"className":19441,"style":540},[148],[59,19443,19445],{"className":19444},[153,154,155,156],[59,19446,335],{"className":19447},[119,192,156],[59,19449,227],{"className":19450},[226],[59,19452,19454],{"className":19453},[136],[59,19455,19457],{"className":19456,"style":17341},[140],[59,19458],{},[11,19460,19462],{"id":19461},"actor-critic-演员-评论家算法","Actor-Critic 演员-评论家算法",[15,19464,19465,19466,19537],{},"为了衡量每个动作的相对好坏的情况，我们令奖励 ",[59,19467,19469,19487],{"className":19468},[62],[59,19470,19472],{"className":19471},[66],[68,19473,19474],{"xmlns":70},[72,19475,19476,19484],{},[75,19477,19478],{},[88,19479,19480,19482],{},[91,19481,470],{},[91,19483,777],{},[98,19485,19486],{"encoding":100},"R_t",[59,19488,19490],{"className":19489,"ariaHidden":106},[105],[59,19491,19493,19496],{"className":19492},[110],[59,19494],{"className":19495,"style":6974},[114],[59,19497,19499,19502],{"className":19498},[119],[59,19500,470],{"className":19501,"style":14109},[119,192],[59,19503,19505],{"className":19504},[196],[59,19506,19508,19529],{"className":19507},[131,132],[59,19509,19511,19526],{"className":19510},[136],[59,19512,19514],{"className":19513,"style":12732},[140],[59,19515,19517,19520],{"style":19516},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[59,19518],{"className":19519,"style":540},[148],[59,19521,19523],{"className":19522},[153,154,155,156],[59,19524,777],{"className":19525},[119,192,156],[59,19527,227],{"className":19528},[226],[59,19530,19532],{"className":19531},[136],[59,19533,19535],{"className":19534,"style":1270},[140],[59,19536],{}," 减去一个基准值。",[15,19539,19540],{},[59,19541,19543,19656],{"className":19542},[62],[59,19544,19546],{"className":19545},[66],[68,19547,19548],{"xmlns":70},[72,19549,19550,19653],{},[75,19551,19552,19558,19572,19590,19592,19600,19602,19605,19607,19615,19617,19619,19621,19623,19625,19631,19633,19641,19643,19651],{},[78,19553,19554,19556],{},[81,19555,83],{},[91,19557,7322],{},[1822,19559,19560,19562,19570],{},[461,19561,1826],{},[75,19563,19564,19566,19568],{},[91,19565,335],{},[461,19567,815],{},[81,19569,83],{},[91,19571,7322],{},[1822,19573,19574,19576,19584],{},[461,19575,1826],{},[75,19577,19578,19580,19582],{},[91,19579,777],{},[461,19581,815],{},[81,19583,83],{},[88,19585,19586,19588],{},[91,19587,13242],{},[91,19589,335],{},[461,19591,798],{"stretchy":797},[1822,19593,19594,19596,19598],{},[91,19595,470],{},[91,19597,777],{},[91,19599,335],{},[461,19601,18365],{},[91,19603,19604],{},"B",[461,19606,798],{"stretchy":797},[1822,19608,19609,19611,19613],{},[91,19610,12802],{},[91,19612,777],{},[91,19614,335],{},[461,19616,812],{"stretchy":797},[461,19618,812],{"stretchy":797},[91,19620,14380],{"mathvariant":818},[91,19622,14805],{},[461,19624,822],{},[88,19626,19627,19629],{},[91,19628,5604],{},[91,19630,14051],{},[461,19632,798],{"stretchy":797},[1822,19634,19635,19637,19639],{},[91,19636,22],{},[91,19638,777],{},[91,19640,335],{},[91,19642,12941],{"mathvariant":818},[1822,19644,19645,19647,19649],{},[91,19646,12802],{},[91,19648,777],{},[91,19650,335],{},[461,19652,812],{"stretchy":797},[98,19654,19655],{"encoding":100}," \\frac{1}{N} \\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} (R_t^n - B(s_t^n))\\nabla \\log P_\\theta(a_t^n|s_t^n)",[59,19657,19659,19970],{"className":19658,"ariaHidden":106},[105],[59,19660,19662,19665,19733,19736,19800,19803,19907,19910,19961,19964,19967],{"className":19661},[110],[59,19663],{"className":19664,"style":18482},[114],[59,19666,19668,19671,19730],{"className":19667},[119],[59,19669],{"className":19670},[123,124],[59,19672,19674],{"className":19673},[78],[59,19675,19677,19722],{"className":19676},[131,132],[59,19678,19680,19719],{"className":19679},[136],[59,19681,19683,19697,19705],{"className":19682,"style":141},[140],[59,19684,19685,19688],{"style":18506},[59,19686],{"className":19687,"style":149},[148],[59,19689,19691],{"className":19690},[153,154,155,156],[59,19692,19694],{"className":19693},[119,156],[59,19695,7322],{"className":19696,"style":7397},[119,192,156],[59,19698,19699,19702],{"style":274},[59,19700],{"className":19701,"style":149},[148],[59,19703],{"className":19704,"style":282},[281],[59,19706,19707,19710],{"style":285},[59,19708],{"className":19709,"style":149},[148],[59,19711,19713],{"className":19712},[153,154,155,156],[59,19714,19716],{"className":19715},[119,156],[59,19717,83],{"className":19718},[119,156],[59,19720,227],{"className":19721},[226],[59,19723,19725],{"className":19724},[136],[59,19726,19728],{"className":19727,"style":18550},[140],[59,19729],{},[59,19731],{"className":19732},[313,124],[59,19734],{"className":19735,"style":916},[499],[59,19737,19739],{"className":19738},[951,14238],[59,19740,19742,19792],{"className":19741},[131,132],[59,19743,19745,19789],{"className":19744},[136],[59,19746,19748,19768,19778],{"className":19747,"style":18571},[140],[59,19749,19750,19753],{"style":18574},[59,19751],{"className":19752,"style":149},[148],[59,19754,19756],{"className":19755},[153,154,155,156],[59,19757,19759,19762,19765],{"className":19758},[119,156],[59,19760,335],{"className":19761},[119,192,156],[59,19763,815],{"className":19764},[504,156],[59,19766,83],{"className":19767},[119,156],[59,19769,19770,19773],{"style":178},[59,19771],{"className":19772,"style":149},[148],[59,19774,19775],{},[59,19776,1826],{"className":19777},[951,1910,1911],[59,19779,19780,19783],{"style":18605},[59,19781],{"className":19782,"style":149},[148],[59,19784,19786],{"className":19785},[153,154,155,156],[59,19787,7322],{"className":19788,"style":7397},[119,192,156],[59,19790,227],{"className":19791},[226],[59,19793,19795],{"className":19794},[136],[59,19796,19798],{"className":19797,"style":18624},[140],[59,19799],{},[59,19801],{"className":19802,"style":916},[499],[59,19804,19806],{"className":19805},[951,14238],[59,19807,19809,19899],{"className":19808},[131,132],[59,19810,19812,19896],{"className":19811},[136],[59,19813,19815,19835,19845],{"className":19814,"style":18642},[140],[59,19816,19817,19820],{"style":18574},[59,19818],{"className":19819,"style":149},[148],[59,19821,19823],{"className":19822},[153,154,155,156],[59,19824,19826,19829,19832],{"className":19825},[119,156],[59,19827,777],{"className":19828},[119,192,156],[59,19830,815],{"className":19831},[504,156],[59,19833,83],{"className":19834},[119,156],[59,19836,19837,19840],{"style":178},[59,19838],{"className":19839,"style":149},[148],[59,19841,19842],{},[59,19843,1826],{"className":19844},[951,1910,1911],[59,19846,19847,19850],{"style":18675},[59,19848],{"className":19849,"style":149},[148],[59,19851,19853],{"className":19852},[153,154,155,156],[59,19854,19856],{"className":19855},[119,156],[59,19857,19859,19862],{"className":19858},[119,156],[59,19860,13242],{"className":19861,"style":3185},[119,192,156],[59,19863,19865],{"className":19864},[196],[59,19866,19868,19888],{"className":19867},[131,132],[59,19869,19871,19885],{"className":19870},[136],[59,19872,19874],{"className":19873,"style":4302},[140],[59,19875,19876,19879],{"style":17212},[59,19877],{"className":19878,"style":213},[148],[59,19880,19882],{"className":19881},[153,217,218,156],[59,19883,335],{"className":19884},[119,192,156],[59,19886,227],{"className":19887},[226],[59,19889,19891],{"className":19890},[136],[59,19892,19894],{"className":19893,"style":4324},[140],[59,19895],{},[59,19897,227],{"className":19898},[226],[59,19900,19902],{"className":19901},[136],[59,19903,19905],{"className":19904,"style":18624},[140],[59,19906],{},[59,19908,798],{"className":19909},[123],[59,19911,19913,19916],{"className":19912},[119],[59,19914,470],{"className":19915,"style":14109},[119,192],[59,19917,19919],{"className":19918},[196],[59,19920,19922,19953],{"className":19921},[131,132],[59,19923,19925,19950],{"className":19924},[136],[59,19926,19928,19939],{"className":19927,"style":16622},[140],[59,19929,19930,19933],{"style":19427},[59,19931],{"className":19932,"style":540},[148],[59,19934,19936],{"className":19935},[153,154,155,156],[59,19937,777],{"className":19938},[119,192,156],[59,19940,19941,19944],{"style":536},[59,19942],{"className":19943,"style":540},[148],[59,19945,19947],{"className":19946},[153,154,155,156],[59,19948,335],{"className":19949},[119,192,156],[59,19951,227],{"className":19952},[226],[59,19954,19956],{"className":19955},[136],[59,19957,19959],{"className":19958,"style":17341},[140],[59,19960],{},[59,19962],{"className":19963,"style":1872},[499],[59,19965,18365],{"className":19966},[553],[59,19968],{"className":19969,"style":1872},[499],[59,19971,19973,19976,19980,19983,20034,20037,20040,20043,20048,20051,20091,20094,20145,20148,20199],{"className":19972},[110],[59,19974],{"className":19975,"style":877},[114],[59,19977,19604],{"className":19978,"style":19979},[119,192],"margin-right:0.05017em;",[59,19981,798],{"className":19982},[123],[59,19984,19986,19989],{"className":19985},[119],[59,19987,12802],{"className":19988},[119,192],[59,19990,19992],{"className":19991},[196],[59,19993,19995,20026],{"className":19994},[131,132],[59,19996,19998,20023],{"className":19997},[136],[59,19999,20001,20012],{"className":20000,"style":16622},[140],[59,20002,20003,20006],{"style":17311},[59,20004],{"className":20005,"style":540},[148],[59,20007,20009],{"className":20008},[153,154,155,156],[59,20010,777],{"className":20011},[119,192,156],[59,20013,20014,20017],{"style":536},[59,20015],{"className":20016,"style":540},[148],[59,20018,20020],{"className":20019},[153,154,155,156],[59,20021,335],{"className":20022},[119,192,156],[59,20024,227],{"className":20025},[226],[59,20027,20029],{"className":20028},[136],[59,20030,20032],{"className":20031,"style":17341},[140],[59,20033],{},[59,20035,7924],{"className":20036},[313],[59,20038,14380],{"className":20039},[119],[59,20041],{"className":20042,"style":916},[499],[59,20044,16850,20046],{"className":20045},[951],[59,20047,16854],{"style":16853},[59,20049],{"className":20050,"style":916},[499],[59,20052,20054,20057],{"className":20053},[119],[59,20055,5604],{"className":20056,"style":3185},[119,192],[59,20058,20060],{"className":20059},[196],[59,20061,20063,20083],{"className":20062},[131,132],[59,20064,20066,20080],{"className":20065},[136],[59,20067,20069],{"className":20068,"style":1248},[140],[59,20070,20071,20074],{"style":7460},[59,20072],{"className":20073,"style":540},[148],[59,20075,20077],{"className":20076},[153,154,155,156],[59,20078,14051],{"className":20079,"style":3210},[119,192,156],[59,20081,227],{"className":20082},[226],[59,20084,20086],{"className":20085},[136],[59,20087,20089],{"className":20088,"style":1270},[140],[59,20090],{},[59,20092,798],{"className":20093},[123],[59,20095,20097,20100],{"className":20096},[119],[59,20098,22],{"className":20099},[119,192],[59,20101,20103],{"className":20102},[196],[59,20104,20106,20137],{"className":20105},[131,132],[59,20107,20109,20134],{"className":20108},[136],[59,20110,20112,20123],{"className":20111,"style":16622},[140],[59,20113,20114,20117],{"style":17311},[59,20115],{"className":20116,"style":540},[148],[59,20118,20120],{"className":20119},[153,154,155,156],[59,20121,777],{"className":20122},[119,192,156],[59,20124,20125,20128],{"style":536},[59,20126],{"className":20127,"style":540},[148],[59,20129,20131],{"className":20130},[153,154,155,156],[59,20132,335],{"className":20133},[119,192,156],[59,20135,227],{"className":20136},[226],[59,20138,20140],{"className":20139},[136],[59,20141,20143],{"className":20142,"style":17341},[140],[59,20144],{},[59,20146,12941],{"className":20147},[119],[59,20149,20151,20154],{"className":20150},[119],[59,20152,12802],{"className":20153},[119,192],[59,20155,20157],{"className":20156},[196],[59,20158,20160,20191],{"className":20159},[131,132],[59,20161,20163,20188],{"className":20162},[136],[59,20164,20166,20177],{"className":20165,"style":16622},[140],[59,20167,20168,20171],{"style":17311},[59,20169],{"className":20170,"style":540},[148],[59,20172,20174],{"className":20173},[153,154,155,156],[59,20175,777],{"className":20176},[119,192,156],[59,20178,20179,20182],{"style":536},[59,20180],{"className":20181,"style":540},[148],[59,20183,20185],{"className":20184},[153,154,155,156],[59,20186,335],{"className":20187},[119,192,156],[59,20189,227],{"className":20190},[226],[59,20192,20194],{"className":20193},[136],[59,20195,20197],{"className":20196,"style":17341},[140],[59,20198],{},[59,20200,812],{"className":20201},[313],[15,20203,20204,20205,20288,20289,20387,20388,20416,20417,20445],{},"但是，",[59,20206,20208,20228],{"className":20207},[62],[59,20209,20211],{"className":20210},[66],[68,20212,20213],{"xmlns":70},[72,20214,20215,20225],{},[75,20216,20217],{},[1822,20218,20219,20221,20223],{},[91,20220,470],{},[91,20222,777],{},[91,20224,335],{},[98,20226,20227],{"encoding":100},"R_t^n",[59,20229,20231],{"className":20230,"ariaHidden":106},[105],[59,20232,20234,20237],{"className":20233},[110],[59,20235],{"className":20236,"style":19406},[114],[59,20238,20240,20243],{"className":20239},[119],[59,20241,470],{"className":20242,"style":14109},[119,192],[59,20244,20246],{"className":20245},[196],[59,20247,20249,20280],{"className":20248},[131,132],[59,20250,20252,20277],{"className":20251},[136],[59,20253,20255,20266],{"className":20254,"style":16622},[140],[59,20256,20257,20260],{"style":19427},[59,20258],{"className":20259,"style":540},[148],[59,20261,20263],{"className":20262},[153,154,155,156],[59,20264,777],{"className":20265},[119,192,156],[59,20267,20268,20271],{"style":536},[59,20269],{"className":20270,"style":540},[148],[59,20272,20274],{"className":20273},[153,154,155,156],[59,20275,335],{"className":20276},[119,192,156],[59,20278,227],{"className":20279},[226],[59,20281,20283],{"className":20282},[136],[59,20284,20286],{"className":20285,"style":17341},[140],[59,20287],{}," 都是一次随机采样，方差很大，训练不稳定。我们引入 ",[59,20290,20292,20320],{"className":20291},[62],[59,20293,20295],{"className":20294},[66],[68,20296,20297],{"xmlns":70},[72,20298,20299,20317],{},[75,20300,20301,20307,20309,20311,20313,20315],{},[88,20302,20303,20305],{},[91,20304,459],{},[91,20306,14051],{},[461,20308,798],{"stretchy":797},[91,20310,12802],{},[461,20312,803],{"separator":106},[91,20314,22],{},[461,20316,812],{"stretchy":797},[98,20318,20319],{"encoding":100},"Q_\\theta(s,a)",[59,20321,20323],{"className":20322,"ariaHidden":106},[105],[59,20324,20326,20329,20369,20372,20375,20378,20381,20384],{"className":20325},[110],[59,20327],{"className":20328,"style":877},[114],[59,20330,20332,20335],{"className":20331},[119],[59,20333,459],{"className":20334},[119,192],[59,20336,20338],{"className":20337},[196],[59,20339,20341,20361],{"className":20340},[131,132],[59,20342,20344,20358],{"className":20343},[136],[59,20345,20347],{"className":20346,"style":1248},[140],[59,20348,20349,20352],{"style":1251},[59,20350],{"className":20351,"style":540},[148],[59,20353,20355],{"className":20354},[153,154,155,156],[59,20356,14051],{"className":20357,"style":3210},[119,192,156],[59,20359,227],{"className":20360},[226],[59,20362,20364],{"className":20363},[136],[59,20365,20367],{"className":20366,"style":1270},[140],[59,20368],{},[59,20370,798],{"className":20371},[123],[59,20373,12802],{"className":20374},[119,192],[59,20376,803],{"className":20377},[912],[59,20379],{"className":20380,"style":916},[499],[59,20382,22],{"className":20383},[119,192],[59,20385,812],{"className":20386},[313],"，表示在状态 ",[59,20389,20391,20404],{"className":20390},[62],[59,20392,20394],{"className":20393},[66],[68,20395,20396],{"xmlns":70},[72,20397,20398,20402],{},[75,20399,20400],{},[91,20401,12802],{},[98,20403,12802],{"encoding":100},[59,20405,20407],{"className":20406,"ariaHidden":106},[105],[59,20408,20410,20413],{"className":20409},[110],[59,20411],{"className":20412,"style":347},[114],[59,20414,12802],{"className":20415},[119,192],"，下，做出动作 ",[59,20418,20420,20433],{"className":20419},[62],[59,20421,20423],{"className":20422},[66],[68,20424,20425],{"xmlns":70},[72,20426,20427,20431],{},[75,20428,20429],{},[91,20430,22],{},[98,20432,22],{"encoding":100},[59,20434,20436],{"className":20435,"ariaHidden":106},[105],[59,20437,20439,20442],{"className":20438},[110],[59,20440],{"className":20441,"style":347},[114],[59,20443,22],{"className":20444},[119,192]," 后期望的回报，即动作价值函数。",[15,20447,20448,20449,20535,20536,20564],{},"令 ",[59,20450,20452,20476],{"className":20451},[62],[59,20453,20455],{"className":20454},[66],[68,20456,20457],{"xmlns":70},[72,20458,20459,20473],{},[75,20460,20461,20467,20469,20471],{},[88,20462,20463,20465],{},[91,20464,672],{},[91,20466,14051],{},[461,20468,798],{"stretchy":797},[91,20470,12802],{},[461,20472,812],{"stretchy":797},[98,20474,20475],{"encoding":100},"V_\\theta(s)",[59,20477,20479],{"className":20478,"ariaHidden":106},[105],[59,20480,20482,20485,20526,20529,20532],{"className":20481},[110],[59,20483],{"className":20484,"style":877},[114],[59,20486,20488,20491],{"className":20487},[119],[59,20489,672],{"className":20490,"style":702},[119,192],[59,20492,20494],{"className":20493},[196],[59,20495,20497,20518],{"className":20496},[131,132],[59,20498,20500,20515],{"className":20499},[136],[59,20501,20503],{"className":20502,"style":1248},[140],[59,20504,20506,20509],{"style":20505},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[59,20507],{"className":20508,"style":540},[148],[59,20510,20512],{"className":20511},[153,154,155,156],[59,20513,14051],{"className":20514,"style":3210},[119,192,156],[59,20516,227],{"className":20517},[226],[59,20519,20521],{"className":20520},[136],[59,20522,20524],{"className":20523,"style":1270},[140],[59,20525],{},[59,20527,798],{"className":20528},[123],[59,20530,12802],{"className":20531},[119,192],[59,20533,812],{"className":20534},[313]," 在状态 ",[59,20537,20539,20552],{"className":20538},[62],[59,20540,20542],{"className":20541},[66],[68,20543,20544],{"xmlns":70},[72,20545,20546,20550],{},[75,20547,20548],{},[91,20549,12802],{},[98,20551,12802],{"encoding":100},[59,20553,20555],{"className":20554,"ariaHidden":106},[105],[59,20556,20558,20561],{"className":20557},[110],[59,20559],{"className":20560,"style":347},[114],[59,20562,12802],{"className":20563},[119,192]," 下期望的回报，即状态价值函数",[15,20566,20567,20834,20835,20863,20864,20892],{},[59,20568,20570,20630],{"className":20569},[62],[59,20571,20573],{"className":20572},[66],[68,20574,20575],{"xmlns":70},[72,20576,20577,20627],{},[75,20578,20579,20585,20587,20589,20591,20593,20595,20597,20603,20605,20607,20609,20611,20613,20615,20621,20623,20625],{},[88,20580,20581,20583],{},[91,20582,774],{},[91,20584,14051],{},[461,20586,798],{"stretchy":797},[91,20588,12802],{},[461,20590,803],{"separator":106},[91,20592,22],{},[461,20594,812],{"stretchy":797},[461,20596,815],{},[88,20598,20599,20601],{},[91,20600,459],{},[91,20602,14051],{},[461,20604,798],{"stretchy":797},[91,20606,12802],{},[461,20608,803],{"separator":106},[91,20610,22],{},[461,20612,812],{"stretchy":797},[461,20614,18365],{},[88,20616,20617,20619],{},[91,20618,672],{},[91,20620,14051],{},[461,20622,798],{"stretchy":797},[91,20624,12802],{},[461,20626,812],{"stretchy":797},[98,20628,20629],{"encoding":100},"A_\\theta(s,a) = Q_\\theta(s,a) - V_\\theta(s)",[59,20631,20633,20706,20779],{"className":20632,"ariaHidden":106},[105],[59,20634,20636,20639,20679,20682,20685,20688,20691,20694,20697,20700,20703],{"className":20635},[110],[59,20637],{"className":20638,"style":877},[114],[59,20640,20642,20645],{"className":20641},[119],[59,20643,774],{"className":20644},[119,192],[59,20646,20648],{"className":20647},[196],[59,20649,20651,20671],{"className":20650},[131,132],[59,20652,20654,20668],{"className":20653},[136],[59,20655,20657],{"className":20656,"style":1248},[140],[59,20658,20659,20662],{"style":1251},[59,20660],{"className":20661,"style":540},[148],[59,20663,20665],{"className":20664},[153,154,155,156],[59,20666,14051],{"className":20667,"style":3210},[119,192,156],[59,20669,227],{"className":20670},[226],[59,20672,20674],{"className":20673},[136],[59,20675,20677],{"className":20676,"style":1270},[140],[59,20678],{},[59,20680,798],{"className":20681},[123],[59,20683,12802],{"className":20684},[119,192],[59,20686,803],{"className":20687},[912],[59,20689],{"className":20690,"style":916},[499],[59,20692,22],{"className":20693},[119,192],[59,20695,812],{"className":20696},[313],[59,20698],{"className":20699,"style":500},[499],[59,20701,815],{"className":20702},[504],[59,20704],{"className":20705,"style":500},[499],[59,20707,20709,20712,20752,20755,20758,20761,20764,20767,20770,20773,20776],{"className":20708},[110],[59,20710],{"className":20711,"style":877},[114],[59,20713,20715,20718],{"className":20714},[119],[59,20716,459],{"className":20717},[119,192],[59,20719,20721],{"className":20720},[196],[59,20722,20724,20744],{"className":20723},[131,132],[59,20725,20727,20741],{"className":20726},[136],[59,20728,20730],{"className":20729,"style":1248},[140],[59,20731,20732,20735],{"style":1251},[59,20733],{"className":20734,"style":540},[148],[59,20736,20738],{"className":20737},[153,154,155,156],[59,20739,14051],{"className":20740,"style":3210},[119,192,156],[59,20742,227],{"className":20743},[226],[59,20745,20747],{"className":20746},[136],[59,20748,20750],{"className":20749,"style":1270},[140],[59,20751],{},[59,20753,798],{"className":20754},[123],[59,20756,12802],{"className":20757},[119,192],[59,20759,803],{"className":20760},[912],[59,20762],{"className":20763,"style":916},[499],[59,20765,22],{"className":20766},[119,192],[59,20768,812],{"className":20769},[313],[59,20771],{"className":20772,"style":1872},[499],[59,20774,18365],{"className":20775},[553],[59,20777],{"className":20778,"style":1872},[499],[59,20780,20782,20785,20825,20828,20831],{"className":20781},[110],[59,20783],{"className":20784,"style":877},[114],[59,20786,20788,20791],{"className":20787},[119],[59,20789,672],{"className":20790,"style":702},[119,192],[59,20792,20794],{"className":20793},[196],[59,20795,20797,20817],{"className":20796},[131,132],[59,20798,20800,20814],{"className":20799},[136],[59,20801,20803],{"className":20802,"style":1248},[140],[59,20804,20805,20808],{"style":20505},[59,20806],{"className":20807,"style":540},[148],[59,20809,20811],{"className":20810},[153,154,155,156],[59,20812,14051],{"className":20813,"style":3210},[119,192,156],[59,20815,227],{"className":20816},[226],[59,20818,20820],{"className":20819},[136],[59,20821,20823],{"className":20822,"style":1270},[140],[59,20824],{},[59,20826,798],{"className":20827},[123],[59,20829,12802],{"className":20830},[119,192],[59,20832,812],{"className":20833},[313]," 表示优势函数（Advantage Function），衡量在状态 ",[59,20836,20838,20851],{"className":20837},[62],[59,20839,20841],{"className":20840},[66],[68,20842,20843],{"xmlns":70},[72,20844,20845,20849],{},[75,20846,20847],{},[91,20848,12802],{},[98,20850,12802],{"encoding":100},[59,20852,20854],{"className":20853,"ariaHidden":106},[105],[59,20855,20857,20860],{"className":20856},[110],[59,20858],{"className":20859,"style":347},[114],[59,20861,12802],{"className":20862},[119,192]," 下，动作 ",[59,20865,20867,20880],{"className":20866},[62],[59,20868,20870],{"className":20869},[66],[68,20871,20872],{"xmlns":70},[72,20873,20874,20878],{},[75,20875,20876],{},[91,20877,22],{},[98,20879,22],{"encoding":100},[59,20881,20883],{"className":20882,"ariaHidden":106},[105],[59,20884,20886,20889],{"className":20885},[110],[59,20887],{"className":20888,"style":347},[114],[59,20890,22],{"className":20891},[119,192]," 相对于平均水平的优势。",[15,20894,20895],{},"讲上述的公式改写为",[15,20897,20898],{},[59,20899,20901,21015],{"className":20900},[62],[59,20902,20904],{"className":20903},[66],[68,20905,20906],{"xmlns":70},[72,20907,20908,21012],{},[75,20909,20910,20918,20932,20950,20956,20958,20966,20968,20976,20978,20980,20982,20984,20990,20992,21000,21002,21010],{},[14374,20911,20912],{"scriptlevel":1754,"displaystyle":106},[78,20913,20914,20916],{},[81,20915,83],{},[91,20917,7322],{},[1822,20919,20920,20922,20930],{},[461,20921,1826],{},[75,20923,20924,20926,20928],{},[91,20925,335],{},[461,20927,815],{},[81,20929,83],{},[91,20931,7322],{},[1822,20933,20934,20936,20944],{},[461,20935,1826],{},[75,20937,20938,20940,20942],{},[91,20939,777],{},[461,20941,815],{},[81,20943,83],{},[88,20945,20946,20948],{},[91,20947,13242],{},[91,20949,335],{},[88,20951,20952,20954],{},[91,20953,774],{},[91,20955,14051],{},[461,20957,798],{"stretchy":797},[1822,20959,20960,20962,20964],{},[91,20961,12802],{},[91,20963,777],{},[91,20965,335],{},[461,20967,803],{"separator":106},[1822,20969,20970,20972,20974],{},[91,20971,22],{},[91,20973,777],{},[91,20975,335],{},[461,20977,812],{"stretchy":797},[91,20979,14380],{"mathvariant":818},[91,20981,14805],{},[461,20983,822],{},[88,20985,20986,20988],{},[91,20987,5604],{},[91,20989,14051],{},[461,20991,798],{"stretchy":797},[1822,20993,20994,20996,20998],{},[91,20995,22],{},[91,20997,777],{},[91,20999,335],{},[91,21001,12941],{"mathvariant":818},[1822,21003,21004,21006,21008],{},[91,21005,12802],{},[91,21007,777],{},[91,21009,335],{},[461,21011,812],{"stretchy":797},[98,21013,21014],{"encoding":100},"\\dfrac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} A_\\theta(s_t^n,a_t^n)\\nabla \\log P_\\theta(a_t^n|s_t^n)",[59,21016,21018],{"className":21017,"ariaHidden":106},[105],[59,21019,21021,21024,21086,21089,21153,21156,21260,21263,21303,21306,21357,21360,21363,21414,21417,21420,21423,21428,21431,21471,21474,21525,21528,21579],{"className":21020},[110],[59,21022],{"className":21023,"style":18482},[114],[59,21025,21027,21030,21083],{"className":21026},[119],[59,21028],{"className":21029},[123,124],[59,21031,21033],{"className":21032},[78],[59,21034,21036,21075],{"className":21035},[131,132],[59,21037,21039,21072],{"className":21038},[136],[59,21040,21042,21053,21061],{"className":21041,"style":16279},[140],[59,21043,21044,21047],{"style":15819},[59,21045],{"className":21046,"style":149},[148],[59,21048,21050],{"className":21049},[119],[59,21051,7322],{"className":21052,"style":7397},[119,192],[59,21054,21055,21058],{"style":274},[59,21056],{"className":21057,"style":149},[148],[59,21059],{"className":21060,"style":282},[281],[59,21062,21063,21066],{"style":15885},[59,21064],{"className":21065,"style":149},[148],[59,21067,21069],{"className":21068},[119],[59,21070,83],{"className":21071},[119],[59,21073,227],{"className":21074},[226],[59,21076,21078],{"className":21077},[136],[59,21079,21081],{"className":21080,"style":16319},[140],[59,21082],{},[59,21084],{"className":21085},[313,124],[59,21087],{"className":21088,"style":916},[499],[59,21090,21092],{"className":21091},[951,14238],[59,21093,21095,21145],{"className":21094},[131,132],[59,21096,21098,21142],{"className":21097},[136],[59,21099,21101,21121,21131],{"className":21100,"style":18571},[140],[59,21102,21103,21106],{"style":18574},[59,21104],{"className":21105,"style":149},[148],[59,21107,21109],{"className":21108},[153,154,155,156],[59,21110,21112,21115,21118],{"className":21111},[119,156],[59,21113,335],{"className":21114},[119,192,156],[59,21116,815],{"className":21117},[504,156],[59,21119,83],{"className":21120},[119,156],[59,21122,21123,21126],{"style":178},[59,21124],{"className":21125,"style":149},[148],[59,21127,21128],{},[59,21129,1826],{"className":21130},[951,1910,1911],[59,21132,21133,21136],{"style":18605},[59,21134],{"className":21135,"style":149},[148],[59,21137,21139],{"className":21138},[153,154,155,156],[59,21140,7322],{"className":21141,"style":7397},[119,192,156],[59,21143,227],{"className":21144},[226],[59,21146,21148],{"className":21147},[136],[59,21149,21151],{"className":21150,"style":18624},[140],[59,21152],{},[59,21154],{"className":21155,"style":916},[499],[59,21157,21159],{"className":21158},[951,14238],[59,21160,21162,21252],{"className":21161},[131,132],[59,21163,21165,21249],{"className":21164},[136],[59,21166,21168,21188,21198],{"className":21167,"style":18642},[140],[59,21169,21170,21173],{"style":18574},[59,21171],{"className":21172,"style":149},[148],[59,21174,21176],{"className":21175},[153,154,155,156],[59,21177,21179,21182,21185],{"className":21178},[119,156],[59,21180,777],{"className":21181},[119,192,156],[59,21183,815],{"className":21184},[504,156],[59,21186,83],{"className":21187},[119,156],[59,21189,21190,21193],{"style":178},[59,21191],{"className":21192,"style":149},[148],[59,21194,21195],{},[59,21196,1826],{"className":21197},[951,1910,1911],[59,21199,21200,21203],{"style":18675},[59,21201],{"className":21202,"style":149},[148],[59,21204,21206],{"className":21205},[153,154,155,156],[59,21207,21209],{"className":21208},[119,156],[59,21210,21212,21215],{"className":21211},[119,156],[59,21213,13242],{"className":21214,"style":3185},[119,192,156],[59,21216,21218],{"className":21217},[196],[59,21219,21221,21241],{"className":21220},[131,132],[59,21222,21224,21238],{"className":21223},[136],[59,21225,21227],{"className":21226,"style":4302},[140],[59,21228,21229,21232],{"style":17212},[59,21230],{"className":21231,"style":213},[148],[59,21233,21235],{"className":21234},[153,217,218,156],[59,21236,335],{"className":21237},[119,192,156],[59,21239,227],{"className":21240},[226],[59,21242,21244],{"className":21243},[136],[59,21245,21247],{"className":21246,"style":4324},[140],[59,21248],{},[59,21250,227],{"className":21251},[226],[59,21253,21255],{"className":21254},[136],[59,21256,21258],{"className":21257,"style":18624},[140],[59,21259],{},[59,21261],{"className":21262,"style":916},[499],[59,21264,21266,21269],{"className":21265},[119],[59,21267,774],{"className":21268},[119,192],[59,21270,21272],{"className":21271},[196],[59,21273,21275,21295],{"className":21274},[131,132],[59,21276,21278,21292],{"className":21277},[136],[59,21279,21281],{"className":21280,"style":1248},[140],[59,21282,21283,21286],{"style":1251},[59,21284],{"className":21285,"style":540},[148],[59,21287,21289],{"className":21288},[153,154,155,156],[59,21290,14051],{"className":21291,"style":3210},[119,192,156],[59,21293,227],{"className":21294},[226],[59,21296,21298],{"className":21297},[136],[59,21299,21301],{"className":21300,"style":1270},[140],[59,21302],{},[59,21304,798],{"className":21305},[123],[59,21307,21309,21312],{"className":21308},[119],[59,21310,12802],{"className":21311},[119,192],[59,21313,21315],{"className":21314},[196],[59,21316,21318,21349],{"className":21317},[131,132],[59,21319,21321,21346],{"className":21320},[136],[59,21322,21324,21335],{"className":21323,"style":16622},[140],[59,21325,21326,21329],{"style":17311},[59,21327],{"className":21328,"style":540},[148],[59,21330,21332],{"className":21331},[153,154,155,156],[59,21333,777],{"className":21334},[119,192,156],[59,21336,21337,21340],{"style":536},[59,21338],{"className":21339,"style":540},[148],[59,21341,21343],{"className":21342},[153,154,155,156],[59,21344,335],{"className":21345},[119,192,156],[59,21347,227],{"className":21348},[226],[59,21350,21352],{"className":21351},[136],[59,21353,21355],{"className":21354,"style":17341},[140],[59,21356],{},[59,21358,803],{"className":21359},[912],[59,21361],{"className":21362,"style":916},[499],[59,21364,21366,21369],{"className":21365},[119],[59,21367,22],{"className":21368},[119,192],[59,21370,21372],{"className":21371},[196],[59,21373,21375,21406],{"className":21374},[131,132],[59,21376,21378,21403],{"className":21377},[136],[59,21379,21381,21392],{"className":21380,"style":16622},[140],[59,21382,21383,21386],{"style":17311},[59,21384],{"className":21385,"style":540},[148],[59,21387,21389],{"className":21388},[153,154,155,156],[59,21390,777],{"className":21391},[119,192,156],[59,21393,21394,21397],{"style":536},[59,21395],{"className":21396,"style":540},[148],[59,21398,21400],{"className":21399},[153,154,155,156],[59,21401,335],{"className":21402},[119,192,156],[59,21404,227],{"className":21405},[226],[59,21407,21409],{"className":21408},[136],[59,21410,21412],{"className":21411,"style":17341},[140],[59,21413],{},[59,21415,812],{"className":21416},[313],[59,21418,14380],{"className":21419},[119],[59,21421],{"className":21422,"style":916},[499],[59,21424,16850,21426],{"className":21425},[951],[59,21427,16854],{"style":16853},[59,21429],{"className":21430,"style":916},[499],[59,21432,21434,21437],{"className":21433},[119],[59,21435,5604],{"className":21436,"style":3185},[119,192],[59,21438,21440],{"className":21439},[196],[59,21441,21443,21463],{"className":21442},[131,132],[59,21444,21446,21460],{"className":21445},[136],[59,21447,21449],{"className":21448,"style":1248},[140],[59,21450,21451,21454],{"style":7460},[59,21452],{"className":21453,"style":540},[148],[59,21455,21457],{"className":21456},[153,154,155,156],[59,21458,14051],{"className":21459,"style":3210},[119,192,156],[59,21461,227],{"className":21462},[226],[59,21464,21466],{"className":21465},[136],[59,21467,21469],{"className":21468,"style":1270},[140],[59,21470],{},[59,21472,798],{"className":21473},[123],[59,21475,21477,21480],{"className":21476},[119],[59,21478,22],{"className":21479},[119,192],[59,21481,21483],{"className":21482},[196],[59,21484,21486,21517],{"className":21485},[131,132],[59,21487,21489,21514],{"className":21488},[136],[59,21490,21492,21503],{"className":21491,"style":16622},[140],[59,21493,21494,21497],{"style":17311},[59,21495],{"className":21496,"style":540},[148],[59,21498,21500],{"className":21499},[153,154,155,156],[59,21501,777],{"className":21502},[119,192,156],[59,21504,21505,21508],{"style":536},[59,21506],{"className":21507,"style":540},[148],[59,21509,21511],{"className":21510},[153,154,155,156],[59,21512,335],{"className":21513},[119,192,156],[59,21515,227],{"className":21516},[226],[59,21518,21520],{"className":21519},[136],[59,21521,21523],{"className":21522,"style":17341},[140],[59,21524],{},[59,21526,12941],{"className":21527},[119],[59,21529,21531,21534],{"className":21530},[119],[59,21532,12802],{"className":21533},[119,192],[59,21535,21537],{"className":21536},[196],[59,21538,21540,21571],{"className":21539},[131,132],[59,21541,21543,21568],{"className":21542},[136],[59,21544,21546,21557],{"className":21545,"style":16622},[140],[59,21547,21548,21551],{"style":17311},[59,21549],{"className":21550,"style":540},[148],[59,21552,21554],{"className":21553},[153,154,155,156],[59,21555,777],{"className":21556},[119,192,156],[59,21558,21559,21562],{"style":536},[59,21560],{"className":21561,"style":540},[148],[59,21563,21565],{"className":21564},[153,154,155,156],[59,21566,335],{"className":21567},[119,192,156],[59,21569,227],{"className":21570},[226],[59,21572,21574],{"className":21573},[136],[59,21575,21577],{"className":21576,"style":17341},[140],[59,21578],{},[59,21580,812],{"className":21581},[313],[59,21583,21585],{"className":21584},[14350],[59,21586,21588,21793],{"className":21587},[62],[59,21589,21591],{"className":21590},[66],[68,21592,21593],{"xmlns":70,"display":9481},[72,21594,21595,21790],{},[75,21596,21597,21603,21605,21611,21613,21615,21617,21619,21625,21627,21629,21635,21637,21649,21651,21654,21660,21662,21668,21670,21672,21674,21676,21682,21684,21686,21692,21694,21706,21708,21710,21716,21718,21724,21726,21728,21734,21736,21748,21750,21752,21764,21766,21768,21774,21776,21788],{},[88,21598,21599,21601],{},[91,21600,459],{},[91,21602,14051],{},[461,21604,798],{"stretchy":797},[88,21606,21607,21609],{},[91,21608,12802],{},[91,21610,777],{},[461,21612,803],{"separator":106},[91,21614,22],{},[461,21616,812],{"stretchy":797},[461,21618,815],{},[88,21620,21621,21623],{},[91,21622,7826],{},[91,21624,777],{},[461,21626,5777],{},[91,21628,18995],{},[88,21630,21631,21633],{},[91,21632,672],{},[91,21634,14051],{},[461,21636,798],{"stretchy":797},[88,21638,21639,21641],{},[91,21640,12802],{},[75,21642,21643,21645,21647],{},[91,21644,777],{},[461,21646,5777],{},[81,21648,83],{},[461,21650,812],{"stretchy":797},[499,21652],{"linebreak":21653},"newline",[88,21655,21656,21658],{},[91,21657,774],{},[91,21659,14051],{},[461,21661,798],{"stretchy":797},[88,21663,21664,21666],{},[91,21665,12802],{},[91,21667,777],{},[461,21669,803],{"separator":106},[91,21671,22],{},[461,21673,812],{"stretchy":797},[461,21675,815],{},[88,21677,21678,21680],{},[91,21679,7826],{},[91,21681,777],{},[461,21683,5777],{},[91,21685,18995],{},[88,21687,21688,21690],{},[91,21689,672],{},[91,21691,14051],{},[461,21693,798],{"stretchy":797},[88,21695,21696,21698],{},[91,21697,12802],{},[75,21699,21700,21702,21704],{},[91,21701,777],{},[461,21703,5777],{},[81,21705,83],{},[461,21707,812],{"stretchy":797},[461,21709,18365],{},[88,21711,21712,21714],{},[91,21713,672],{},[91,21715,14051],{},[461,21717,798],{"stretchy":797},[88,21719,21720,21722],{},[91,21721,12802],{},[91,21723,777],{},[461,21725,812],{"stretchy":797},[499,21727],{"linebreak":21653},[88,21729,21730,21732],{},[91,21731,672],{},[91,21733,14051],{},[461,21735,798],{"stretchy":797},[88,21737,21738,21740],{},[91,21739,12802],{},[75,21741,21742,21744,21746],{},[91,21743,777],{},[461,21745,5777],{},[81,21747,83],{},[461,21749,812],{"stretchy":797},[461,21751,14675],{},[88,21753,21754,21756],{},[91,21755,7826],{},[75,21757,21758,21760,21762],{},[91,21759,777],{},[461,21761,5777],{},[81,21763,83],{},[461,21765,5777],{},[91,21767,18995],{},[88,21769,21770,21772],{},[91,21771,672],{},[91,21773,14051],{},[461,21775,798],{"stretchy":797},[88,21777,21778,21780],{},[91,21779,12802],{},[75,21781,21782,21784,21786],{},[91,21783,777],{},[461,21785,5777],{},[81,21787,2821],{},[461,21789,812],{"stretchy":797},[98,21791,21792],{"encoding":100}," Q_\\theta(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) \\\\\n\n A_\\theta(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n V_\\theta(s_{t+1}) \\approx r_{t+1} + \\gamma V_\\theta(s_{t+2})",[59,21794,21796,21906,21962,22066,22069,22179,22234,22347,22439,22442,22552,22617],{"className":21795,"ariaHidden":106},[105],[59,21797,21799,21802,21842,21845,21885,21888,21891,21894,21897,21900,21903],{"className":21798},[110],[59,21800],{"className":21801,"style":877},[114],[59,21803,21805,21808],{"className":21804},[119],[59,21806,459],{"className":21807},[119,192],[59,21809,21811],{"className":21810},[196],[59,21812,21814,21834],{"className":21813},[131,132],[59,21815,21817,21831],{"className":21816},[136],[59,21818,21820],{"className":21819,"style":1248},[140],[59,21821,21822,21825],{"style":1251},[59,21823],{"className":21824,"style":540},[148],[59,21826,21828],{"className":21827},[153,154,155,156],[59,21829,14051],{"className":21830,"style":3210},[119,192,156],[59,21832,227],{"className":21833},[226],[59,21835,21837],{"className":21836},[136],[59,21838,21840],{"className":21839,"style":1270},[140],[59,21841],{},[59,21843,798],{"className":21844},[123],[59,21846,21848,21851],{"className":21847},[119],[59,21849,12802],{"className":21850},[119,192],[59,21852,21854],{"className":21853},[196],[59,21855,21857,21877],{"className":21856},[131,132],[59,21858,21860,21874],{"className":21859},[136],[59,21861,21863],{"className":21862,"style":12732},[140],[59,21864,21865,21868],{"style":1251},[59,21866],{"className":21867,"style":540},[148],[59,21869,21871],{"className":21870},[153,154,155,156],[59,21872,777],{"className":21873},[119,192,156],[59,21875,227],{"className":21876},[226],[59,21878,21880],{"className":21879},[136],[59,21881,21883],{"className":21882,"style":1270},[140],[59,21884],{},[59,21886,803],{"className":21887},[912],[59,21889],{"className":21890,"style":916},[499],[59,21892,22],{"className":21893},[119,192],[59,21895,812],{"className":21896},[313],[59,21898],{"className":21899,"style":500},[499],[59,21901,815],{"className":21902},[504],[59,21904],{"className":21905,"style":500},[499],[59,21907,21909,21913,21953,21956,21959],{"className":21908},[110],[59,21910],{"className":21911,"style":21912},[114],"height:0.7333em;vertical-align:-0.15em;",[59,21914,21916,21919],{"className":21915},[119],[59,21917,7826],{"className":21918,"style":3210},[119,192],[59,21920,21922],{"className":21921},[196],[59,21923,21925,21945],{"className":21924},[131,132],[59,21926,21928,21942],{"className":21927},[136],[59,21929,21931],{"className":21930,"style":12732},[140],[59,21932,21933,21936],{"style":13105},[59,21934],{"className":21935,"style":540},[148],[59,21937,21939],{"className":21938},[153,154,155,156],[59,21940,777],{"className":21941},[119,192,156],[59,21943,227],{"className":21944},[226],[59,21946,21948],{"className":21947},[136],[59,21949,21951],{"className":21950,"style":1270},[140],[59,21952],{},[59,21954],{"className":21955,"style":1872},[499],[59,21957,5777],{"className":21958},[553],[59,21960],{"className":21961,"style":1872},[499],[59,21963,21965,21968,21971,22011,22014,22063],{"className":21964},[110],[59,21966],{"className":21967,"style":877},[114],[59,21969,18995],{"className":21970,"style":19242},[119,192],[59,21972,21974,21977],{"className":21973},[119],[59,21975,672],{"className":21976,"style":702},[119,192],[59,21978,21980],{"className":21979},[196],[59,21981,21983,22003],{"className":21982},[131,132],[59,21984,21986,22000],{"className":21985},[136],[59,21987,21989],{"className":21988,"style":1248},[140],[59,21990,21991,21994],{"style":20505},[59,21992],{"className":21993,"style":540},[148],[59,21995,21997],{"className":21996},[153,154,155,156],[59,21998,14051],{"className":21999,"style":3210},[119,192,156],[59,22001,227],{"className":22002},[226],[59,22004,22006],{"className":22005},[136],[59,22007,22009],{"className":22008,"style":1270},[140],[59,22010],{},[59,22012,798],{"className":22013},[123],[59,22015,22017,22020],{"className":22016},[119],[59,22018,12802],{"className":22019},[119,192],[59,22021,22023],{"className":22022},[196],[59,22024,22026,22055],{"className":22025},[131,132],[59,22027,22029,22052],{"className":22028},[136],[59,22030,22032],{"className":22031,"style":3091},[140],[59,22033,22034,22037],{"style":1251},[59,22035],{"className":22036,"style":540},[148],[59,22038,22040],{"className":22039},[153,154,155,156],[59,22041,22043,22046,22049],{"className":22042},[119,156],[59,22044,777],{"className":22045},[119,192,156],[59,22047,5777],{"className":22048},[553,156],[59,22050,83],{"className":22051},[119,156],[59,22053,227],{"className":22054},[226],[59,22056,22058],{"className":22057},[136],[59,22059,22061],{"className":22060,"style":6938},[140],[59,22062],{},[59,22064,812],{"className":22065},[313],[59,22067],{"className":22068},[499,21653],[59,22070,22072,22075,22115,22118,22158,22161,22164,22167,22170,22173,22176],{"className":22071},[110],[59,22073],{"className":22074,"style":877},[114],[59,22076,22078,22081],{"className":22077},[119],[59,22079,774],{"className":22080},[119,192],[59,22082,22084],{"className":22083},[196],[59,22085,22087,22107],{"className":22086},[131,132],[59,22088,22090,22104],{"className":22089},[136],[59,22091,22093],{"className":22092,"style":1248},[140],[59,22094,22095,22098],{"style":1251},[59,22096],{"className":22097,"style":540},[148],[59,22099,22101],{"className":22100},[153,154,155,156],[59,22102,14051],{"className":22103,"style":3210},[119,192,156],[59,22105,227],{"className":22106},[226],[59,22108,22110],{"className":22109},[136],[59,22111,22113],{"className":22112,"style":1270},[140],[59,22114],{},[59,22116,798],{"className":22117},[123],[59,22119,22121,22124],{"className":22120},[119],[59,22122,12802],{"className":22123},[119,192],[59,22125,22127],{"className":22126},[196],[59,22128,22130,22150],{"className":22129},[131,132],[59,22131,22133,22147],{"className":22132},[136],[59,22134,22136],{"className":22135,"style":12732},[140],[59,22137,22138,22141],{"style":1251},[59,22139],{"className":22140,"style":540},[148],[59,22142,22144],{"className":22143},[153,154,155,156],[59,22145,777],{"className":22146},[119,192,156],[59,22148,227],{"className":22149},[226],[59,22151,22153],{"className":22152},[136],[59,22154,22156],{"className":22155,"style":1270},[140],[59,22157],{},[59,22159,803],{"className":22160},[912],[59,22162],{"className":22163,"style":916},[499],[59,22165,22],{"className":22166},[119,192],[59,22168,812],{"className":22169},[313],[59,22171],{"className":22172,"style":500},[499],[59,22174,815],{"className":22175},[504],[59,22177],{"className":22178,"style":500},[499],[59,22180,22182,22185,22225,22228,22231],{"className":22181},[110],[59,22183],{"className":22184,"style":21912},[114],[59,22186,22188,22191],{"className":22187},[119],[59,22189,7826],{"className":22190,"style":3210},[119,192],[59,22192,22194],{"className":22193},[196],[59,22195,22197,22217],{"className":22196},[131,132],[59,22198,22200,22214],{"className":22199},[136],[59,22201,22203],{"className":22202,"style":12732},[140],[59,22204,22205,22208],{"style":13105},[59,22206],{"className":22207,"style":540},[148],[59,22209,22211],{"className":22210},[153,154,155,156],[59,22212,777],{"className":22213},[119,192,156],[59,22215,227],{"className":22216},[226],[59,22218,22220],{"className":22219},[136],[59,22221,22223],{"className":22222,"style":1270},[140],[59,22224],{},[59,22226],{"className":22227,"style":1872},[499],[59,22229,5777],{"className":22230},[553],[59,22232],{"className":22233,"style":1872},[499],[59,22235,22237,22240,22243,22283,22286,22335,22338,22341,22344],{"className":22236},[110],[59,22238],{"className":22239,"style":877},[114],[59,22241,18995],{"className":22242,"style":19242},[119,192],[59,22244,22246,22249],{"className":22245},[119],[59,22247,672],{"className":22248,"style":702},[119,192],[59,22250,22252],{"className":22251},[196],[59,22253,22255,22275],{"className":22254},[131,132],[59,22256,22258,22272],{"className":22257},[136],[59,22259,22261],{"className":22260,"style":1248},[140],[59,22262,22263,22266],{"style":20505},[59,22264],{"className":22265,"style":540},[148],[59,22267,22269],{"className":22268},[153,154,155,156],[59,22270,14051],{"className":22271,"style":3210},[119,192,156],[59,22273,227],{"className":22274},[226],[59,22276,22278],{"className":22277},[136],[59,22279,22281],{"className":22280,"style":1270},[140],[59,22282],{},[59,22284,798],{"className":22285},[123],[59,22287,22289,22292],{"className":22288},[119],[59,22290,12802],{"className":22291},[119,192],[59,22293,22295],{"className":22294},[196],[59,22296,22298,22327],{"className":22297},[131,132],[59,22299,22301,22324],{"className":22300},[136],[59,22302,22304],{"className":22303,"style":3091},[140],[59,22305,22306,22309],{"style":1251},[59,22307],{"className":22308,"style":540},[148],[59,22310,22312],{"className":22311},[153,154,155,156],[59,22313,22315,22318,22321],{"className":22314},[119,156],[59,22316,777],{"className":22317},[119,192,156],[59,22319,5777],{"className":22320},[553,156],[59,22322,83],{"className":22323},[119,156],[59,22325,227],{"className":22326},[226],[59,22328,22330],{"className":22329},[136],[59,22331,22333],{"className":22332,"style":6938},[140],[59,22334],{},[59,22336,812],{"className":22337},[313],[59,22339],{"className":22340,"style":1872},[499],[59,22342,18365],{"className":22343},[553],[59,22345],{"className":22346,"style":1872},[499],[59,22348,22350,22353,22393,22396,22436],{"className":22349},[110],[59,22351],{"className":22352,"style":877},[114],[59,22354,22356,22359],{"className":22355},[119],[59,22357,672],{"className":22358,"style":702},[119,192],[59,22360,22362],{"className":22361},[196],[59,22363,22365,22385],{"className":22364},[131,132],[59,22366,22368,22382],{"className":22367},[136],[59,22369,22371],{"className":22370,"style":1248},[140],[59,22372,22373,22376],{"style":20505},[59,22374],{"className":22375,"style":540},[148],[59,22377,22379],{"className":22378},[153,154,155,156],[59,22380,14051],{"className":22381,"style":3210},[119,192,156],[59,22383,227],{"className":22384},[226],[59,22386,22388],{"className":22387},[136],[59,22389,22391],{"className":22390,"style":1270},[140],[59,22392],{},[59,22394,798],{"className":22395},[123],[59,22397,22399,22402],{"className":22398},[119],[59,22400,12802],{"className":22401},[119,192],[59,22403,22405],{"className":22404},[196],[59,22406,22408,22428],{"className":22407},[131,132],[59,22409,22411,22425],{"className":22410},[136],[59,22412,22414],{"className":22413,"style":12732},[140],[59,22415,22416,22419],{"style":1251},[59,22417],{"className":22418,"style":540},[148],[59,22420,22422],{"className":22421},[153,154,155,156],[59,22423,777],{"className":22424},[119,192,156],[59,22426,227],{"className":22427},[226],[59,22429,22431],{"className":22430},[136],[59,22432,22434],{"className":22433,"style":1270},[140],[59,22435],{},[59,22437,812],{"className":22438},[313],[59,22440],{"className":22441},[499,21653],[59,22443,22445,22448,22488,22491,22540,22543,22546,22549],{"className":22444},[110],[59,22446],{"className":22447,"style":877},[114],[59,22449,22451,22454],{"className":22450},[119],[59,22452,672],{"className":22453,"style":702},[119,192],[59,22455,22457],{"className":22456},[196],[59,22458,22460,22480],{"className":22459},[131,132],[59,22461,22463,22477],{"className":22462},[136],[59,22464,22466],{"className":22465,"style":1248},[140],[59,22467,22468,22471],{"style":20505},[59,22469],{"className":22470,"style":540},[148],[59,22472,22474],{"className":22473},[153,154,155,156],[59,22475,14051],{"className":22476,"style":3210},[119,192,156],[59,22478,227],{"className":22479},[226],[59,22481,22483],{"className":22482},[136],[59,22484,22486],{"className":22485,"style":1270},[140],[59,22487],{},[59,22489,798],{"className":22490},[123],[59,22492,22494,22497],{"className":22493},[119],[59,22495,12802],{"className":22496},[119,192],[59,22498,22500],{"className":22499},[196],[59,22501,22503,22532],{"className":22502},[131,132],[59,22504,22506,22529],{"className":22505},[136],[59,22507,22509],{"className":22508,"style":3091},[140],[59,22510,22511,22514],{"style":1251},[59,22512],{"className":22513,"style":540},[148],[59,22515,22517],{"className":22516},[153,154,155,156],[59,22518,22520,22523,22526],{"className":22519},[119,156],[59,22521,777],{"className":22522},[119,192,156],[59,22524,5777],{"className":22525},[553,156],[59,22527,83],{"className":22528},[119,156],[59,22530,227],{"className":22531},[226],[59,22533,22535],{"className":22534},[136],[59,22536,22538],{"className":22537,"style":6938},[140],[59,22539],{},[59,22541,812],{"className":22542},[313],[59,22544],{"className":22545,"style":500},[499],[59,22547,14675],{"className":22548},[504],[59,22550],{"className":22551,"style":500},[499],[59,22553,22555,22559,22608,22611,22614],{"className":22554},[110],[59,22556],{"className":22557,"style":22558},[114],"height:0.7917em;vertical-align:-0.2083em;",[59,22560,22562,22565],{"className":22561},[119],[59,22563,7826],{"className":22564,"style":3210},[119,192],[59,22566,22568],{"className":22567},[196],[59,22569,22571,22600],{"className":22570},[131,132],[59,22572,22574,22597],{"className":22573},[136],[59,22575,22577],{"className":22576,"style":3091},[140],[59,22578,22579,22582],{"style":13105},[59,22580],{"className":22581,"style":540},[148],[59,22583,22585],{"className":22584},[153,154,155,156],[59,22586,22588,22591,22594],{"className":22587},[119,156],[59,22589,777],{"className":22590},[119,192,156],[59,22592,5777],{"className":22593},[553,156],[59,22595,83],{"className":22596},[119,156],[59,22598,227],{"className":22599},[226],[59,22601,22603],{"className":22602},[136],[59,22604,22606],{"className":22605,"style":6938},[140],[59,22607],{},[59,22609],{"className":22610,"style":1872},[499],[59,22612,5777],{"className":22613},[553],[59,22615],{"className":22616,"style":1872},[499],[59,22618,22620,22623,22626,22666,22669,22718],{"className":22619},[110],[59,22621],{"className":22622,"style":877},[114],[59,22624,18995],{"className":22625,"style":19242},[119,192],[59,22627,22629,22632],{"className":22628},[119],[59,22630,672],{"className":22631,"style":702},[119,192],[59,22633,22635],{"className":22634},[196],[59,22636,22638,22658],{"className":22637},[131,132],[59,22639,22641,22655],{"className":22640},[136],[59,22642,22644],{"className":22643,"style":1248},[140],[59,22645,22646,22649],{"style":20505},[59,22647],{"className":22648,"style":540},[148],[59,22650,22652],{"className":22651},[153,154,155,156],[59,22653,14051],{"className":22654,"style":3210},[119,192,156],[59,22656,227],{"className":22657},[226],[59,22659,22661],{"className":22660},[136],[59,22662,22664],{"className":22663,"style":1270},[140],[59,22665],{},[59,22667,798],{"className":22668},[123],[59,22670,22672,22675],{"className":22671},[119],[59,22673,12802],{"className":22674},[119,192],[59,22676,22678],{"className":22677},[196],[59,22679,22681,22710],{"className":22680},[131,132],[59,22682,22684,22707],{"className":22683},[136],[59,22685,22687],{"className":22686,"style":3091},[140],[59,22688,22689,22692],{"style":1251},[59,22690],{"className":22691,"style":540},[148],[59,22693,22695],{"className":22694},[153,154,155,156],[59,22696,22698,22701,22704],{"className":22697},[119,156],[59,22699,777],{"className":22700},[119,192,156],[59,22702,5777],{"className":22703},[553,156],[59,22705,2821],{"className":22706},[119,156],[59,22708,227],{"className":22709},[226],[59,22711,22713],{"className":22712},[136],[59,22714,22716],{"className":22715,"style":6938},[140],[59,22717],{},[59,22719,812],{"className":22720},[313],[15,22722,22723,22724,22822],{},"通过上述公式变化，关于 ",[59,22725,22727,22755],{"className":22726},[62],[59,22728,22730],{"className":22729},[66],[68,22731,22732],{"xmlns":70},[72,22733,22734,22752],{},[75,22735,22736,22742,22744,22746,22748,22750],{},[88,22737,22738,22740],{},[91,22739,774],{},[91,22741,14051],{},[461,22743,798],{"stretchy":797},[91,22745,12802],{},[461,22747,803],{"separator":106},[91,22749,22],{},[461,22751,812],{"stretchy":797},[98,22753,22754],{"encoding":100},"A_\\theta(s,a)",[59,22756,22758],{"className":22757,"ariaHidden":106},[105],[59,22759,22761,22764,22804,22807,22810,22813,22816,22819],{"className":22760},[110],[59,22762],{"className":22763,"style":877},[114],[59,22765,22767,22770],{"className":22766},[119],[59,22768,774],{"className":22769},[119,192],[59,22771,22773],{"className":22772},[196],[59,22774,22776,22796],{"className":22775},[131,132],[59,22777,22779,22793],{"className":22778},[136],[59,22780,22782],{"className":22781,"style":1248},[140],[59,22783,22784,22787],{"style":1251},[59,22785],{"className":22786,"style":540},[148],[59,22788,22790],{"className":22789},[153,154,155,156],[59,22791,14051],{"className":22792,"style":3210},[119,192,156],[59,22794,227],{"className":22795},[226],[59,22797,22799],{"className":22798},[136],[59,22800,22802],{"className":22801,"style":1270},[140],[59,22803],{},[59,22805,798],{"className":22806},[123],[59,22808,12802],{"className":22809},[119,192],[59,22811,803],{"className":22812},[912],[59,22814],{"className":22815,"style":916},[499],[59,22817,22],{"className":22818},[119,192],[59,22820,812],{"className":22821},[313]," 的推到公式仅剩一个相关变量，可以降低公式的复杂性。",[15,22824,22825],{},"之后再根据采样步数进行估算。采样越多，情况越多，整体估计的偏差就会偏小，同样的，碰到极值的概率越大，方差就越大。",[59,22827,22829],{"className":22828},[14350],[59,22830,22832,23282],{"className":22831},[62],[59,22833,22835],{"className":22834},[66],[68,22836,22837],{"xmlns":70,"display":9481},[72,22838,22839,23279],{},[75,22840,22841,22849,22851,22857,22859,22861,22863,22865,22871,22873,22875,22881,22883,22895,22897,22899,22905,22907,22913,22915,22917,22925,22927,22933,22935,22937,22939,22941,22947,22949,22951,22963,22965,22971,22977,22979,22991,22993,22995,23001,23003,23009,23011,23013,23021,23023,23029,23031,23033,23035,23037,23043,23045,23047,23050,23062,23064,23070,23072,23084,23086,23092,23098,23100,23112,23114,23116,23122,23124,23130,23132,23134,23146,23148,23156,23158,23164,23166,23168,23170,23172,23178,23180,23182,23184,23196,23198,23204,23206,23218,23220,23226,23228,23240,23242,23245,23247,23253,23255,23261,23263,23269,23271,23277],{},[1822,22842,22843,22845,22847],{},[91,22844,774],{},[91,22846,14051],{},[81,22848,83],{},[461,22850,798],{"stretchy":797},[88,22852,22853,22855],{},[91,22854,12802],{},[91,22856,777],{},[461,22858,803],{"separator":106},[91,22860,22],{},[461,22862,812],{"stretchy":797},[461,22864,815],{},[88,22866,22867,22869],{},[91,22868,7826],{},[91,22870,777],{},[461,22872,5777],{},[91,22874,18995],{},[88,22876,22877,22879],{},[91,22878,672],{},[91,22880,14051],{},[461,22882,798],{"stretchy":797},[88,22884,22885,22887],{},[91,22886,12802],{},[75,22888,22889,22891,22893],{},[91,22890,777],{},[461,22892,5777],{},[81,22894,83],{},[461,22896,812],{"stretchy":797},[461,22898,18365],{},[88,22900,22901,22903],{},[91,22902,672],{},[91,22904,14051],{},[461,22906,798],{"stretchy":797},[88,22908,22909,22911],{},[91,22910,12802],{},[91,22912,777],{},[461,22914,812],{"stretchy":797},[499,22916],{"linebreak":21653},[1822,22918,22919,22921,22923],{},[91,22920,774],{},[91,22922,14051],{},[81,22924,2821],{},[461,22926,798],{"stretchy":797},[88,22928,22929,22931],{},[91,22930,12802],{},[91,22932,777],{},[461,22934,803],{"separator":106},[91,22936,22],{},[461,22938,812],{"stretchy":797},[461,22940,815],{},[88,22942,22943,22945],{},[91,22944,7826],{},[91,22946,777],{},[461,22948,5777],{},[91,22950,18995],{},[88,22952,22953,22955],{},[91,22954,7826],{},[75,22956,22957,22959,22961],{},[91,22958,777],{},[461,22960,5777],{},[81,22962,83],{},[461,22964,5777],{},[465,22966,22967,22969],{},[91,22968,18995],{},[81,22970,2821],{},[88,22972,22973,22975],{},[91,22974,672],{},[91,22976,14051],{},[461,22978,798],{"stretchy":797},[88,22980,22981,22983],{},[91,22982,12802],{},[75,22984,22985,22987,22989],{},[91,22986,777],{},[461,22988,5777],{},[81,22990,2821],{},[461,22992,812],{"stretchy":797},[461,22994,18365],{},[88,22996,22997,22999],{},[91,22998,672],{},[91,23000,14051],{},[461,23002,798],{"stretchy":797},[88,23004,23005,23007],{},[91,23006,12802],{},[91,23008,777],{},[461,23010,812],{"stretchy":797},[499,23012],{"linebreak":21653},[1822,23014,23015,23017,23019],{},[91,23016,774],{},[91,23018,14051],{},[81,23020,4671],{},[461,23022,798],{"stretchy":797},[88,23024,23025,23027],{},[91,23026,12802],{},[91,23028,777],{},[461,23030,803],{"separator":106},[91,23032,22],{},[461,23034,812],{"stretchy":797},[461,23036,815],{},[88,23038,23039,23041],{},[91,23040,7826],{},[91,23042,777],{},[461,23044,5777],{},[91,23046,18995],{},[461,23048,23049],{},"∗",[88,23051,23052,23054],{},[91,23053,7826],{},[75,23055,23056,23058,23060],{},[91,23057,777],{},[461,23059,5777],{},[81,23061,83],{},[461,23063,5777],{},[465,23065,23066,23068],{},[91,23067,18995],{},[81,23069,2821],{},[461,23071,23049],{},[88,23073,23074,23076],{},[91,23075,7826],{},[75,23077,23078,23080,23082],{},[91,23079,777],{},[461,23081,5777],{},[81,23083,2821],{},[461,23085,5777],{},[465,23087,23088,23090],{},[91,23089,18995],{},[81,23091,4671],{},[88,23093,23094,23096],{},[91,23095,672],{},[91,23097,14051],{},[461,23099,798],{"stretchy":797},[88,23101,23102,23104],{},[91,23103,12802],{},[75,23105,23106,23108,23110],{},[91,23107,777],{},[461,23109,5777],{},[81,23111,4671],{},[461,23113,812],{"stretchy":797},[461,23115,18365],{},[88,23117,23118,23120],{},[91,23119,672],{},[91,23121,14051],{},[461,23123,798],{"stretchy":797},[88,23125,23126,23128],{},[91,23127,12802],{},[91,23129,777],{},[461,23131,812],{"stretchy":797},[499,23133],{"linebreak":21653},[75,23135,23136,23139],{},[91,23137,23138],{"mathvariant":818},"⋮",[23140,23141,23142],"mpadded",{"height":14366,"voffset":14366},[499,23143],{"mathbackground":23144,"width":14366,"height":23145},"black","1.5em",[499,23147],{"linebreak":21653},[1822,23149,23150,23152,23154],{},[91,23151,774],{},[91,23153,14051],{},[91,23155,13242],{},[461,23157,798],{"stretchy":797},[88,23159,23160,23162],{},[91,23161,12802],{},[91,23163,777],{},[461,23165,803],{"separator":106},[91,23167,22],{},[461,23169,812],{"stretchy":797},[461,23171,815],{},[88,23173,23174,23176],{},[91,23175,7826],{},[91,23177,777],{},[461,23179,5777],{},[91,23181,18995],{},[461,23183,23049],{},[88,23185,23186,23188],{},[91,23187,7826],{},[75,23189,23190,23192,23194],{},[91,23191,777],{},[461,23193,5777],{},[81,23195,83],{},[461,23197,5777],{},[465,23199,23200,23202],{},[91,23201,18995],{},[81,23203,2821],{},[461,23205,23049],{},[88,23207,23208,23210],{},[91,23209,7826],{},[75,23211,23212,23214,23216],{},[91,23213,777],{},[461,23215,5777],{},[81,23217,2821],{},[461,23219,5777],{},[465,23221,23222,23224],{},[91,23223,18995],{},[81,23225,4671],{},[461,23227,23049],{},[88,23229,23230,23232],{},[91,23231,7826],{},[75,23233,23234,23236,23238],{},[91,23235,777],{},[461,23237,5777],{},[81,23239,4671],{},[461,23241,5777],{},[461,23243,23244],{},"⋯",[461,23246,5777],{},[465,23248,23249,23251],{},[91,23250,18995],{},[91,23252,13242],{},[461,23254,23049],{},[88,23256,23257,23259],{},[91,23258,7826],{},[91,23260,13242],{},[461,23262,18365],{},[88,23264,23265,23267],{},[91,23266,672],{},[91,23268,14051],{},[461,23270,798],{"stretchy":797},[88,23272,23273,23275],{},[91,23274,12802],{},[91,23276,777],{},[461,23278,812],{"stretchy":797},[98,23280,23281],{"encoding":100}," A_\\theta^1(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n A_\\theta^2(s_t, a) = r_t + \\gamma r_{t+1} + \\gamma^2 V_\\theta(s_{t+2}) - V_\\theta(s_t)\\\\\n A_\\theta^3(s_t, a) = r_t + \\gamma * r_{t+1} + \\gamma^2 * r_{t+2} + \\gamma^3 V_\\theta(s_{t+3}) - V_\\theta(s_t)\\\\\n \\vdots \\\\\n A_\\theta^T(s_t, a) = r_t + \\gamma * r_{t+1} + \\gamma^2 * r_{t+2} + \\gamma^3 * r_{t+3} + \\cdots + \\gamma^T * r_T - V_\\theta(s_t)",[59,23283,23285,23408,23463,23576,23668,23671,23792,23847,23914,24053,24145,24148,24269,24324,24343,24407,24452,24516,24655,24747,24750,24768,24771,24894,24949,24967,25031,25075,25139,25183,25247,25266,25311,25366],{"className":23284,"ariaHidden":106},[105],[59,23286,23288,23292,23344,23347,23387,23390,23393,23396,23399,23402,23405],{"className":23287},[110],[59,23289],{"className":23290,"style":23291},[114],"height:1.1141em;vertical-align:-0.25em;",[59,23293,23295,23298],{"className":23294},[119],[59,23296,774],{"className":23297},[119,192],[59,23299,23301],{"className":23300},[196],[59,23302,23304,23336],{"className":23303},[131,132],[59,23305,23307,23333],{"className":23306},[136],[59,23308,23311,23322],{"className":23309,"style":23310},[140],"height:0.8641em;",[59,23312,23313,23316],{"style":17311},[59,23314],{"className":23315,"style":540},[148],[59,23317,23319],{"className":23318},[153,154,155,156],[59,23320,14051],{"className":23321,"style":3210},[119,192,156],[59,23323,23324,23327],{"style":16426},[59,23325],{"className":23326,"style":540},[148],[59,23328,23330],{"className":23329},[153,154,155,156],[59,23331,83],{"className":23332},[119,156],[59,23334,227],{"className":23335},[226],[59,23337,23339],{"className":23338},[136],[59,23340,23342],{"className":23341,"style":17341},[140],[59,23343],{},[59,23345,798],{"className":23346},[123],[59,23348,23350,23353],{"className":23349},[119],[59,23351,12802],{"className":23352},[119,192],[59,23354,23356],{"className":23355},[196],[59,23357,23359,23379],{"className":23358},[131,132],[59,23360,23362,23376],{"className":23361},[136],[59,23363,23365],{"className":23364,"style":12732},[140],[59,23366,23367,23370],{"style":1251},[59,23368],{"className":23369,"style":540},[148],[59,23371,23373],{"className":23372},[153,154,155,156],[59,23374,777],{"className":23375},[119,192,156],[59,23377,227],{"className":23378},[226],[59,23380,23382],{"className":23381},[136],[59,23383,23385],{"className":23384,"style":1270},[140],[59,23386],{},[59,23388,803],{"className":23389},[912],[59,23391],{"className":23392,"style":916},[499],[59,23394,22],{"className":23395},[119,192],[59,23397,812],{"className":23398},[313],[59,23400],{"className":23401,"style":500},[499],[59,23403,815],{"className":23404},[504],[59,23406],{"className":23407,"style":500},[499],[59,23409,23411,23414,23454,23457,23460],{"className":23410},[110],[59,23412],{"className":23413,"style":21912},[114],[59,23415,23417,23420],{"className":23416},[119],[59,23418,7826],{"className":23419,"style":3210},[119,192],[59,23421,23423],{"className":23422},[196],[59,23424,23426,23446],{"className":23425},[131,132],[59,23427,23429,23443],{"className":23428},[136],[59,23430,23432],{"className":23431,"style":12732},[140],[59,23433,23434,23437],{"style":13105},[59,23435],{"className":23436,"style":540},[148],[59,23438,23440],{"className":23439},[153,154,155,156],[59,23441,777],{"className":23442},[119,192,156],[59,23444,227],{"className":23445},[226],[59,23447,23449],{"className":23448},[136],[59,23450,23452],{"className":23451,"style":1270},[140],[59,23453],{},[59,23455],{"className":23456,"style":1872},[499],[59,23458,5777],{"className":23459},[553],[59,23461],{"className":23462,"style":1872},[499],[59,23464,23466,23469,23472,23512,23515,23564,23567,23570,23573],{"className":23465},[110],[59,23467],{"className":23468,"style":877},[114],[59,23470,18995],{"className":23471,"style":19242},[119,192],[59,23473,23475,23478],{"className":23474},[119],[59,23476,672],{"className":23477,"style":702},[119,192],[59,23479,23481],{"className":23480},[196],[59,23482,23484,23504],{"className":23483},[131,132],[59,23485,23487,23501],{"className":23486},[136],[59,23488,23490],{"className":23489,"style":1248},[140],[59,23491,23492,23495],{"style":20505},[59,23493],{"className":23494,"style":540},[148],[59,23496,23498],{"className":23497},[153,154,155,156],[59,23499,14051],{"className":23500,"style":3210},[119,192,156],[59,23502,227],{"className":23503},[226],[59,23505,23507],{"className":23506},[136],[59,23508,23510],{"className":23509,"style":1270},[140],[59,23511],{},[59,23513,798],{"className":23514},[123],[59,23516,23518,23521],{"className":23517},[119],[59,23519,12802],{"className":23520},[119,192],[59,23522,23524],{"className":23523},[196],[59,23525,23527,23556],{"className":23526},[131,132],[59,23528,23530,23553],{"className":23529},[136],[59,23531,23533],{"className":23532,"style":3091},[140],[59,23534,23535,23538],{"style":1251},[59,23536],{"className":23537,"style":540},[148],[59,23539,23541],{"className":23540},[153,154,155,156],[59,23542,23544,23547,23550],{"className":23543},[119,156],[59,23545,777],{"className":23546},[119,192,156],[59,23548,5777],{"className":23549},[553,156],[59,23551,83],{"className":23552},[119,156],[59,23554,227],{"className":23555},[226],[59,23557,23559],{"className":23558},[136],[59,23560,23562],{"className":23561,"style":6938},[140],[59,23563],{},[59,23565,812],{"className":23566},[313],[59,23568],{"className":23569,"style":1872},[499],[59,23571,18365],{"className":23572},[553],[59,23574],{"className":23575,"style":1872},[499],[59,23577,23579,23582,23622,23625,23665],{"className":23578},[110],[59,23580],{"className":23581,"style":877},[114],[59,23583,23585,23588],{"className":23584},[119],[59,23586,672],{"className":23587,"style":702},[119,192],[59,23589,23591],{"className":23590},[196],[59,23592,23594,23614],{"className":23593},[131,132],[59,23595,23597,23611],{"className":23596},[136],[59,23598,23600],{"className":23599,"style":1248},[140],[59,23601,23602,23605],{"style":20505},[59,23603],{"className":23604,"style":540},[148],[59,23606,23608],{"className":23607},[153,154,155,156],[59,23609,14051],{"className":23610,"style":3210},[119,192,156],[59,23612,227],{"className":23613},[226],[59,23615,23617],{"className":23616},[136],[59,23618,23620],{"className":23619,"style":1270},[140],[59,23621],{},[59,23623,798],{"className":23624},[123],[59,23626,23628,23631],{"className":23627},[119],[59,23629,12802],{"className":23630},[119,192],[59,23632,23634],{"className":23633},[196],[59,23635,23637,23657],{"className":23636},[131,132],[59,23638,23640,23654],{"className":23639},[136],[59,23641,23643],{"className":23642,"style":12732},[140],[59,23644,23645,23648],{"style":1251},[59,23646],{"className":23647,"style":540},[148],[59,23649,23651],{"className":23650},[153,154,155,156],[59,23652,777],{"className":23653},[119,192,156],[59,23655,227],{"className":23656},[226],[59,23658,23660],{"className":23659},[136],[59,23661,23663],{"className":23662,"style":1270},[140],[59,23664],{},[59,23666,812],{"className":23667},[313],[59,23669],{"className":23670},[499,21653],[59,23672,23674,23677,23728,23731,23771,23774,23777,23780,23783,23786,23789],{"className":23673},[110],[59,23675],{"className":23676,"style":23291},[114],[59,23678,23680,23683],{"className":23679},[119],[59,23681,774],{"className":23682},[119,192],[59,23684,23686],{"className":23685},[196],[59,23687,23689,23720],{"className":23688},[131,132],[59,23690,23692,23717],{"className":23691},[136],[59,23693,23695,23706],{"className":23694,"style":23310},[140],[59,23696,23697,23700],{"style":17311},[59,23698],{"className":23699,"style":540},[148],[59,23701,23703],{"className":23702},[153,154,155,156],[59,23704,14051],{"className":23705,"style":3210},[119,192,156],[59,23707,23708,23711],{"style":16426},[59,23709],{"className":23710,"style":540},[148],[59,23712,23714],{"className":23713},[153,154,155,156],[59,23715,2821],{"className":23716},[119,156],[59,23718,227],{"className":23719},[226],[59,23721,23723],{"className":23722},[136],[59,23724,23726],{"className":23725,"style":17341},[140],[59,23727],{},[59,23729,798],{"className":23730},[123],[59,23732,23734,23737],{"className":23733},[119],[59,23735,12802],{"className":23736},[119,192],[59,23738,23740],{"className":23739},[196],[59,23741,23743,23763],{"className":23742},[131,132],[59,23744,23746,23760],{"className":23745},[136],[59,23747,23749],{"className":23748,"style":12732},[140],[59,23750,23751,23754],{"style":1251},[59,23752],{"className":23753,"style":540},[148],[59,23755,23757],{"className":23756},[153,154,155,156],[59,23758,777],{"className":23759},[119,192,156],[59,23761,227],{"className":23762},[226],[59,23764,23766],{"className":23765},[136],[59,23767,23769],{"className":23768,"style":1270},[140],[59,23770],{},[59,23772,803],{"className":23773},[912],[59,23775],{"className":23776,"style":916},[499],[59,23778,22],{"className":23779},[119,192],[59,23781,812],{"className":23782},[313],[59,23784],{"className":23785,"style":500},[499],[59,23787,815],{"className":23788},[504],[59,23790],{"className":23791,"style":500},[499],[59,23793,23795,23798,23838,23841,23844],{"className":23794},[110],[59,23796],{"className":23797,"style":21912},[114],[59,23799,23801,23804],{"className":23800},[119],[59,23802,7826],{"className":23803,"style":3210},[119,192],[59,23805,23807],{"className":23806},[196],[59,23808,23810,23830],{"className":23809},[131,132],[59,23811,23813,23827],{"className":23812},[136],[59,23814,23816],{"className":23815,"style":12732},[140],[59,23817,23818,23821],{"style":13105},[59,23819],{"className":23820,"style":540},[148],[59,23822,23824],{"className":23823},[153,154,155,156],[59,23825,777],{"className":23826},[119,192,156],[59,23828,227],{"className":23829},[226],[59,23831,23833],{"className":23832},[136],[59,23834,23836],{"className":23835,"style":1270},[140],[59,23837],{},[59,23839],{"className":23840,"style":1872},[499],[59,23842,5777],{"className":23843},[553],[59,23845],{"className":23846,"style":1872},[499],[59,23848,23850,23853,23856,23905,23908,23911],{"className":23849},[110],[59,23851],{"className":23852,"style":22558},[114],[59,23854,18995],{"className":23855,"style":19242},[119,192],[59,23857,23859,23862],{"className":23858},[119],[59,23860,7826],{"className":23861,"style":3210},[119,192],[59,23863,23865],{"className":23864},[196],[59,23866,23868,23897],{"className":23867},[131,132],[59,23869,23871,23894],{"className":23870},[136],[59,23872,23874],{"className":23873,"style":3091},[140],[59,23875,23876,23879],{"style":13105},[59,23877],{"className":23878,"style":540},[148],[59,23880,23882],{"className":23881},[153,154,155,156],[59,23883,23885,23888,23891],{"className":23884},[119,156],[59,23886,777],{"className":23887},[119,192,156],[59,23889,5777],{"className":23890},[553,156],[59,23892,83],{"className":23893},[119,156],[59,23895,227],{"className":23896},[226],[59,23898,23900],{"className":23899},[136],[59,23901,23903],{"className":23902,"style":6938},[140],[59,23904],{},[59,23906],{"className":23907,"style":1872},[499],[59,23909,5777],{"className":23910},[553],[59,23912],{"className":23913,"style":1872},[499],[59,23915,23917,23920,23949,23989,23992,24041,24044,24047,24050],{"className":23916},[110],[59,23918],{"className":23919,"style":23291},[114],[59,23921,23923,23926],{"className":23922},[119],[59,23924,18995],{"className":23925,"style":19242},[119,192],[59,23927,23929],{"className":23928},[196],[59,23930,23932],{"className":23931},[131],[59,23933,23935],{"className":23934},[136],[59,23936,23938],{"className":23937,"style":23310},[140],[59,23939,23940,23943],{"style":16426},[59,23941],{"className":23942,"style":540},[148],[59,23944,23946],{"className":23945},[153,154,155,156],[59,23947,2821],{"className":23948},[119,156],[59,23950,23952,23955],{"className":23951},[119],[59,23953,672],{"className":23954,"style":702},[119,192],[59,23956,23958],{"className":23957},[196],[59,23959,23961,23981],{"className":23960},[131,132],[59,23962,23964,23978],{"className":23963},[136],[59,23965,23967],{"className":23966,"style":1248},[140],[59,23968,23969,23972],{"style":20505},[59,23970],{"className":23971,"style":540},[148],[59,23973,23975],{"className":23974},[153,154,155,156],[59,23976,14051],{"className":23977,"style":3210},[119,192,156],[59,23979,227],{"className":23980},[226],[59,23982,23984],{"className":23983},[136],[59,23985,23987],{"className":23986,"style":1270},[140],[59,23988],{},[59,23990,798],{"className":23991},[123],[59,23993,23995,23998],{"className":23994},[119],[59,23996,12802],{"className":23997},[119,192],[59,23999,24001],{"className":24000},[196],[59,24002,24004,24033],{"className":24003},[131,132],[59,24005,24007,24030],{"className":24006},[136],[59,24008,24010],{"className":24009,"style":3091},[140],[59,24011,24012,24015],{"style":1251},[59,24013],{"className":24014,"style":540},[148],[59,24016,24018],{"className":24017},[153,154,155,156],[59,24019,24021,24024,24027],{"className":24020},[119,156],[59,24022,777],{"className":24023},[119,192,156],[59,24025,5777],{"className":24026},[553,156],[59,24028,2821],{"className":24029},[119,156],[59,24031,227],{"className":24032},[226],[59,24034,24036],{"className":24035},[136],[59,24037,24039],{"className":24038,"style":6938},[140],[59,24040],{},[59,24042,812],{"className":24043},[313],[59,24045],{"className":24046,"style":1872},[499],[59,24048,18365],{"className":24049},[553],[59,24051],{"className":24052,"style":1872},[499],[59,24054,24056,24059,24099,24102,24142],{"className":24055},[110],[59,24057],{"className":24058,"style":877},[114],[59,24060,24062,24065],{"className":24061},[119],[59,24063,672],{"className":24064,"style":702},[119,192],[59,24066,24068],{"className":24067},[196],[59,24069,24071,24091],{"className":24070},[131,132],[59,24072,24074,24088],{"className":24073},[136],[59,24075,24077],{"className":24076,"style":1248},[140],[59,24078,24079,24082],{"style":20505},[59,24080],{"className":24081,"style":540},[148],[59,24083,24085],{"className":24084},[153,154,155,156],[59,24086,14051],{"className":24087,"style":3210},[119,192,156],[59,24089,227],{"className":24090},[226],[59,24092,24094],{"className":24093},[136],[59,24095,24097],{"className":24096,"style":1270},[140],[59,24098],{},[59,24100,798],{"className":24101},[123],[59,24103,24105,24108],{"className":24104},[119],[59,24106,12802],{"className":24107},[119,192],[59,24109,24111],{"className":24110},[196],[59,24112,24114,24134],{"className":24113},[131,132],[59,24115,24117,24131],{"className":24116},[136],[59,24118,24120],{"className":24119,"style":12732},[140],[59,24121,24122,24125],{"style":1251},[59,24123],{"className":24124,"style":540},[148],[59,24126,24128],{"className":24127},[153,154,155,156],[59,24129,777],{"className":24130},[119,192,156],[59,24132,227],{"className":24133},[226],[59,24135,24137],{"className":24136},[136],[59,24138,24140],{"className":24139,"style":1270},[140],[59,24141],{},[59,24143,812],{"className":24144},[313],[59,24146],{"className":24147},[499,21653],[59,24149,24151,24154,24205,24208,24248,24251,24254,24257,24260,24263,24266],{"className":24150},[110],[59,24152],{"className":24153,"style":23291},[114],[59,24155,24157,24160],{"className":24156},[119],[59,24158,774],{"className":24159},[119,192],[59,24161,24163],{"className":24162},[196],[59,24164,24166,24197],{"className":24165},[131,132],[59,24167,24169,24194],{"className":24168},[136],[59,24170,24172,24183],{"className":24171,"style":23310},[140],[59,24173,24174,24177],{"style":17311},[59,24175],{"className":24176,"style":540},[148],[59,24178,24180],{"className":24179},[153,154,155,156],[59,24181,14051],{"className":24182,"style":3210},[119,192,156],[59,24184,24185,24188],{"style":16426},[59,24186],{"className":24187,"style":540},[148],[59,24189,24191],{"className":24190},[153,154,155,156],[59,24192,4671],{"className":24193},[119,156],[59,24195,227],{"className":24196},[226],[59,24198,24200],{"className":24199},[136],[59,24201,24203],{"className":24202,"style":17341},[140],[59,24204],{},[59,24206,798],{"className":24207},[123],[59,24209,24211,24214],{"className":24210},[119],[59,24212,12802],{"className":24213},[119,192],[59,24215,24217],{"className":24216},[196],[59,24218,24220,24240],{"className":24219},[131,132],[59,24221,24223,24237],{"className":24222},[136],[59,24224,24226],{"className":24225,"style":12732},[140],[59,24227,24228,24231],{"style":1251},[59,24229],{"className":24230,"style":540},[148],[59,24232,24234],{"className":24233},[153,154,155,156],[59,24235,777],{"className":24236},[119,192,156],[59,24238,227],{"className":24239},[226],[59,24241,24243],{"className":24242},[136],[59,24244,24246],{"className":24245,"style":1270},[140],[59,24247],{},[59,24249,803],{"className":24250},[912],[59,24252],{"className":24253,"style":916},[499],[59,24255,22],{"className":24256},[119,192],[59,24258,812],{"className":24259},[313],[59,24261],{"className":24262,"style":500},[499],[59,24264,815],{"className":24265},[504],[59,24267],{"className":24268,"style":500},[499],[59,24270,24272,24275,24315,24318,24321],{"className":24271},[110],[59,24273],{"className":24274,"style":21912},[114],[59,24276,24278,24281],{"className":24277},[119],[59,24279,7826],{"className":24280,"style":3210},[119,192],[59,24282,24284],{"className":24283},[196],[59,24285,24287,24307],{"className":24286},[131,132],[59,24288,24290,24304],{"className":24289},[136],[59,24291,24293],{"className":24292,"style":12732},[140],[59,24294,24295,24298],{"style":13105},[59,24296],{"className":24297,"style":540},[148],[59,24299,24301],{"className":24300},[153,154,155,156],[59,24302,777],{"className":24303},[119,192,156],[59,24305,227],{"className":24306},[226],[59,24308,24310],{"className":24309},[136],[59,24311,24313],{"className":24312,"style":1270},[140],[59,24314],{},[59,24316],{"className":24317,"style":1872},[499],[59,24319,5777],{"className":24320},[553],[59,24322],{"className":24323,"style":1872},[499],[59,24325,24327,24331,24334,24337,24340],{"className":24326},[110],[59,24328],{"className":24329,"style":24330},[114],"height:0.6597em;vertical-align:-0.1944em;",[59,24332,18995],{"className":24333,"style":19242},[119,192],[59,24335],{"className":24336,"style":1872},[499],[59,24338,23049],{"className":24339},[553],[59,24341],{"className":24342,"style":1872},[499],[59,24344,24346,24349,24398,24401,24404],{"className":24345},[110],[59,24347],{"className":24348,"style":22558},[114],[59,24350,24352,24355],{"className":24351},[119],[59,24353,7826],{"className":24354,"style":3210},[119,192],[59,24356,24358],{"className":24357},[196],[59,24359,24361,24390],{"className":24360},[131,132],[59,24362,24364,24387],{"className":24363},[136],[59,24365,24367],{"className":24366,"style":3091},[140],[59,24368,24369,24372],{"style":13105},[59,24370],{"className":24371,"style":540},[148],[59,24373,24375],{"className":24374},[153,154,155,156],[59,24376,24378,24381,24384],{"className":24377},[119,156],[59,24379,777],{"className":24380},[119,192,156],[59,24382,5777],{"className":24383},[553,156],[59,24385,83],{"className":24386},[119,156],[59,24388,227],{"className":24389},[226],[59,24391,24393],{"className":24392},[136],[59,24394,24396],{"className":24395,"style":6938},[140],[59,24397],{},[59,24399],{"className":24400,"style":1872},[499],[59,24402,5777],{"className":24403},[553],[59,24405],{"className":24406,"style":1872},[499],[59,24408,24410,24414,24443,24446,24449],{"className":24409},[110],[59,24411],{"className":24412,"style":24413},[114],"height:1.0585em;vertical-align:-0.1944em;",[59,24415,24417,24420],{"className":24416},[119],[59,24418,18995],{"className":24419,"style":19242},[119,192],[59,24421,24423],{"className":24422},[196],[59,24424,24426],{"className":24425},[131],[59,24427,24429],{"className":24428},[136],[59,24430,24432],{"className":24431,"style":23310},[140],[59,24433,24434,24437],{"style":16426},[59,24435],{"className":24436,"style":540},[148],[59,24438,24440],{"className":24439},[153,154,155,156],[59,24441,2821],{"className":24442},[119,156],[59,24444],{"className":24445,"style":1872},[499],[59,24447,23049],{"className":24448},[553],[59,24450],{"className":24451,"style":1872},[499],[59,24453,24455,24458,24507,24510,24513],{"className":24454},[110],[59,24456],{"className":24457,"style":22558},[114],[59,24459,24461,24464],{"className":24460},[119],[59,24462,7826],{"className":24463,"style":3210},[119,192],[59,24465,24467],{"className":24466},[196],[59,24468,24470,24499],{"className":24469},[131,132],[59,24471,24473,24496],{"className":24472},[136],[59,24474,24476],{"className":24475,"style":3091},[140],[59,24477,24478,24481],{"style":13105},[59,24479],{"className":24480,"style":540},[148],[59,24482,24484],{"className":24483},[153,154,155,156],[59,24485,24487,24490,24493],{"className":24486},[119,156],[59,24488,777],{"className":24489},[119,192,156],[59,24491,5777],{"className":24492},[553,156],[59,24494,2821],{"className":24495},[119,156],[59,24497,227],{"className":24498},[226],[59,24500,24502],{"className":24501},[136],[59,24503,24505],{"className":24504,"style":6938},[140],[59,24506],{},[59,24508],{"className":24509,"style":1872},[499],[59,24511,5777],{"className":24512},[553],[59,24514],{"className":24515,"style":1872},[499],[59,24517,24519,24522,24551,24591,24594,24643,24646,24649,24652],{"className":24518},[110],[59,24520],{"className":24521,"style":23291},[114],[59,24523,24525,24528],{"className":24524},[119],[59,24526,18995],{"className":24527,"style":19242},[119,192],[59,24529,24531],{"className":24530},[196],[59,24532,24534],{"className":24533},[131],[59,24535,24537],{"className":24536},[136],[59,24538,24540],{"className":24539,"style":23310},[140],[59,24541,24542,24545],{"style":16426},[59,24543],{"className":24544,"style":540},[148],[59,24546,24548],{"className":24547},[153,154,155,156],[59,24549,4671],{"className":24550},[119,156],[59,24552,24554,24557],{"className":24553},[119],[59,24555,672],{"className":24556,"style":702},[119,192],[59,24558,24560],{"className":24559},[196],[59,24561,24563,24583],{"className":24562},[131,132],[59,24564,24566,24580],{"className":24565},[136],[59,24567,24569],{"className":24568,"style":1248},[140],[59,24570,24571,24574],{"style":20505},[59,24572],{"className":24573,"style":540},[148],[59,24575,24577],{"className":24576},[153,154,155,156],[59,24578,14051],{"className":24579,"style":3210},[119,192,156],[59,24581,227],{"className":24582},[226],[59,24584,24586],{"className":24585},[136],[59,24587,24589],{"className":24588,"style":1270},[140],[59,24590],{},[59,24592,798],{"className":24593},[123],[59,24595,24597,24600],{"className":24596},[119],[59,24598,12802],{"className":24599},[119,192],[59,24601,24603],{"className":24602},[196],[59,24604,24606,24635],{"className":24605},[131,132],[59,24607,24609,24632],{"className":24608},[136],[59,24610,24612],{"className":24611,"style":3091},[140],[59,24613,24614,24617],{"style":1251},[59,24615],{"className":24616,"style":540},[148],[59,24618,24620],{"className":24619},[153,154,155,156],[59,24621,24623,24626,24629],{"className":24622},[119,156],[59,24624,777],{"className":24625},[119,192,156],[59,24627,5777],{"className":24628},[553,156],[59,24630,4671],{"className":24631},[119,156],[59,24633,227],{"className":24634},[226],[59,24636,24638],{"className":24637},[136],[59,24639,24641],{"className":24640,"style":6938},[140],[59,24642],{},[59,24644,812],{"className":24645},[313],[59,24647],{"className":24648,"style":1872},[499],[59,24650,18365],{"className":24651},[553],[59,24653],{"className":24654,"style":1872},[499],[59,24656,24658,24661,24701,24704,24744],{"className":24657},[110],[59,24659],{"className":24660,"style":877},[114],[59,24662,24664,24667],{"className":24663},[119],[59,24665,672],{"className":24666,"style":702},[119,192],[59,24668,24670],{"className":24669},[196],[59,24671,24673,24693],{"className":24672},[131,132],[59,24674,24676,24690],{"className":24675},[136],[59,24677,24679],{"className":24678,"style":1248},[140],[59,24680,24681,24684],{"style":20505},[59,24682],{"className":24683,"style":540},[148],[59,24685,24687],{"className":24686},[153,154,155,156],[59,24688,14051],{"className":24689,"style":3210},[119,192,156],[59,24691,227],{"className":24692},[226],[59,24694,24696],{"className":24695},[136],[59,24697,24699],{"className":24698,"style":1270},[140],[59,24700],{},[59,24702,798],{"className":24703},[123],[59,24705,24707,24710],{"className":24706},[119],[59,24708,12802],{"className":24709},[119,192],[59,24711,24713],{"className":24712},[196],[59,24714,24716,24736],{"className":24715},[131,132],[59,24717,24719,24733],{"className":24718},[136],[59,24720,24722],{"className":24721,"style":12732},[140],[59,24723,24724,24727],{"style":1251},[59,24725],{"className":24726,"style":540},[148],[59,24728,24730],{"className":24729},[153,154,155,156],[59,24731,777],{"className":24732},[119,192,156],[59,24734,227],{"className":24735},[226],[59,24737,24739],{"className":24738},[136],[59,24740,24742],{"className":24741,"style":1270},[140],[59,24743],{},[59,24745,812],{"className":24746},[313],[59,24748],{"className":24749},[499,21653],[59,24751,24753,24757],{"className":24752},[110],[59,24754],{"className":24755,"style":24756},[114],"height:1.53em;vertical-align:-0.03em;",[59,24758,24760,24763],{"className":24759},[119],[59,24761,23138],{"className":24762},[119],[59,24764],{"className":24765,"style":24767},[119,24766],"rule","border-right-width:0em;border-top-width:1.5em;bottom:0em;",[59,24769],{"className":24770},[499,21653],[59,24772,24774,24778,24830,24833,24873,24876,24879,24882,24885,24888,24891],{"className":24773},[110],[59,24775],{"className":24776,"style":24777},[114],"height:1.1413em;vertical-align:-0.25em;",[59,24779,24781,24784],{"className":24780},[119],[59,24782,774],{"className":24783},[119,192],[59,24785,24787],{"className":24786},[196],[59,24788,24790,24822],{"className":24789},[131,132],[59,24791,24793,24819],{"className":24792},[136],[59,24794,24797,24808],{"className":24795,"style":24796},[140],"height:0.8913em;",[59,24798,24799,24802],{"style":17311},[59,24800],{"className":24801,"style":540},[148],[59,24803,24805],{"className":24804},[153,154,155,156],[59,24806,14051],{"className":24807,"style":3210},[119,192,156],[59,24809,24810,24813],{"style":16426},[59,24811],{"className":24812,"style":540},[148],[59,24814,24816],{"className":24815},[153,154,155,156],[59,24817,13242],{"className":24818,"style":3185},[119,192,156],[59,24820,227],{"className":24821},[226],[59,24823,24825],{"className":24824},[136],[59,24826,24828],{"className":24827,"style":17341},[140],[59,24829],{},[59,24831,798],{"className":24832},[123],[59,24834,24836,24839],{"className":24835},[119],[59,24837,12802],{"className":24838},[119,192],[59,24840,24842],{"className":24841},[196],[59,24843,24845,24865],{"className":24844},[131,132],[59,24846,24848,24862],{"className":24847},[136],[59,24849,24851],{"className":24850,"style":12732},[140],[59,24852,24853,24856],{"style":1251},[59,24854],{"className":24855,"style":540},[148],[59,24857,24859],{"className":24858},[153,154,155,156],[59,24860,777],{"className":24861},[119,192,156],[59,24863,227],{"className":24864},[226],[59,24866,24868],{"className":24867},[136],[59,24869,24871],{"className":24870,"style":1270},[140],[59,24872],{},[59,24874,803],{"className":24875},[912],[59,24877],{"className":24878,"style":916},[499],[59,24880,22],{"className":24881},[119,192],[59,24883,812],{"className":24884},[313],[59,24886],{"className":24887,"style":500},[499],[59,24889,815],{"className":24890},[504],[59,24892],{"className":24893,"style":500},[499],[59,24895,24897,24900,24940,24943,24946],{"className":24896},[110],[59,24898],{"className":24899,"style":21912},[114],[59,24901,24903,24906],{"className":24902},[119],[59,24904,7826],{"className":24905,"style":3210},[119,192],[59,24907,24909],{"className":24908},[196],[59,24910,24912,24932],{"className":24911},[131,132],[59,24913,24915,24929],{"className":24914},[136],[59,24916,24918],{"className":24917,"style":12732},[140],[59,24919,24920,24923],{"style":13105},[59,24921],{"className":24922,"style":540},[148],[59,24924,24926],{"className":24925},[153,154,155,156],[59,24927,777],{"className":24928},[119,192,156],[59,24930,227],{"className":24931},[226],[59,24933,24935],{"className":24934},[136],[59,24936,24938],{"className":24937,"style":1270},[140],[59,24939],{},[59,24941],{"className":24942,"style":1872},[499],[59,24944,5777],{"className":24945},[553],[59,24947],{"className":24948,"style":1872},[499],[59,24950,24952,24955,24958,24961,24964],{"className":24951},[110],[59,24953],{"className":24954,"style":24330},[114],[59,24956,18995],{"className":24957,"style":19242},[119,192],[59,24959],{"className":24960,"style":1872},[499],[59,24962,23049],{"className":24963},[553],[59,24965],{"className":24966,"style":1872},[499],[59,24968,24970,24973,25022,25025,25028],{"className":24969},[110],[59,24971],{"className":24972,"style":22558},[114],[59,24974,24976,24979],{"className":24975},[119],[59,24977,7826],{"className":24978,"style":3210},[119,192],[59,24980,24982],{"className":24981},[196],[59,24983,24985,25014],{"className":24984},[131,132],[59,24986,24988,25011],{"className":24987},[136],[59,24989,24991],{"className":24990,"style":3091},[140],[59,24992,24993,24996],{"style":13105},[59,24994],{"className":24995,"style":540},[148],[59,24997,24999],{"className":24998},[153,154,155,156],[59,25000,25002,25005,25008],{"className":25001},[119,156],[59,25003,777],{"className":25004},[119,192,156],[59,25006,5777],{"className":25007},[553,156],[59,25009,83],{"className":25010},[119,156],[59,25012,227],{"className":25013},[226],[59,25015,25017],{"className":25016},[136],[59,25018,25020],{"className":25019,"style":6938},[140],[59,25021],{},[59,25023],{"className":25024,"style":1872},[499],[59,25026,5777],{"className":25027},[553],[59,25029],{"className":25030,"style":1872},[499],[59,25032,25034,25037,25066,25069,25072],{"className":25033},[110],[59,25035],{"className":25036,"style":24413},[114],[59,25038,25040,25043],{"className":25039},[119],[59,25041,18995],{"className":25042,"style":19242},[119,192],[59,25044,25046],{"className":25045},[196],[59,25047,25049],{"className":25048},[131],[59,25050,25052],{"className":25051},[136],[59,25053,25055],{"className":25054,"style":23310},[140],[59,25056,25057,25060],{"style":16426},[59,25058],{"className":25059,"style":540},[148],[59,25061,25063],{"className":25062},[153,154,155,156],[59,25064,2821],{"className":25065},[119,156],[59,25067],{"className":25068,"style":1872},[499],[59,25070,23049],{"className":25071},[553],[59,25073],{"className":25074,"style":1872},[499],[59,25076,25078,25081,25130,25133,25136],{"className":25077},[110],[59,25079],{"className":25080,"style":22558},[114],[59,25082,25084,25087],{"className":25083},[119],[59,25085,7826],{"className":25086,"style":3210},[119,192],[59,25088,25090],{"className":25089},[196],[59,25091,25093,25122],{"className":25092},[131,132],[59,25094,25096,25119],{"className":25095},[136],[59,25097,25099],{"className":25098,"style":3091},[140],[59,25100,25101,25104],{"style":13105},[59,25102],{"className":25103,"style":540},[148],[59,25105,25107],{"className":25106},[153,154,155,156],[59,25108,25110,25113,25116],{"className":25109},[119,156],[59,25111,777],{"className":25112},[119,192,156],[59,25114,5777],{"className":25115},[553,156],[59,25117,2821],{"className":25118},[119,156],[59,25120,227],{"className":25121},[226],[59,25123,25125],{"className":25124},[136],[59,25126,25128],{"className":25127,"style":6938},[140],[59,25129],{},[59,25131],{"className":25132,"style":1872},[499],[59,25134,5777],{"className":25135},[553],[59,25137],{"className":25138,"style":1872},[499],[59,25140,25142,25145,25174,25177,25180],{"className":25141},[110],[59,25143],{"className":25144,"style":24413},[114],[59,25146,25148,25151],{"className":25147},[119],[59,25149,18995],{"className":25150,"style":19242},[119,192],[59,25152,25154],{"className":25153},[196],[59,25155,25157],{"className":25156},[131],[59,25158,25160],{"className":25159},[136],[59,25161,25163],{"className":25162,"style":23310},[140],[59,25164,25165,25168],{"style":16426},[59,25166],{"className":25167,"style":540},[148],[59,25169,25171],{"className":25170},[153,154,155,156],[59,25172,4671],{"className":25173},[119,156],[59,25175],{"className":25176,"style":1872},[499],[59,25178,23049],{"className":25179},[553],[59,25181],{"className":25182,"style":1872},[499],[59,25184,25186,25189,25238,25241,25244],{"className":25185},[110],[59,25187],{"className":25188,"style":22558},[114],[59,25190,25192,25195],{"className":25191},[119],[59,25193,7826],{"className":25194,"style":3210},[119,192],[59,25196,25198],{"className":25197},[196],[59,25199,25201,25230],{"className":25200},[131,132],[59,25202,25204,25227],{"className":25203},[136],[59,25205,25207],{"className":25206,"style":3091},[140],[59,25208,25209,25212],{"style":13105},[59,25210],{"className":25211,"style":540},[148],[59,25213,25215],{"className":25214},[153,154,155,156],[59,25216,25218,25221,25224],{"className":25217},[119,156],[59,25219,777],{"className":25220},[119,192,156],[59,25222,5777],{"className":25223},[553,156],[59,25225,4671],{"className":25226},[119,156],[59,25228,227],{"className":25229},[226],[59,25231,25233],{"className":25232},[136],[59,25234,25236],{"className":25235,"style":6938},[140],[59,25237],{},[59,25239],{"className":25240,"style":1872},[499],[59,25242,5777],{"className":25243},[553],[59,25245],{"className":25246,"style":1872},[499],[59,25248,25250,25254,25257,25260,25263],{"className":25249},[110],[59,25251],{"className":25252,"style":25253},[114],"height:0.6667em;vertical-align:-0.0833em;",[59,25255,23244],{"className":25256},[962],[59,25258],{"className":25259,"style":1872},[499],[59,25261,5777],{"className":25262},[553],[59,25264],{"className":25265,"style":1872},[499],[59,25267,25269,25273,25302,25305,25308],{"className":25268},[110],[59,25270],{"className":25271,"style":25272},[114],"height:1.0858em;vertical-align:-0.1944em;",[59,25274,25276,25279],{"className":25275},[119],[59,25277,18995],{"className":25278,"style":19242},[119,192],[59,25280,25282],{"className":25281},[196],[59,25283,25285],{"className":25284},[131],[59,25286,25288],{"className":25287},[136],[59,25289,25291],{"className":25290,"style":24796},[140],[59,25292,25293,25296],{"style":16426},[59,25294],{"className":25295,"style":540},[148],[59,25297,25299],{"className":25298},[153,154,155,156],[59,25300,13242],{"className":25301,"style":3185},[119,192,156],[59,25303],{"className":25304,"style":1872},[499],[59,25306,23049],{"className":25307},[553],[59,25309],{"className":25310,"style":1872},[499],[59,25312,25314,25317,25357,25360,25363],{"className":25313},[110],[59,25315],{"className":25316,"style":21912},[114],[59,25318,25320,25323],{"className":25319},[119],[59,25321,7826],{"className":25322,"style":3210},[119,192],[59,25324,25326],{"className":25325},[196],[59,25327,25329,25349],{"className":25328},[131,132],[59,25330,25332,25346],{"className":25331},[136],[59,25333,25335],{"className":25334,"style":13465},[140],[59,25336,25337,25340],{"style":13105},[59,25338],{"className":25339,"style":540},[148],[59,25341,25343],{"className":25342},[153,154,155,156],[59,25344,13242],{"className":25345,"style":3185},[119,192,156],[59,25347,227],{"className":25348},[226],[59,25350,25352],{"className":25351},[136],[59,25353,25355],{"className":25354,"style":1270},[140],[59,25356],{},[59,25358],{"className":25359,"style":1872},[499],[59,25361,18365],{"className":25362},[553],[59,25364],{"className":25365,"style":1872},[499],[59,25367,25369,25372,25412,25415,25455],{"className":25368},[110],[59,25370],{"className":25371,"style":877},[114],[59,25373,25375,25378],{"className":25374},[119],[59,25376,672],{"className":25377,"style":702},[119,192],[59,25379,25381],{"className":25380},[196],[59,25382,25384,25404],{"className":25383},[131,132],[59,25385,25387,25401],{"className":25386},[136],[59,25388,25390],{"className":25389,"style":1248},[140],[59,25391,25392,25395],{"style":20505},[59,25393],{"className":25394,"style":540},[148],[59,25396,25398],{"className":25397},[153,154,155,156],[59,25399,14051],{"className":25400,"style":3210},[119,192,156],[59,25402,227],{"className":25403},[226],[59,25405,25407],{"className":25406},[136],[59,25408,25410],{"className":25409,"style":1270},[140],[59,25411],{},[59,25413,798],{"className":25414},[123],[59,25416,25418,25421],{"className":25417},[119],[59,25419,12802],{"className":25420},[119,192],[59,25422,25424],{"className":25423},[196],[59,25425,25427,25447],{"className":25426},[131,132],[59,25428,25430,25444],{"className":25429},[136],[59,25431,25433],{"className":25432,"style":12732},[140],[59,25434,25435,25438],{"style":1251},[59,25436],{"className":25437,"style":540},[148],[59,25439,25441],{"className":25440},[153,154,155,156],[59,25442,777],{"className":25443},[119,192,156],[59,25445,227],{"className":25446},[226],[59,25448,25450],{"className":25449},[136],[59,25451,25453],{"className":25452,"style":1270},[140],[59,25454],{},[59,25456,812],{"className":25457},[313],[15,25459,25460,25461,25534,25535,25563],{},"为了让式子表示整洁，我们定义一个中间变量，令 ",[59,25462,25464,25483],{"className":25463},[62],[59,25465,25467],{"className":25466},[66],[68,25468,25469],{"xmlns":70},[72,25470,25471,25480],{},[75,25472,25473],{},[88,25474,25475,25478],{},[91,25476,25477],{},"δ",[91,25479,777],{},[98,25481,25482],{"encoding":100},"\\delta_t",[59,25484,25486],{"className":25485,"ariaHidden":106},[105],[59,25487,25489,25492],{"className":25488},[110],[59,25490],{"className":25491,"style":1229},[114],[59,25493,25495,25499],{"className":25494},[119],[59,25496,25477],{"className":25497,"style":25498},[119,192],"margin-right:0.03785em;",[59,25500,25502],{"className":25501},[196],[59,25503,25505,25526],{"className":25504},[131,132],[59,25506,25508,25523],{"className":25507},[136],[59,25509,25511],{"className":25510,"style":12732},[140],[59,25512,25514,25517],{"style":25513},"top:-2.55em;margin-left:-0.0379em;margin-right:0.05em;",[59,25515],{"className":25516,"style":540},[148],[59,25518,25520],{"className":25519},[153,154,155,156],[59,25521,777],{"className":25522},[119,192,156],[59,25524,227],{"className":25525},[226],[59,25527,25529],{"className":25528},[136],[59,25530,25532],{"className":25531,"style":1270},[140],[59,25533],{}," 表示在第 ",[59,25536,25538,25551],{"className":25537},[62],[59,25539,25541],{"className":25540},[66],[68,25542,25543],{"xmlns":70},[72,25544,25545,25549],{},[75,25546,25547],{},[91,25548,777],{},[98,25550,777],{"encoding":100},[59,25552,25554],{"className":25553,"ariaHidden":106},[105],[59,25555,25557,25560],{"className":25556},[110],[59,25558],{"className":25559,"style":12781},[114],[59,25561,777],{"className":25562},[119,192]," 步采取特定动作带来的优势。",[59,25565,25567],{"className":25566},[14350],[59,25568,25570,25896],{"className":25569},[62],[59,25571,25573],{"className":25572},[66],[68,25574,25575],{"xmlns":70,"display":9481},[72,25576,25577,25893],{},[75,25578,25579,25587,25589,25595,25597,25599,25601,25607,25609,25621,25623,25625,25631,25633,25639,25641,25643,25657,25659,25671,25673,25675,25677,25683,25685,25697,25699,25701,25707,25709,25721,25723,25725,25733,25735,25741,25743,25745,25747,25749,25757,25759,25767,25769,25775,25777,25779,25781,25783,25791,25793,25795,25809,25811,25819,25821,25827,25829,25831,25833,25835,25843,25845,25847,25861,25863,25869,25883,25885],{},[1822,25580,25581,25583,25585],{},[91,25582,25477],{},[91,25584,777],{},[91,25586,672],{},[461,25588,815],{},[88,25590,25591,25593],{},[91,25592,7826],{},[91,25594,777],{},[461,25596,5777],{},[91,25598,18995],{},[461,25600,23049],{},[88,25602,25603,25605],{},[91,25604,672],{},[91,25606,14051],{},[461,25608,798],{"stretchy":797},[88,25610,25611,25613],{},[91,25612,12802],{},[75,25614,25615,25617,25619],{},[91,25616,777],{},[461,25618,5777],{},[81,25620,83],{},[461,25622,812],{"stretchy":797},[461,25624,18365],{},[88,25626,25627,25629],{},[91,25628,672],{},[91,25630,14051],{},[461,25632,798],{"stretchy":797},[88,25634,25635,25637],{},[91,25636,12802],{},[91,25638,777],{},[461,25640,812],{"stretchy":797},[499,25642],{"linebreak":21653},[1822,25644,25645,25647,25655],{},[91,25646,25477],{},[75,25648,25649,25651,25653],{},[91,25650,777],{},[461,25652,5777],{},[81,25654,83],{},[91,25656,672],{},[461,25658,815],{},[88,25660,25661,25663],{},[91,25662,7826],{},[75,25664,25665,25667,25669],{},[91,25666,777],{},[461,25668,5777],{},[81,25670,83],{},[461,25672,5777],{},[91,25674,18995],{},[461,25676,23049],{},[88,25678,25679,25681],{},[91,25680,672],{},[91,25682,14051],{},[461,25684,798],{"stretchy":797},[88,25686,25687,25689],{},[91,25688,12802],{},[75,25690,25691,25693,25695],{},[91,25692,777],{},[461,25694,5777],{},[81,25696,2821],{},[461,25698,812],{"stretchy":797},[461,25700,18365],{},[88,25702,25703,25705],{},[91,25704,672],{},[91,25706,14051],{},[461,25708,798],{"stretchy":797},[88,25710,25711,25713],{},[91,25712,12802],{},[75,25714,25715,25717,25719],{},[91,25716,777],{},[461,25718,5777],{},[81,25720,83],{},[461,25722,812],{"stretchy":797},[499,25724],{"linebreak":21653},[1822,25726,25727,25729,25731],{},[91,25728,774],{},[91,25730,14051],{},[81,25732,83],{},[461,25734,798],{"stretchy":797},[88,25736,25737,25739],{},[91,25738,12802],{},[91,25740,777],{},[461,25742,803],{"separator":106},[91,25744,22],{},[461,25746,812],{"stretchy":797},[461,25748,815],{},[1822,25750,25751,25753,25755],{},[91,25752,25477],{},[91,25754,777],{},[91,25756,672],{},[499,25758],{"linebreak":21653},[1822,25760,25761,25763,25765],{},[91,25762,774],{},[91,25764,14051],{},[81,25766,2821],{},[461,25768,798],{"stretchy":797},[88,25770,25771,25773],{},[91,25772,12802],{},[91,25774,777],{},[461,25776,803],{"separator":106},[91,25778,22],{},[461,25780,812],{"stretchy":797},[461,25782,815],{},[1822,25784,25785,25787,25789],{},[91,25786,25477],{},[91,25788,777],{},[91,25790,672],{},[461,25792,5777],{},[91,25794,18995],{},[1822,25796,25797,25799,25807],{},[91,25798,25477],{},[75,25800,25801,25803,25805],{},[91,25802,777],{},[461,25804,5777],{},[81,25806,83],{},[91,25808,672],{},[499,25810],{"linebreak":21653},[1822,25812,25813,25815,25817],{},[91,25814,774],{},[91,25816,14051],{},[81,25818,4671],{},[461,25820,798],{"stretchy":797},[88,25822,25823,25825],{},[91,25824,12802],{},[91,25826,777],{},[461,25828,803],{"separator":106},[91,25830,22],{},[461,25832,812],{"stretchy":797},[461,25834,815],{},[1822,25836,25837,25839,25841],{},[91,25838,25477],{},[91,25840,777],{},[91,25842,672],{},[461,25844,5777],{},[91,25846,18995],{},[1822,25848,25849,25851,25859],{},[91,25850,25477],{},[75,25852,25853,25855,25857],{},[91,25854,777],{},[461,25856,5777],{},[81,25858,83],{},[91,25860,672],{},[461,25862,5777],{},[465,25864,25865,25867],{},[91,25866,18995],{},[81,25868,2821],{},[1822,25870,25871,25873,25881],{},[91,25872,25477],{},[75,25874,25875,25877,25879],{},[91,25876,777],{},[461,25878,5777],{},[81,25880,2821],{},[91,25882,672],{},[499,25884],{"linebreak":21653},[75,25886,25887,25889],{},[91,25888,23138],{"mathvariant":818},[23140,25890,25891],{"height":14366,"voffset":14366},[499,25892],{"mathbackground":23144,"width":14366,"height":23145},[98,25894,25895],{"encoding":100},"\n\\delta_t^V = r_t + \\gamma * V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n\n\\delta_{t+1}^V = r_{t+1} + \\gamma * V_\\theta(s_{t+2}) - V_\\theta(s_{t+1})\\\\\n\nA_\\theta^1(s_t, a) = \\delta_t^V\\\\\n\nA_\\theta^2(s_t, a) = \\delta_t^V + \\gamma \\delta_{t+1}^V\\\\\n\nA_\\theta^3(s_t, a) = \\delta_t^V + \\gamma \\delta_{t+1}^V + \\gamma^2 \\delta_{t+2}^V\\\\\n\n\\vdots",[59,25897,25899,25967,26022,26040,26150,26242,26245,26322,26386,26404,26514,26615,26618,26739,26796,26799,26920,26986,27055,27058,27179,27245,27323,27418,27421],{"className":25898,"ariaHidden":106},[105],[59,25900,25902,25906,25958,25961,25964],{"className":25901},[110],[59,25903],{"className":25904,"style":25905},[114],"height:1.1383em;vertical-align:-0.247em;",[59,25907,25909,25912],{"className":25908},[119],[59,25910,25477],{"className":25911,"style":25498},[119,192],[59,25913,25915],{"className":25914},[196],[59,25916,25918,25950],{"className":25917},[131,132],[59,25919,25921,25947],{"className":25920},[136],[59,25922,25924,25936],{"className":25923,"style":24796},[140],[59,25925,25927,25930],{"style":25926},"top:-2.453em;margin-left:-0.0379em;margin-right:0.05em;",[59,25928],{"className":25929,"style":540},[148],[59,25931,25933],{"className":25932},[153,154,155,156],[59,25934,777],{"className":25935},[119,192,156],[59,25937,25938,25941],{"style":16426},[59,25939],{"className":25940,"style":540},[148],[59,25942,25944],{"className":25943},[153,154,155,156],[59,25945,672],{"className":25946,"style":702},[119,192,156],[59,25948,227],{"className":25949},[226],[59,25951,25953],{"className":25952},[136],[59,25954,25956],{"className":25955,"style":17341},[140],[59,25957],{},[59,25959],{"className":25960,"style":500},[499],[59,25962,815],{"className":25963},[504],[59,25965],{"className":25966,"style":500},[499],[59,25968,25970,25973,26013,26016,26019],{"className":25969},[110],[59,25971],{"className":25972,"style":21912},[114],[59,25974,25976,25979],{"className":25975},[119],[59,25977,7826],{"className":25978,"style":3210},[119,192],[59,25980,25982],{"className":25981},[196],[59,25983,25985,26005],{"className":25984},[131,132],[59,25986,25988,26002],{"className":25987},[136],[59,25989,25991],{"className":25990,"style":12732},[140],[59,25992,25993,25996],{"style":13105},[59,25994],{"className":25995,"style":540},[148],[59,25997,25999],{"className":25998},[153,154,155,156],[59,26000,777],{"className":26001},[119,192,156],[59,26003,227],{"className":26004},[226],[59,26006,26008],{"className":26007},[136],[59,26009,26011],{"className":26010,"style":1270},[140],[59,26012],{},[59,26014],{"className":26015,"style":1872},[499],[59,26017,5777],{"className":26018},[553],[59,26020],{"className":26021,"style":1872},[499],[59,26023,26025,26028,26031,26034,26037],{"className":26024},[110],[59,26026],{"className":26027,"style":24330},[114],[59,26029,18995],{"className":26030,"style":19242},[119,192],[59,26032],{"className":26033,"style":1872},[499],[59,26035,23049],{"className":26036},[553],[59,26038],{"className":26039,"style":1872},[499],[59,26041,26043,26046,26086,26089,26138,26141,26144,26147],{"className":26042},[110],[59,26044],{"className":26045,"style":877},[114],[59,26047,26049,26052],{"className":26048},[119],[59,26050,672],{"className":26051,"style":702},[119,192],[59,26053,26055],{"className":26054},[196],[59,26056,26058,26078],{"className":26057},[131,132],[59,26059,26061,26075],{"className":26060},[136],[59,26062,26064],{"className":26063,"style":1248},[140],[59,26065,26066,26069],{"style":20505},[59,26067],{"className":26068,"style":540},[148],[59,26070,26072],{"className":26071},[153,154,155,156],[59,26073,14051],{"className":26074,"style":3210},[119,192,156],[59,26076,227],{"className":26077},[226],[59,26079,26081],{"className":26080},[136],[59,26082,26084],{"className":26083,"style":1270},[140],[59,26085],{},[59,26087,798],{"className":26088},[123],[59,26090,26092,26095],{"className":26091},[119],[59,26093,12802],{"className":26094},[119,192],[59,26096,26098],{"className":26097},[196],[59,26099,26101,26130],{"className":26100},[131,132],[59,26102,26104,26127],{"className":26103},[136],[59,26105,26107],{"className":26106,"style":3091},[140],[59,26108,26109,26112],{"style":1251},[59,26110],{"className":26111,"style":540},[148],[59,26113,26115],{"className":26114},[153,154,155,156],[59,26116,26118,26121,26124],{"className":26117},[119,156],[59,26119,777],{"className":26120},[119,192,156],[59,26122,5777],{"className":26123},[553,156],[59,26125,83],{"className":26126},[119,156],[59,26128,227],{"className":26129},[226],[59,26131,26133],{"className":26132},[136],[59,26134,26136],{"className":26135,"style":6938},[140],[59,26137],{},[59,26139,812],{"className":26140},[313],[59,26142],{"className":26143,"style":1872},[499],[59,26145,18365],{"className":26146},[553],[59,26148],{"className":26149,"style":1872},[499],[59,26151,26153,26156,26196,26199,26239],{"className":26152},[110],[59,26154],{"className":26155,"style":877},[114],[59,26157,26159,26162],{"className":26158},[119],[59,26160,672],{"className":26161,"style":702},[119,192],[59,26163,26165],{"className":26164},[196],[59,26166,26168,26188],{"className":26167},[131,132],[59,26169,26171,26185],{"className":26170},[136],[59,26172,26174],{"className":26173,"style":1248},[140],[59,26175,26176,26179],{"style":20505},[59,26177],{"className":26178,"style":540},[148],[59,26180,26182],{"className":26181},[153,154,155,156],[59,26183,14051],{"className":26184,"style":3210},[119,192,156],[59,26186,227],{"className":26187},[226],[59,26189,26191],{"className":26190},[136],[59,26192,26194],{"className":26193,"style":1270},[140],[59,26195],{},[59,26197,798],{"className":26198},[123],[59,26200,26202,26205],{"className":26201},[119],[59,26203,12802],{"className":26204},[119,192],[59,26206,26208],{"className":26207},[196],[59,26209,26211,26231],{"className":26210},[131,132],[59,26212,26214,26228],{"className":26213},[136],[59,26215,26217],{"className":26216,"style":12732},[140],[59,26218,26219,26222],{"style":1251},[59,26220],{"className":26221,"style":540},[148],[59,26223,26225],{"className":26224},[153,154,155,156],[59,26226,777],{"className":26227},[119,192,156],[59,26229,227],{"className":26230},[226],[59,26232,26234],{"className":26233},[136],[59,26235,26237],{"className":26236,"style":1270},[140],[59,26238],{},[59,26240,812],{"className":26241},[313],[59,26243],{"className":26244},[499,21653],[59,26246,26248,26252,26313,26316,26319],{"className":26247},[110],[59,26249],{"className":26250,"style":26251},[114],"height:1.1967em;vertical-align:-0.3053em;",[59,26253,26255,26258],{"className":26254},[119],[59,26256,25477],{"className":26257,"style":25498},[119,192],[59,26259,26261],{"className":26260},[196],[59,26262,26264,26304],{"className":26263},[131,132],[59,26265,26267,26301],{"className":26266},[136],[59,26268,26270,26290],{"className":26269,"style":24796},[140],[59,26271,26272,26275],{"style":25926},[59,26273],{"className":26274,"style":540},[148],[59,26276,26278],{"className":26277},[153,154,155,156],[59,26279,26281,26284,26287],{"className":26280},[119,156],[59,26282,777],{"className":26283},[119,192,156],[59,26285,5777],{"className":26286},[553,156],[59,26288,83],{"className":26289},[119,156],[59,26291,26292,26295],{"style":16426},[59,26293],{"className":26294,"style":540},[148],[59,26296,26298],{"className":26297},[153,154,155,156],[59,26299,672],{"className":26300,"style":702},[119,192,156],[59,26302,227],{"className":26303},[226],[59,26305,26307],{"className":26306},[136],[59,26308,26311],{"className":26309,"style":26310},[140],"height:0.3053em;",[59,26312],{},[59,26314],{"className":26315,"style":500},[499],[59,26317,815],{"className":26318},[504],[59,26320],{"className":26321,"style":500},[499],[59,26323,26325,26328,26377,26380,26383],{"className":26324},[110],[59,26326],{"className":26327,"style":22558},[114],[59,26329,26331,26334],{"className":26330},[119],[59,26332,7826],{"className":26333,"style":3210},[119,192],[59,26335,26337],{"className":26336},[196],[59,26338,26340,26369],{"className":26339},[131,132],[59,26341,26343,26366],{"className":26342},[136],[59,26344,26346],{"className":26345,"style":3091},[140],[59,26347,26348,26351],{"style":13105},[59,26349],{"className":26350,"style":540},[148],[59,26352,26354],{"className":26353},[153,154,155,156],[59,26355,26357,26360,26363],{"className":26356},[119,156],[59,26358,777],{"className":26359},[119,192,156],[59,26361,5777],{"className":26362},[553,156],[59,26364,83],{"className":26365},[119,156],[59,26367,227],{"className":26368},[226],[59,26370,26372],{"className":26371},[136],[59,26373,26375],{"className":26374,"style":6938},[140],[59,26376],{},[59,26378],{"className":26379,"style":1872},[499],[59,26381,5777],{"className":26382},[553],[59,26384],{"className":26385,"style":1872},[499],[59,26387,26389,26392,26395,26398,26401],{"className":26388},[110],[59,26390],{"className":26391,"style":24330},[114],[59,26393,18995],{"className":26394,"style":19242},[119,192],[59,26396],{"className":26397,"style":1872},[499],[59,26399,23049],{"className":26400},[553],[59,26402],{"className":26403,"style":1872},[499],[59,26405,26407,26410,26450,26453,26502,26505,26508,26511],{"className":26406},[110],[59,26408],{"className":26409,"style":877},[114],[59,26411,26413,26416],{"className":26412},[119],[59,26414,672],{"className":26415,"style":702},[119,192],[59,26417,26419],{"className":26418},[196],[59,26420,26422,26442],{"className":26421},[131,132],[59,26423,26425,26439],{"className":26424},[136],[59,26426,26428],{"className":26427,"style":1248},[140],[59,26429,26430,26433],{"style":20505},[59,26431],{"className":26432,"style":540},[148],[59,26434,26436],{"className":26435},[153,154,155,156],[59,26437,14051],{"className":26438,"style":3210},[119,192,156],[59,26440,227],{"className":26441},[226],[59,26443,26445],{"className":26444},[136],[59,26446,26448],{"className":26447,"style":1270},[140],[59,26449],{},[59,26451,798],{"className":26452},[123],[59,26454,26456,26459],{"className":26455},[119],[59,26457,12802],{"className":26458},[119,192],[59,26460,26462],{"className":26461},[196],[59,26463,26465,26494],{"className":26464},[131,132],[59,26466,26468,26491],{"className":26467},[136],[59,26469,26471],{"className":26470,"style":3091},[140],[59,26472,26473,26476],{"style":1251},[59,26474],{"className":26475,"style":540},[148],[59,26477,26479],{"className":26478},[153,154,155,156],[59,26480,26482,26485,26488],{"className":26481},[119,156],[59,26483,777],{"className":26484},[119,192,156],[59,26486,5777],{"className":26487},[553,156],[59,26489,2821],{"className":26490},[119,156],[59,26492,227],{"className":26493},[226],[59,26495,26497],{"className":26496},[136],[59,26498,26500],{"className":26499,"style":6938},[140],[59,26501],{},[59,26503,812],{"className":26504},[313],[59,26506],{"className":26507,"style":1872},[499],[59,26509,18365],{"className":26510},[553],[59,26512],{"className":26513,"style":1872},[499],[59,26515,26517,26520,26560,26563,26612],{"className":26516},[110],[59,26518],{"className":26519,"style":877},[114],[59,26521,26523,26526],{"className":26522},[119],[59,26524,672],{"className":26525,"style":702},[119,192],[59,26527,26529],{"className":26528},[196],[59,26530,26532,26552],{"className":26531},[131,132],[59,26533,26535,26549],{"className":26534},[136],[59,26536,26538],{"className":26537,"style":1248},[140],[59,26539,26540,26543],{"style":20505},[59,26541],{"className":26542,"style":540},[148],[59,26544,26546],{"className":26545},[153,154,155,156],[59,26547,14051],{"className":26548,"style":3210},[119,192,156],[59,26550,227],{"className":26551},[226],[59,26553,26555],{"className":26554},[136],[59,26556,26558],{"className":26557,"style":1270},[140],[59,26559],{},[59,26561,798],{"className":26562},[123],[59,26564,26566,26569],{"className":26565},[119],[59,26567,12802],{"className":26568},[119,192],[59,26570,26572],{"className":26571},[196],[59,26573,26575,26604],{"className":26574},[131,132],[59,26576,26578,26601],{"className":26577},[136],[59,26579,26581],{"className":26580,"style":3091},[140],[59,26582,26583,26586],{"style":1251},[59,26584],{"className":26585,"style":540},[148],[59,26587,26589],{"className":26588},[153,154,155,156],[59,26590,26592,26595,26598],{"className":26591},[119,156],[59,26593,777],{"className":26594},[119,192,156],[59,26596,5777],{"className":26597},[553,156],[59,26599,83],{"className":26600},[119,156],[59,26602,227],{"className":26603},[226],[59,26605,26607],{"className":26606},[136],[59,26608,26610],{"className":26609,"style":6938},[140],[59,26611],{},[59,26613,812],{"className":26614},[313],[59,26616],{"className":26617},[499,21653],[59,26619,26621,26624,26675,26678,26718,26721,26724,26727,26730,26733,26736],{"className":26620},[110],[59,26622],{"className":26623,"style":23291},[114],[59,26625,26627,26630],{"className":26626},[119],[59,26628,774],{"className":26629},[119,192],[59,26631,26633],{"className":26632},[196],[59,26634,26636,26667],{"className":26635},[131,132],[59,26637,26639,26664],{"className":26638},[136],[59,26640,26642,26653],{"className":26641,"style":23310},[140],[59,26643,26644,26647],{"style":17311},[59,26645],{"className":26646,"style":540},[148],[59,26648,26650],{"className":26649},[153,154,155,156],[59,26651,14051],{"className":26652,"style":3210},[119,192,156],[59,26654,26655,26658],{"style":16426},[59,26656],{"className":26657,"style":540},[148],[59,26659,26661],{"className":26660},[153,154,155,156],[59,26662,83],{"className":26663},[119,156],[59,26665,227],{"className":26666},[226],[59,26668,26670],{"className":26669},[136],[59,26671,26673],{"className":26672,"style":17341},[140],[59,26674],{},[59,26676,798],{"className":26677},[123],[59,26679,26681,26684],{"className":26680},[119],[59,26682,12802],{"className":26683},[119,192],[59,26685,26687],{"className":26686},[196],[59,26688,26690,26710],{"className":26689},[131,132],[59,26691,26693,26707],{"className":26692},[136],[59,26694,26696],{"className":26695,"style":12732},[140],[59,26697,26698,26701],{"style":1251},[59,26699],{"className":26700,"style":540},[148],[59,26702,26704],{"className":26703},[153,154,155,156],[59,26705,777],{"className":26706},[119,192,156],[59,26708,227],{"className":26709},[226],[59,26711,26713],{"className":26712},[136],[59,26714,26716],{"className":26715,"style":1270},[140],[59,26717],{},[59,26719,803],{"className":26720},[912],[59,26722],{"className":26723,"style":916},[499],[59,26725,22],{"className":26726},[119,192],[59,26728,812],{"className":26729},[313],[59,26731],{"className":26732,"style":500},[499],[59,26734,815],{"className":26735},[504],[59,26737],{"className":26738,"style":500},[499],[59,26740,26742,26745],{"className":26741},[110],[59,26743],{"className":26744,"style":25905},[114],[59,26746,26748,26751],{"className":26747},[119],[59,26749,25477],{"className":26750,"style":25498},[119,192],[59,26752,26754],{"className":26753},[196],[59,26755,26757,26788],{"className":26756},[131,132],[59,26758,26760,26785],{"className":26759},[136],[59,26761,26763,26774],{"className":26762,"style":24796},[140],[59,26764,26765,26768],{"style":25926},[59,26766],{"className":26767,"style":540},[148],[59,26769,26771],{"className":26770},[153,154,155,156],[59,26772,777],{"className":26773},[119,192,156],[59,26775,26776,26779],{"style":16426},[59,26777],{"className":26778,"style":540},[148],[59,26780,26782],{"className":26781},[153,154,155,156],[59,26783,672],{"className":26784,"style":702},[119,192,156],[59,26786,227],{"className":26787},[226],[59,26789,26791],{"className":26790},[136],[59,26792,26794],{"className":26793,"style":17341},[140],[59,26795],{},[59,26797],{"className":26798},[499,21653],[59,26800,26802,26805,26856,26859,26899,26902,26905,26908,26911,26914,26917],{"className":26801},[110],[59,26803],{"className":26804,"style":23291},[114],[59,26806,26808,26811],{"className":26807},[119],[59,26809,774],{"className":26810},[119,192],[59,26812,26814],{"className":26813},[196],[59,26815,26817,26848],{"className":26816},[131,132],[59,26818,26820,26845],{"className":26819},[136],[59,26821,26823,26834],{"className":26822,"style":23310},[140],[59,26824,26825,26828],{"style":17311},[59,26826],{"className":26827,"style":540},[148],[59,26829,26831],{"className":26830},[153,154,155,156],[59,26832,14051],{"className":26833,"style":3210},[119,192,156],[59,26835,26836,26839],{"style":16426},[59,26837],{"className":26838,"style":540},[148],[59,26840,26842],{"className":26841},[153,154,155,156],[59,26843,2821],{"className":26844},[119,156],[59,26846,227],{"className":26847},[226],[59,26849,26851],{"className":26850},[136],[59,26852,26854],{"className":26853,"style":17341},[140],[59,26855],{},[59,26857,798],{"className":26858},[123],[59,26860,26862,26865],{"className":26861},[119],[59,26863,12802],{"className":26864},[119,192],[59,26866,26868],{"className":26867},[196],[59,26869,26871,26891],{"className":26870},[131,132],[59,26872,26874,26888],{"className":26873},[136],[59,26875,26877],{"className":26876,"style":12732},[140],[59,26878,26879,26882],{"style":1251},[59,26880],{"className":26881,"style":540},[148],[59,26883,26885],{"className":26884},[153,154,155,156],[59,26886,777],{"className":26887},[119,192,156],[59,26889,227],{"className":26890},[226],[59,26892,26894],{"className":26893},[136],[59,26895,26897],{"className":26896,"style":1270},[140],[59,26898],{},[59,26900,803],{"className":26901},[912],[59,26903],{"className":26904,"style":916},[499],[59,26906,22],{"className":26907},[119,192],[59,26909,812],{"className":26910},[313],[59,26912],{"className":26913,"style":500},[499],[59,26915,815],{"className":26916},[504],[59,26918],{"className":26919,"style":500},[499],[59,26921,26923,26926,26977,26980,26983],{"className":26922},[110],[59,26924],{"className":26925,"style":25905},[114],[59,26927,26929,26932],{"className":26928},[119],[59,26930,25477],{"className":26931,"style":25498},[119,192],[59,26933,26935],{"className":26934},[196],[59,26936,26938,26969],{"className":26937},[131,132],[59,26939,26941,26966],{"className":26940},[136],[59,26942,26944,26955],{"className":26943,"style":24796},[140],[59,26945,26946,26949],{"style":25926},[59,26947],{"className":26948,"style":540},[148],[59,26950,26952],{"className":26951},[153,154,155,156],[59,26953,777],{"className":26954},[119,192,156],[59,26956,26957,26960],{"style":16426},[59,26958],{"className":26959,"style":540},[148],[59,26961,26963],{"className":26962},[153,154,155,156],[59,26964,672],{"className":26965,"style":702},[119,192,156],[59,26967,227],{"className":26968},[226],[59,26970,26972],{"className":26971},[136],[59,26973,26975],{"className":26974,"style":17341},[140],[59,26976],{},[59,26978],{"className":26979,"style":1872},[499],[59,26981,5777],{"className":26982},[553],[59,26984],{"className":26985,"style":1872},[499],[59,26987,26989,26992,26995],{"className":26988},[110],[59,26990],{"className":26991,"style":26251},[114],[59,26993,18995],{"className":26994,"style":19242},[119,192],[59,26996,26998,27001],{"className":26997},[119],[59,26999,25477],{"className":27000,"style":25498},[119,192],[59,27002,27004],{"className":27003},[196],[59,27005,27007,27047],{"className":27006},[131,132],[59,27008,27010,27044],{"className":27009},[136],[59,27011,27013,27033],{"className":27012,"style":24796},[140],[59,27014,27015,27018],{"style":25926},[59,27016],{"className":27017,"style":540},[148],[59,27019,27021],{"className":27020},[153,154,155,156],[59,27022,27024,27027,27030],{"className":27023},[119,156],[59,27025,777],{"className":27026},[119,192,156],[59,27028,5777],{"className":27029},[553,156],[59,27031,83],{"className":27032},[119,156],[59,27034,27035,27038],{"style":16426},[59,27036],{"className":27037,"style":540},[148],[59,27039,27041],{"className":27040},[153,154,155,156],[59,27042,672],{"className":27043,"style":702},[119,192,156],[59,27045,227],{"className":27046},[226],[59,27048,27050],{"className":27049},[136],[59,27051,27053],{"className":27052,"style":26310},[140],[59,27054],{},[59,27056],{"className":27057},[499,21653],[59,27059,27061,27064,27115,27118,27158,27161,27164,27167,27170,27173,27176],{"className":27060},[110],[59,27062],{"className":27063,"style":23291},[114],[59,27065,27067,27070],{"className":27066},[119],[59,27068,774],{"className":27069},[119,192],[59,27071,27073],{"className":27072},[196],[59,27074,27076,27107],{"className":27075},[131,132],[59,27077,27079,27104],{"className":27078},[136],[59,27080,27082,27093],{"className":27081,"style":23310},[140],[59,27083,27084,27087],{"style":17311},[59,27085],{"className":27086,"style":540},[148],[59,27088,27090],{"className":27089},[153,154,155,156],[59,27091,14051],{"className":27092,"style":3210},[119,192,156],[59,27094,27095,27098],{"style":16426},[59,27096],{"className":27097,"style":540},[148],[59,27099,27101],{"className":27100},[153,154,155,156],[59,27102,4671],{"className":27103},[119,156],[59,27105,227],{"className":27106},[226],[59,27108,27110],{"className":27109},[136],[59,27111,27113],{"className":27112,"style":17341},[140],[59,27114],{},[59,27116,798],{"className":27117},[123],[59,27119,27121,27124],{"className":27120},[119],[59,27122,12802],{"className":27123},[119,192],[59,27125,27127],{"className":27126},[196],[59,27128,27130,27150],{"className":27129},[131,132],[59,27131,27133,27147],{"className":27132},[136],[59,27134,27136],{"className":27135,"style":12732},[140],[59,27137,27138,27141],{"style":1251},[59,27139],{"className":27140,"style":540},[148],[59,27142,27144],{"className":27143},[153,154,155,156],[59,27145,777],{"className":27146},[119,192,156],[59,27148,227],{"className":27149},[226],[59,27151,27153],{"className":27152},[136],[59,27154,27156],{"className":27155,"style":1270},[140],[59,27157],{},[59,27159,803],{"className":27160},[912],[59,27162],{"className":27163,"style":916},[499],[59,27165,22],{"className":27166},[119,192],[59,27168,812],{"className":27169},[313],[59,27171],{"className":27172,"style":500},[499],[59,27174,815],{"className":27175},[504],[59,27177],{"className":27178,"style":500},[499],[59,27180,27182,27185,27236,27239,27242],{"className":27181},[110],[59,27183],{"className":27184,"style":25905},[114],[59,27186,27188,27191],{"className":27187},[119],[59,27189,25477],{"className":27190,"style":25498},[119,192],[59,27192,27194],{"className":27193},[196],[59,27195,27197,27228],{"className":27196},[131,132],[59,27198,27200,27225],{"className":27199},[136],[59,27201,27203,27214],{"className":27202,"style":24796},[140],[59,27204,27205,27208],{"style":25926},[59,27206],{"className":27207,"style":540},[148],[59,27209,27211],{"className":27210},[153,154,155,156],[59,27212,777],{"className":27213},[119,192,156],[59,27215,27216,27219],{"style":16426},[59,27217],{"className":27218,"style":540},[148],[59,27220,27222],{"className":27221},[153,154,155,156],[59,27223,672],{"className":27224,"style":702},[119,192,156],[59,27226,227],{"className":27227},[226],[59,27229,27231],{"className":27230},[136],[59,27232,27234],{"className":27233,"style":17341},[140],[59,27235],{},[59,27237],{"className":27238,"style":1872},[499],[59,27240,5777],{"className":27241},[553],[59,27243],{"className":27244,"style":1872},[499],[59,27246,27248,27251,27254,27314,27317,27320],{"className":27247},[110],[59,27249],{"className":27250,"style":26251},[114],[59,27252,18995],{"className":27253,"style":19242},[119,192],[59,27255,27257,27260],{"className":27256},[119],[59,27258,25477],{"className":27259,"style":25498},[119,192],[59,27261,27263],{"className":27262},[196],[59,27264,27266,27306],{"className":27265},[131,132],[59,27267,27269,27303],{"className":27268},[136],[59,27270,27272,27292],{"className":27271,"style":24796},[140],[59,27273,27274,27277],{"style":25926},[59,27275],{"className":27276,"style":540},[148],[59,27278,27280],{"className":27279},[153,154,155,156],[59,27281,27283,27286,27289],{"className":27282},[119,156],[59,27284,777],{"className":27285},[119,192,156],[59,27287,5777],{"className":27288},[553,156],[59,27290,83],{"className":27291},[119,156],[59,27293,27294,27297],{"style":16426},[59,27295],{"className":27296,"style":540},[148],[59,27298,27300],{"className":27299},[153,154,155,156],[59,27301,672],{"className":27302,"style":702},[119,192,156],[59,27304,227],{"className":27305},[226],[59,27307,27309],{"className":27308},[136],[59,27310,27312],{"className":27311,"style":26310},[140],[59,27313],{},[59,27315],{"className":27316,"style":1872},[499],[59,27318,5777],{"className":27319},[553],[59,27321],{"className":27322,"style":1872},[499],[59,27324,27326,27329,27358],{"className":27325},[110],[59,27327],{"className":27328,"style":26251},[114],[59,27330,27332,27335],{"className":27331},[119],[59,27333,18995],{"className":27334,"style":19242},[119,192],[59,27336,27338],{"className":27337},[196],[59,27339,27341],{"className":27340},[131],[59,27342,27344],{"className":27343},[136],[59,27345,27347],{"className":27346,"style":23310},[140],[59,27348,27349,27352],{"style":16426},[59,27350],{"className":27351,"style":540},[148],[59,27353,27355],{"className":27354},[153,154,155,156],[59,27356,2821],{"className":27357},[119,156],[59,27359,27361,27364],{"className":27360},[119],[59,27362,25477],{"className":27363,"style":25498},[119,192],[59,27365,27367],{"className":27366},[196],[59,27368,27370,27410],{"className":27369},[131,132],[59,27371,27373,27407],{"className":27372},[136],[59,27374,27376,27396],{"className":27375,"style":24796},[140],[59,27377,27378,27381],{"style":25926},[59,27379],{"className":27380,"style":540},[148],[59,27382,27384],{"className":27383},[153,154,155,156],[59,27385,27387,27390,27393],{"className":27386},[119,156],[59,27388,777],{"className":27389},[119,192,156],[59,27391,5777],{"className":27392},[553,156],[59,27394,2821],{"className":27395},[119,156],[59,27397,27398,27401],{"style":16426},[59,27399],{"className":27400,"style":540},[148],[59,27402,27404],{"className":27403},[153,154,155,156],[59,27405,672],{"className":27406,"style":702},[119,192,156],[59,27408,227],{"className":27409},[226],[59,27411,27413],{"className":27412},[136],[59,27414,27416],{"className":27415,"style":26310},[140],[59,27417],{},[59,27419],{"className":27420},[499,21653],[59,27422,27424,27427],{"className":27423},[110],[59,27425],{"className":27426,"style":24756},[114],[59,27428,27430,27433],{"className":27429},[119],[59,27431,23138],{"className":27432},[119],[59,27434],{"className":27435,"style":24767},[119,24766],[49,27437,27439],{"id":27438},"generalized-advantage-estimation-gae","Generalized Advantage Estimation (GAE)",[15,27441,27442],{},"这时我们考虑，在运行时我们应该向后采样几步呢？GAE 的答案是，小孩子才做选择，我全都要！",[15,27444,27445,27446,27476],{},"GAE考虑全部的采样结果，并引入一个衰减因子 ",[59,27447,27449,27464],{"className":27448},[62],[59,27450,27452],{"className":27451},[66],[68,27453,27454],{"xmlns":70},[72,27455,27456,27461],{},[75,27457,27458],{},[91,27459,27460],{},"λ",[98,27462,27463],{"encoding":100},"\\lambda",[59,27465,27467],{"className":27466,"ariaHidden":106},[105],[59,27468,27470,27473],{"className":27469},[110],[59,27471],{"className":27472,"style":407},[114],[59,27474,27460],{"className":27475},[119,192]," 。",[59,27478,27480],{"className":27479},[14350],[59,27481,27483,28074],{"className":27482},[62],[59,27484,27486],{"className":27485},[66],[68,27487,27488],{"xmlns":70,"display":9481},[72,27489,27490,28071],{},[14362,27491,27492,27670,27812,27922,28012],{"rowspacing":14364,"columnalign":14365,"columnspacing":14366},[14368,27493,27494,27529],{},[14371,27495,27496],{},[14374,27497,27498],{"scriptlevel":1754,"displaystyle":106},[75,27499,27500,27515,27517,27523,27525,27527],{},[1822,27501,27502,27504,27506],{},[91,27503,774],{},[91,27505,14051],{},[75,27507,27508,27511,27513],{},[91,27509,27510],{},"G",[91,27512,774],{},[91,27514,5607],{},[461,27516,798],{"stretchy":797},[88,27518,27519,27521],{},[91,27520,12802],{},[91,27522,777],{},[461,27524,803],{"separator":106},[91,27526,22],{},[461,27528,812],{"stretchy":797},[14371,27530,27531],{},[14374,27532,27533],{"scriptlevel":1754,"displaystyle":106},[75,27534,27535,27537,27539,27541,27543,27545,27547,27549,27551,27559,27561,27563,27565,27573,27575,27581,27589,27591,27593,27596,27598,27601,27603,27605,27607,27610,27612,27614,27628,27630,27632,27640,27642,27645,27653,27655,27658,27666,27668],{},[75,27536],{},[461,27538,815],{},[461,27540,798],{"stretchy":797},[81,27542,83],{},[461,27544,18365],{},[91,27546,27460],{},[461,27548,812],{"stretchy":797},[461,27550,798],{"stretchy":797},[1822,27552,27553,27555,27557],{},[91,27554,774],{},[91,27556,14051],{},[81,27558,83],{},[461,27560,5777],{},[91,27562,27460],{},[461,27564,23049],{},[1822,27566,27567,27569,27571],{},[91,27568,774],{},[91,27570,14051],{},[81,27572,2821],{},[461,27574,5777],{},[465,27576,27577,27579],{},[91,27578,27460],{},[81,27580,2821],{},[1822,27582,27583,27585,27587],{},[91,27584,774],{},[91,27586,14051],{},[81,27588,4671],{},[461,27590,5777],{},[461,27592,23244],{},[3277,27594,27595],{}," ",[461,27597,812],{"stretchy":797},[499,27599],{"width":27600},"1em",[499,27602],{"width":27600},[91,27604,27460],{},[461,27606,815],{},[81,27608,27609],{},"0.9",[461,27611,7184],{},[499,27613],{"width":27600},[1822,27615,27616,27618,27620],{},[91,27617,774],{},[91,27619,14051],{},[75,27621,27622,27624,27626],{},[91,27623,27510],{},[91,27625,774],{},[91,27627,5607],{},[461,27629,815],{},[81,27631,12466],{},[1822,27633,27634,27636,27638],{},[91,27635,774],{},[91,27637,14051],{},[81,27639,83],{},[461,27641,5777],{},[81,27643,27644],{},"0.09",[1822,27646,27647,27649,27651],{},[91,27648,774],{},[91,27650,14051],{},[81,27652,2821],{},[461,27654,5777],{},[81,27656,27657],{},"0.081",[1822,27659,27660,27662,27664],{},[91,27661,774],{},[91,27663,14051],{},[81,27665,4671],{},[461,27667,5777],{},[461,27669,23244],{},[14368,27671,27672,27678],{},[14371,27673,27674],{},[14374,27675,27676],{"scriptlevel":1754,"displaystyle":106},[75,27677],{},[14371,27679,27680],{},[14374,27681,27682],{"scriptlevel":1754,"displaystyle":106},[75,27683,27684,27686,27688,27690,27692,27694,27696,27698,27700,27708,27710,27712,27714,27716,27724,27726,27728,27742,27744,27746,27752,27754,27762,27764,27766,27780,27782,27788,27802,27804,27806,27808,27810],{},[75,27685],{},[461,27687,815],{},[461,27689,798],{"stretchy":797},[81,27691,83],{},[461,27693,18365],{},[91,27695,27460],{},[461,27697,812],{"stretchy":797},[461,27699,798],{"stretchy":797},[1822,27701,27702,27704,27706],{},[91,27703,25477],{},[91,27705,777],{},[91,27707,672],{},[461,27709,5777],{},[91,27711,27460],{},[461,27713,23049],{},[461,27715,798],{"stretchy":797},[1822,27717,27718,27720,27722],{},[91,27719,25477],{},[91,27721,777],{},[91,27723,672],{},[461,27725,5777],{},[91,27727,18995],{},[1822,27729,27730,27732,27740],{},[91,27731,25477],{},[75,27733,27734,27736,27738],{},[91,27735,777],{},[461,27737,5777],{},[81,27739,83],{},[91,27741,672],{},[461,27743,812],{"stretchy":797},[461,27745,5777],{},[465,27747,27748,27750],{},[91,27749,27460],{},[81,27751,2821],{},[461,27753,798],{"stretchy":797},[1822,27755,27756,27758,27760],{},[91,27757,25477],{},[91,27759,777],{},[91,27761,672],{},[461,27763,5777],{},[91,27765,18995],{},[1822,27767,27768,27770,27778],{},[91,27769,25477],{},[75,27771,27772,27774,27776],{},[91,27773,777],{},[461,27775,5777],{},[81,27777,83],{},[91,27779,672],{},[461,27781,5777],{},[465,27783,27784,27786],{},[91,27785,18995],{},[81,27787,2821],{},[1822,27789,27790,27792,27800],{},[91,27791,25477],{},[75,27793,27794,27796,27798],{},[91,27795,777],{},[461,27797,5777],{},[81,27799,2821],{},[91,27801,672],{},[461,27803,812],{"stretchy":797},[461,27805,5777],{},[461,27807,23244],{},[3277,27809,27595],{},[461,27811,812],{"stretchy":797},[14368,27813,27814,27820],{},[14371,27815,27816],{},[14374,27817,27818],{"scriptlevel":1754,"displaystyle":106},[75,27819],{},[14371,27821,27822],{},[14374,27823,27824],{"scriptlevel":1754,"displaystyle":106},[75,27825,27826,27828,27830,27832,27834,27836,27838,27840,27842,27850,27852,27854,27856,27858,27860,27866,27868,27870,27872,27874,27876,27878,27892,27894,27896,27898,27900,27906,27908,27910,27912,27914,27916,27918,27920],{},[75,27827],{},[461,27829,815],{},[461,27831,798],{"stretchy":797},[81,27833,83],{},[461,27835,18365],{},[91,27837,27460],{},[461,27839,812],{"stretchy":797},[461,27841,798],{"stretchy":797},[1822,27843,27844,27846,27848],{},[91,27845,25477],{},[91,27847,777],{},[91,27849,672],{},[461,27851,798],{"stretchy":797},[81,27853,83],{},[461,27855,5777],{},[91,27857,27460],{},[461,27859,5777],{},[465,27861,27862,27864],{},[91,27863,27460],{},[81,27865,2821],{},[461,27867,5777],{},[461,27869,23244],{},[3277,27871,27595],{},[461,27873,812],{"stretchy":797},[461,27875,5777],{},[91,27877,18995],{},[1822,27879,27880,27882,27890],{},[91,27881,25477],{},[75,27883,27884,27886,27888],{},[91,27885,777],{},[461,27887,5777],{},[81,27889,83],{},[91,27891,672],{},[461,27893,23049],{},[461,27895,798],{"stretchy":797},[91,27897,27460],{},[461,27899,5777],{},[465,27901,27902,27904],{},[91,27903,27460],{},[81,27905,2821],{},[461,27907,5777],{},[461,27909,23244],{},[3277,27911,27595],{},[461,27913,812],{"stretchy":797},[461,27915,5777],{},[461,27917,23244],{},[3277,27919,27595],{},[461,27921,812],{"stretchy":797},[14368,27923,27924,27930],{},[14371,27925,27926],{},[14374,27927,27928],{"scriptlevel":1754,"displaystyle":106},[75,27929],{},[14371,27931,27932],{},[14374,27933,27934],{"scriptlevel":1754,"displaystyle":106},[75,27935,27936,27938,27940,27942,27944,27946,27948,27950],{},[75,27937],{},[461,27939,815],{},[461,27941,798],{"stretchy":797},[81,27943,83],{},[461,27945,18365],{},[91,27947,27460],{},[461,27949,812],{"stretchy":797},[75,27951,27952,27954,27962,27974,27976,27978,27992,28004,28006,28008,28010],{},[461,27953,798],{"fence":106},[1822,27955,27956,27958,27960],{},[91,27957,25477],{},[91,27959,777],{},[91,27961,672],{},[78,27963,27964,27966],{},[81,27965,83],{},[75,27967,27968,27970,27972],{},[81,27969,83],{},[461,27971,18365],{},[91,27973,27460],{},[461,27975,5777],{},[91,27977,18995],{},[1822,27979,27980,27982,27990],{},[91,27981,25477],{},[75,27983,27984,27986,27988],{},[91,27985,777],{},[461,27987,5777],{},[81,27989,83],{},[91,27991,672],{},[78,27993,27994,27996],{},[91,27995,27460],{},[75,27997,27998,28000,28002],{},[81,27999,83],{},[461,28001,18365],{},[91,28003,27460],{},[461,28005,5777],{},[461,28007,23244],{},[3277,28009,27595],{},[461,28011,812],{"fence":106},[14368,28013,28014,28020],{},[14371,28015,28016],{},[14374,28017,28018],{"scriptlevel":1754,"displaystyle":106},[75,28019],{},[14371,28021,28022],{},[14374,28023,28024],{"scriptlevel":1754,"displaystyle":106},[75,28025,28026,28028,28030,28045,28047,28049,28051,28057],{},[75,28027],{},[461,28029,815],{},[14683,28031,28032,28034,28042],{},[461,28033,1826],{},[75,28035,28036,28038,28040],{},[91,28037,7357],{},[461,28039,815],{},[81,28041,1754],{},[91,28043,28044],{"mathvariant":818},"∞",[461,28046,798],{"stretchy":797},[91,28048,18995],{},[91,28050,27460],{},[465,28052,28053,28055],{},[461,28054,812],{"stretchy":797},[91,28056,7357],{},[1822,28058,28059,28061,28069],{},[91,28060,25477],{},[75,28062,28063,28065,28067],{},[91,28064,777],{},[461,28066,5777],{},[91,28068,7357],{},[91,28070,672],{},[98,28072,28073],{"encoding":100},"\\begin{aligned}\nA_\\theta^{GAE}(s_t, a) &= (1 - \\lambda)(A_\\theta^1 + \\lambda * A_\\theta^2 + \\lambda^2 A_\\theta^3 + \\cdots) \\quad\\quad \\lambda = 0.9: \\quad A_\\theta^{GAE} = 0.1A_\\theta^1 + 0.09A_\\theta^2 + 0.081A_\\theta^3 + \\cdots \\\\\n&= (1 - \\lambda)(\\delta_t^V + \\lambda * (\\delta_t^V + \\gamma \\delta_{t+1}^V) + \\lambda^2 (\\delta_t^V + \\gamma \\delta_{t+1}^V + \\gamma^2 \\delta_{t+2}^V) + \\cdots) \\\\\n&= (1 - \\lambda)(\\delta_t^V (1 + \\lambda + \\lambda^2 + \\cdots) + \\gamma \\delta_{t+1}^V * (\\lambda + \\lambda^2 + \\cdots) + \\cdots) \\\\\n&= (1 - \\lambda)\\left(\\delta_t^V \\frac{1}{1 - \\lambda} + \\gamma \\delta_{t+1}^V \\frac{\\lambda}{1 - \\lambda} + \\cdots\\right) \\\\\n&= \\sum\\limits_{b=0}^\\infty (\\gamma\\lambda)^b \\delta_{t+b}^V\n\\end{aligned}",[59,28075,28077],{"className":28076,"ariaHidden":106},[105],[59,28078,28080,28084],{"className":28079},[110],[59,28081],{"className":28082,"style":28083},[114],"height:10.6075em;vertical-align:-5.0538em;",[59,28085,28087],{"className":28086},[119],[59,28088,28090,28276],{"className":28089},[14362],[59,28091,28093],{"className":28092},[15153],[59,28094,28096,28267],{"className":28095},[131,132],[59,28097,28099,28264],{"className":28098},[136],[59,28100,28103,28228,28237,28246,28255],{"className":28101,"style":28102},[140],"height:5.5538em;",[59,28104,28106,28110],{"style":28105},"top:-8.3138em;",[59,28107],{"className":28108,"style":28109},[148],"height:3.6514em;",[59,28111,28113,28173,28176,28216,28219,28222,28225],{"className":28112},[119],[59,28114,28116,28119],{"className":28115},[119],[59,28117,774],{"className":28118},[119,192],[59,28120,28122],{"className":28121},[196],[59,28123,28125,28165],{"className":28124},[131,132],[59,28126,28128,28162],{"className":28127},[136],[59,28129,28131,28142],{"className":28130,"style":24796},[140],[59,28132,28133,28136],{"style":17311},[59,28134],{"className":28135,"style":540},[148],[59,28137,28139],{"className":28138},[153,154,155,156],[59,28140,14051],{"className":28141,"style":3210},[119,192,156],[59,28143,28144,28147],{"style":16426},[59,28145],{"className":28146,"style":540},[148],[59,28148,28150],{"className":28149},[153,154,155,156],[59,28151,28153,28156,28159],{"className":28152},[119,156],[59,28154,27510],{"className":28155},[119,192,156],[59,28157,774],{"className":28158},[119,192,156],[59,28160,5607],{"className":28161,"style":6050},[119,192,156],[59,28163,227],{"className":28164},[226],[59,28166,28168],{"className":28167},[136],[59,28169,28171],{"className":28170,"style":17341},[140],[59,28172],{},[59,28174,798],{"className":28175},[123],[59,28177,28179,28182],{"className":28178},[119],[59,28180,12802],{"className":28181},[119,192],[59,28183,28185],{"className":28184},[196],[59,28186,28188,28208],{"className":28187},[131,132],[59,28189,28191,28205],{"className":28190},[136],[59,28192,28194],{"className":28193,"style":12732},[140],[59,28195,28196,28199],{"style":1251},[59,28197],{"className":28198,"style":540},[148],[59,28200,28202],{"className":28201},[153,154,155,156],[59,28203,777],{"className":28204},[119,192,156],[59,28206,227],{"className":28207},[226],[59,28209,28211],{"className":28210},[136],[59,28212,28214],{"className":28213,"style":1270},[140],[59,28215],{},[59,28217,803],{"className":28218},[912],[59,28220],{"className":28221,"style":916},[499],[59,28223,22],{"className":28224},[119,192],[59,28226,812],{"className":28227},[313],[59,28229,28231,28234],{"style":28230},"top:-6.7625em;",[59,28232],{"className":28233,"style":28109},[148],[59,28235],{"className":28236},[119],[59,28238,28240,28243],{"style":28239},"top:-5.2112em;",[59,28241],{"className":28242,"style":28109},[148],[59,28244],{"className":28245},[119],[59,28247,28249,28252],{"style":28248},"top:-3.1012em;",[59,28250],{"className":28251,"style":28109},[148],[59,28253],{"className":28254},[119],[59,28256,28258,28261],{"style":28257},"top:-0.1997em;",[59,28259],{"className":28260,"style":28109},[148],[59,28262],{"className":28263},[119],[59,28265,227],{"className":28266},[226],[59,28268,28270],{"className":28269},[136],[59,28271,28274],{"className":28272,"style":28273},[140],"height:5.0538em;",[59,28275],{},[59,28277,28279],{"className":28278},[15377],[59,28280,28282,30251],{"className":28281},[131,132],[59,28283,28285,30248],{"className":28284},[136],[59,28286,28288,28857,29385,29715,30062],{"className":28287,"style":28102},[140],[59,28289,28290,28293],{"style":28105},[59,28291],{"className":28292,"style":28109},[148],[59,28294,28296,28299,28302,28305,28308,28311,28314,28317,28320,28323,28326,28329,28332,28383,28386,28389,28392,28395,28398,28401,28404,28455,28458,28461,28464,28493,28544,28547,28550,28553,28556,28559,28562,28566,28569,28572,28575,28578,28581,28584,28587,28590,28593,28596,28656,28659,28662,28665,28668,28719,28722,28725,28728,28731,28782,28785,28788,28791,28794,28845,28848,28851,28854],{"className":28295},[119],[59,28297],{"className":28298},[119],[59,28300],{"className":28301,"style":500},[499],[59,28303,815],{"className":28304},[504],[59,28306],{"className":28307,"style":500},[499],[59,28309,798],{"className":28310},[123],[59,28312,83],{"className":28313},[119],[59,28315],{"className":28316,"style":1872},[499],[59,28318,18365],{"className":28319},[553],[59,28321],{"className":28322,"style":1872},[499],[59,28324,27460],{"className":28325},[119,192],[59,28327,812],{"className":28328},[313],[59,28330,798],{"className":28331},[123],[59,28333,28335,28338],{"className":28334},[119],[59,28336,774],{"className":28337},[119,192],[59,28339,28341],{"className":28340},[196],[59,28342,28344,28375],{"className":28343},[131,132],[59,28345,28347,28372],{"className":28346},[136],[59,28348,28350,28361],{"className":28349,"style":23310},[140],[59,28351,28352,28355],{"style":17311},[59,28353],{"className":28354,"style":540},[148],[59,28356,28358],{"className":28357},[153,154,155,156],[59,28359,14051],{"className":28360,"style":3210},[119,192,156],[59,28362,28363,28366],{"style":16426},[59,28364],{"className":28365,"style":540},[148],[59,28367,28369],{"className":28368},[153,154,155,156],[59,28370,83],{"className":28371},[119,156],[59,28373,227],{"className":28374},[226],[59,28376,28378],{"className":28377},[136],[59,28379,28381],{"className":28380,"style":17341},[140],[59,28382],{},[59,28384],{"className":28385,"style":1872},[499],[59,28387,5777],{"className":28388},[553],[59,28390],{"className":28391,"style":1872},[499],[59,28393,27460],{"className":28394},[119,192],[59,28396],{"className":28397,"style":1872},[499],[59,28399,23049],{"className":28400},[553],[59,28402],{"className":28403,"style":1872},[499],[59,28405,28407,28410],{"className":28406},[119],[59,28408,774],{"className":28409},[119,192],[59,28411,28413],{"className":28412},[196],[59,28414,28416,28447],{"className":28415},[131,132],[59,28417,28419,28444],{"className":28418},[136],[59,28420,28422,28433],{"className":28421,"style":23310},[140],[59,28423,28424,28427],{"style":17311},[59,28425],{"className":28426,"style":540},[148],[59,28428,28430],{"className":28429},[153,154,155,156],[59,28431,14051],{"className":28432,"style":3210},[119,192,156],[59,28434,28435,28438],{"style":16426},[59,28436],{"className":28437,"style":540},[148],[59,28439,28441],{"className":28440},[153,154,155,156],[59,28442,2821],{"className":28443},[119,156],[59,28445,227],{"className":28446},[226],[59,28448,28450],{"className":28449},[136],[59,28451,28453],{"className":28452,"style":17341},[140],[59,28454],{},[59,28456],{"className":28457,"style":1872},[499],[59,28459,5777],{"className":28460},[553],[59,28462],{"className":28463,"style":1872},[499],[59,28465,28467,28470],{"className":28466},[119],[59,28468,27460],{"className":28469},[119,192],[59,28471,28473],{"className":28472},[196],[59,28474,28476],{"className":28475},[131],[59,28477,28479],{"className":28478},[136],[59,28480,28482],{"className":28481,"style":23310},[140],[59,28483,28484,28487],{"style":16426},[59,28485],{"className":28486,"style":540},[148],[59,28488,28490],{"className":28489},[153,154,155,156],[59,28491,2821],{"className":28492},[119,156],[59,28494,28496,28499],{"className":28495},[119],[59,28497,774],{"className":28498},[119,192],[59,28500,28502],{"className":28501},[196],[59,28503,28505,28536],{"className":28504},[131,132],[59,28506,28508,28533],{"className":28507},[136],[59,28509,28511,28522],{"className":28510,"style":23310},[140],[59,28512,28513,28516],{"style":17311},[59,28514],{"className":28515,"style":540},[148],[59,28517,28519],{"className":28518},[153,154,155,156],[59,28520,14051],{"className":28521,"style":3210},[119,192,156],[59,28523,28524,28527],{"style":16426},[59,28525],{"className":28526,"style":540},[148],[59,28528,28530],{"className":28529},[153,154,155,156],[59,28531,4671],{"className":28532},[119,156],[59,28534,227],{"className":28535},[226],[59,28537,28539],{"className":28538},[136],[59,28540,28542],{"className":28541,"style":17341},[140],[59,28543],{},[59,28545],{"className":28546,"style":1872},[499],[59,28548,5777],{"className":28549},[553],[59,28551],{"className":28552,"style":1872},[499],[59,28554,23244],{"className":28555},[962],[59,28557],{"className":28558,"style":916},[499],[59,28560,812],{"className":28561},[313],[59,28563],{"className":28564,"style":28565},[499],"margin-right:1em;",[59,28567],{"className":28568,"style":28565},[499],[59,28570,27460],{"className":28571},[119,192],[59,28573],{"className":28574,"style":500},[499],[59,28576,815],{"className":28577},[504],[59,28579],{"className":28580,"style":500},[499],[59,28582,27609],{"className":28583},[119],[59,28585],{"className":28586,"style":500},[499],[59,28588,7184],{"className":28589},[504],[59,28591],{"className":28592,"style":28565},[499],[59,28594],{"className":28595,"style":500},[499],[59,28597,28599,28602],{"className":28598},[119],[59,28600,774],{"className":28601},[119,192],[59,28603,28605],{"className":28604},[196],[59,28606,28608,28648],{"className":28607},[131,132],[59,28609,28611,28645],{"className":28610},[136],[59,28612,28614,28625],{"className":28613,"style":24796},[140],[59,28615,28616,28619],{"style":17311},[59,28617],{"className":28618,"style":540},[148],[59,28620,28622],{"className":28621},[153,154,155,156],[59,28623,14051],{"className":28624,"style":3210},[119,192,156],[59,28626,28627,28630],{"style":16426},[59,28628],{"className":28629,"style":540},[148],[59,28631,28633],{"className":28632},[153,154,155,156],[59,28634,28636,28639,28642],{"className":28635},[119,156],[59,28637,27510],{"className":28638},[119,192,156],[59,28640,774],{"className":28641},[119,192,156],[59,28643,5607],{"className":28644,"style":6050},[119,192,156],[59,28646,227],{"className":28647},[226],[59,28649,28651],{"className":28650},[136],[59,28652,28654],{"className":28653,"style":17341},[140],[59,28655],{},[59,28657],{"className":28658,"style":500},[499],[59,28660,815],{"className":28661},[504],[59,28663],{"className":28664,"style":500},[499],[59,28666,12466],{"className":28667},[119],[59,28669,28671,28674],{"className":28670},[119],[59,28672,774],{"className":28673},[119,192],[59,28675,28677],{"className":28676},[196],[59,28678,28680,28711],{"className":28679},[131,132],[59,28681,28683,28708],{"className":28682},[136],[59,28684,28686,28697],{"className":28685,"style":23310},[140],[59,28687,28688,28691],{"style":17311},[59,28689],{"className":28690,"style":540},[148],[59,28692,28694],{"className":28693},[153,154,155,156],[59,28695,14051],{"className":28696,"style":3210},[119,192,156],[59,28698,28699,28702],{"style":16426},[59,28700],{"className":28701,"style":540},[148],[59,28703,28705],{"className":28704},[153,154,155,156],[59,28706,83],{"className":28707},[119,156],[59,28709,227],{"className":28710},[226],[59,28712,28714],{"className":28713},[136],[59,28715,28717],{"className":28716,"style":17341},[140],[59,28718],{},[59,28720],{"className":28721,"style":1872},[499],[59,28723,5777],{"className":28724},[553],[59,28726],{"className":28727,"style":1872},[499],[59,28729,27644],{"className":28730},[119],[59,28732,28734,28737],{"className":28733},[119],[59,28735,774],{"className":28736},[119,192],[59,28738,28740],{"className":28739},[196],[59,28741,28743,28774],{"className":28742},[131,132],[59,28744,28746,28771],{"className":28745},[136],[59,28747,28749,28760],{"className":28748,"style":23310},[140],[59,28750,28751,28754],{"style":17311},[59,28752],{"className":28753,"style":540},[148],[59,28755,28757],{"className":28756},[153,154,155,156],[59,28758,14051],{"className":28759,"style":3210},[119,192,156],[59,28761,28762,28765],{"style":16426},[59,28763],{"className":28764,"style":540},[148],[59,28766,28768],{"className":28767},[153,154,155,156],[59,28769,2821],{"className":28770},[119,156],[59,28772,227],{"className":28773},[226],[59,28775,28777],{"className":28776},[136],[59,28778,28780],{"className":28779,"style":17341},[140],[59,28781],{},[59,28783],{"className":28784,"style":1872},[499],[59,28786,5777],{"className":28787},[553],[59,28789],{"className":28790,"style":1872},[499],[59,28792,27657],{"className":28793},[119],[59,28795,28797,28800],{"className":28796},[119],[59,28798,774],{"className":28799},[119,192],[59,28801,28803],{"className":28802},[196],[59,28804,28806,28837],{"className":28805},[131,132],[59,28807,28809,28834],{"className":28808},[136],[59,28810,28812,28823],{"className":28811,"style":23310},[140],[59,28813,28814,28817],{"style":17311},[59,28815],{"className":28816,"style":540},[148],[59,28818,28820],{"className":28819},[153,154,155,156],[59,28821,14051],{"className":28822,"style":3210},[119,192,156],[59,28824,28825,28828],{"style":16426},[59,28826],{"className":28827,"style":540},[148],[59,28829,28831],{"className":28830},[153,154,155,156],[59,28832,4671],{"className":28833},[119,156],[59,28835,227],{"className":28836},[226],[59,28838,28840],{"className":28839},[136],[59,28841,28843],{"className":28842,"style":17341},[140],[59,28844],{},[59,28846],{"className":28847,"style":1872},[499],[59,28849,5777],{"className":28850},[553],[59,28852],{"className":28853,"style":1872},[499],[59,28855,23244],{"className":28856},[962],[59,28858,28859,28862],{"style":28230},[59,28860],{"className":28861,"style":28109},[148],[59,28863,28865,28868,28871,28874,28877,28880,28883,28886,28889,28892,28895,28898,28901,28952,28955,28958,28961,28964,28967,28970,28973,28976,29027,29030,29033,29036,29039,29099,29102,29105,29108,29111,29140,29143,29194,29197,29200,29203,29206,29266,29269,29272,29275,29304,29364,29367,29370,29373,29376,29379,29382],{"className":28864},[119],[59,28866],{"className":28867},[119],[59,28869],{"className":28870,"style":500},[499],[59,28872,815],{"className":28873},[504],[59,28875],{"className":28876,"style":500},[499],[59,28878,798],{"className":28879},[123],[59,28881,83],{"className":28882},[119],[59,28884],{"className":28885,"style":1872},[499],[59,28887,18365],{"className":28888},[553],[59,28890],{"className":28891,"style":1872},[499],[59,28893,27460],{"className":28894},[119,192],[59,28896,812],{"className":28897},[313],[59,28899,798],{"className":28900},[123],[59,28902,28904,28907],{"className":28903},[119],[59,28905,25477],{"className":28906,"style":25498},[119,192],[59,28908,28910],{"className":28909},[196],[59,28911,28913,28944],{"className":28912},[131,132],[59,28914,28916,28941],{"className":28915},[136],[59,28917,28919,28930],{"className":28918,"style":24796},[140],[59,28920,28921,28924],{"style":25926},[59,28922],{"className":28923,"style":540},[148],[59,28925,28927],{"className":28926},[153,154,155,156],[59,28928,777],{"className":28929},[119,192,156],[59,28931,28932,28935],{"style":16426},[59,28933],{"className":28934,"style":540},[148],[59,28936,28938],{"className":28937},[153,154,155,156],[59,28939,672],{"className":28940,"style":702},[119,192,156],[59,28942,227],{"className":28943},[226],[59,28945,28947],{"className":28946},[136],[59,28948,28950],{"className":28949,"style":17341},[140],[59,28951],{},[59,28953],{"className":28954,"style":1872},[499],[59,28956,5777],{"className":28957},[553],[59,28959],{"className":28960,"style":1872},[499],[59,28962,27460],{"className":28963},[119,192],[59,28965],{"className":28966,"style":1872},[499],[59,28968,23049],{"className":28969},[553],[59,28971],{"className":28972,"style":1872},[499],[59,28974,798],{"className":28975},[123],[59,28977,28979,28982],{"className":28978},[119],[59,28980,25477],{"className":28981,"style":25498},[119,192],[59,28983,28985],{"className":28984},[196],[59,28986,28988,29019],{"className":28987},[131,132],[59,28989,28991,29016],{"className":28990},[136],[59,28992,28994,29005],{"className":28993,"style":24796},[140],[59,28995,28996,28999],{"style":25926},[59,28997],{"className":28998,"style":540},[148],[59,29000,29002],{"className":29001},[153,154,155,156],[59,29003,777],{"className":29004},[119,192,156],[59,29006,29007,29010],{"style":16426},[59,29008],{"className":29009,"style":540},[148],[59,29011,29013],{"className":29012},[153,154,155,156],[59,29014,672],{"className":29015,"style":702},[119,192,156],[59,29017,227],{"className":29018},[226],[59,29020,29022],{"className":29021},[136],[59,29023,29025],{"className":29024,"style":17341},[140],[59,29026],{},[59,29028],{"className":29029,"style":1872},[499],[59,29031,5777],{"className":29032},[553],[59,29034],{"className":29035,"style":1872},[499],[59,29037,18995],{"className":29038,"style":19242},[119,192],[59,29040,29042,29045],{"className":29041},[119],[59,29043,25477],{"className":29044,"style":25498},[119,192],[59,29046,29048],{"className":29047},[196],[59,29049,29051,29091],{"className":29050},[131,132],[59,29052,29054,29088],{"className":29053},[136],[59,29055,29057,29077],{"className":29056,"style":24796},[140],[59,29058,29059,29062],{"style":25926},[59,29060],{"className":29061,"style":540},[148],[59,29063,29065],{"className":29064},[153,154,155,156],[59,29066,29068,29071,29074],{"className":29067},[119,156],[59,29069,777],{"className":29070},[119,192,156],[59,29072,5777],{"className":29073},[553,156],[59,29075,83],{"className":29076},[119,156],[59,29078,29079,29082],{"style":16426},[59,29080],{"className":29081,"style":540},[148],[59,29083,29085],{"className":29084},[153,154,155,156],[59,29086,672],{"className":29087,"style":702},[119,192,156],[59,29089,227],{"className":29090},[226],[59,29092,29094],{"className":29093},[136],[59,29095,29097],{"className":29096,"style":26310},[140],[59,29098],{},[59,29100,812],{"className":29101},[313],[59,29103],{"className":29104,"style":1872},[499],[59,29106,5777],{"className":29107},[553],[59,29109],{"className":29110,"style":1872},[499],[59,29112,29114,29117],{"className":29113},[119],[59,29115,27460],{"className":29116},[119,192],[59,29118,29120],{"className":29119},[196],[59,29121,29123],{"className":29122},[131],[59,29124,29126],{"className":29125},[136],[59,29127,29129],{"className":29128,"style":23310},[140],[59,29130,29131,29134],{"style":16426},[59,29132],{"className":29133,"style":540},[148],[59,29135,29137],{"className":29136},[153,154,155,156],[59,29138,2821],{"className":29139},[119,156],[59,29141,798],{"className":29142},[123],[59,29144,29146,29149],{"className":29145},[119],[59,29147,25477],{"className":29148,"style":25498},[119,192],[59,29150,29152],{"className":29151},[196],[59,29153,29155,29186],{"className":29154},[131,132],[59,29156,29158,29183],{"className":29157},[136],[59,29159,29161,29172],{"className":29160,"style":24796},[140],[59,29162,29163,29166],{"style":25926},[59,29164],{"className":29165,"style":540},[148],[59,29167,29169],{"className":29168},[153,154,155,156],[59,29170,777],{"className":29171},[119,192,156],[59,29173,29174,29177],{"style":16426},[59,29175],{"className":29176,"style":540},[148],[59,29178,29180],{"className":29179},[153,154,155,156],[59,29181,672],{"className":29182,"style":702},[119,192,156],[59,29184,227],{"className":29185},[226],[59,29187,29189],{"className":29188},[136],[59,29190,29192],{"className":29191,"style":17341},[140],[59,29193],{},[59,29195],{"className":29196,"style":1872},[499],[59,29198,5777],{"className":29199},[553],[59,29201],{"className":29202,"style":1872},[499],[59,29204,18995],{"className":29205,"style":19242},[119,192],[59,29207,29209,29212],{"className":29208},[119],[59,29210,25477],{"className":29211,"style":25498},[119,192],[59,29213,29215],{"className":29214},[196],[59,29216,29218,29258],{"className":29217},[131,132],[59,29219,29221,29255],{"className":29220},[136],[59,29222,29224,29244],{"className":29223,"style":24796},[140],[59,29225,29226,29229],{"style":25926},[59,29227],{"className":29228,"style":540},[148],[59,29230,29232],{"className":29231},[153,154,155,156],[59,29233,29235,29238,29241],{"className":29234},[119,156],[59,29236,777],{"className":29237},[119,192,156],[59,29239,5777],{"className":29240},[553,156],[59,29242,83],{"className":29243},[119,156],[59,29245,29246,29249],{"style":16426},[59,29247],{"className":29248,"style":540},[148],[59,29250,29252],{"className":29251},[153,154,155,156],[59,29253,672],{"className":29254,"style":702},[119,192,156],[59,29256,227],{"className":29257},[226],[59,29259,29261],{"className":29260},[136],[59,29262,29264],{"className":29263,"style":26310},[140],[59,29265],{},[59,29267],{"className":29268,"style":1872},[499],[59,29270,5777],{"className":29271},[553],[59,29273],{"className":29274,"style":1872},[499],[59,29276,29278,29281],{"className":29277},[119],[59,29279,18995],{"className":29280,"style":19242},[119,192],[59,29282,29284],{"className":29283},[196],[59,29285,29287],{"className":29286},[131],[59,29288,29290],{"className":29289},[136],[59,29291,29293],{"className":29292,"style":23310},[140],[59,29294,29295,29298],{"style":16426},[59,29296],{"className":29297,"style":540},[148],[59,29299,29301],{"className":29300},[153,154,155,156],[59,29302,2821],{"className":29303},[119,156],[59,29305,29307,29310],{"className":29306},[119],[59,29308,25477],{"className":29309,"style":25498},[119,192],[59,29311,29313],{"className":29312},[196],[59,29314,29316,29356],{"className":29315},[131,132],[59,29317,29319,29353],{"className":29318},[136],[59,29320,29322,29342],{"className":29321,"style":24796},[140],[59,29323,29324,29327],{"style":25926},[59,29325],{"className":29326,"style":540},[148],[59,29328,29330],{"className":29329},[153,154,155,156],[59,29331,29333,29336,29339],{"className":29332},[119,156],[59,29334,777],{"className":29335},[119,192,156],[59,29337,5777],{"className":29338},[553,156],[59,29340,2821],{"className":29341},[119,156],[59,29343,29344,29347],{"style":16426},[59,29345],{"className":29346,"style":540},[148],[59,29348,29350],{"className":29349},[153,154,155,156],[59,29351,672],{"className":29352,"style":702},[119,192,156],[59,29354,227],{"className":29355},[226],[59,29357,29359],{"className":29358},[136],[59,29360,29362],{"className":29361,"style":26310},[140],[59,29363],{},[59,29365,812],{"className":29366},[313],[59,29368],{"className":29369,"style":1872},[499],[59,29371,5777],{"className":29372},[553],[59,29374],{"className":29375,"style":1872},[499],[59,29377,23244],{"className":29378},[962],[59,29380],{"className":29381,"style":916},[499],[59,29383,812],{"className":29384},[313],[59,29386,29387,29390],{"style":28239},[59,29388],{"className":29389,"style":28109},[148],[59,29391,29393,29396,29399,29402,29405,29408,29411,29414,29417,29420,29423,29426,29429,29480,29483,29486,29489,29492,29495,29498,29501,29504,29507,29536,29539,29542,29545,29548,29551,29554,29557,29560,29563,29566,29626,29629,29632,29635,29638,29641,29644,29647,29650,29679,29682,29685,29688,29691,29694,29697,29700,29703,29706,29709,29712],{"className":29392},[119],[59,29394],{"className":29395},[119],[59,29397],{"className":29398,"style":500},[499],[59,29400,815],{"className":29401},[504],[59,29403],{"className":29404,"style":500},[499],[59,29406,798],{"className":29407},[123],[59,29409,83],{"className":29410},[119],[59,29412],{"className":29413,"style":1872},[499],[59,29415,18365],{"className":29416},[553],[59,29418],{"className":29419,"style":1872},[499],[59,29421,27460],{"className":29422},[119,192],[59,29424,812],{"className":29425},[313],[59,29427,798],{"className":29428},[123],[59,29430,29432,29435],{"className":29431},[119],[59,29433,25477],{"className":29434,"style":25498},[119,192],[59,29436,29438],{"className":29437},[196],[59,29439,29441,29472],{"className":29440},[131,132],[59,29442,29444,29469],{"className":29443},[136],[59,29445,29447,29458],{"className":29446,"style":24796},[140],[59,29448,29449,29452],{"style":25926},[59,29450],{"className":29451,"style":540},[148],[59,29453,29455],{"className":29454},[153,154,155,156],[59,29456,777],{"className":29457},[119,192,156],[59,29459,29460,29463],{"style":16426},[59,29461],{"className":29462,"style":540},[148],[59,29464,29466],{"className":29465},[153,154,155,156],[59,29467,672],{"className":29468,"style":702},[119,192,156],[59,29470,227],{"className":29471},[226],[59,29473,29475],{"className":29474},[136],[59,29476,29478],{"className":29477,"style":17341},[140],[59,29479],{},[59,29481,798],{"className":29482},[123],[59,29484,83],{"className":29485},[119],[59,29487],{"className":29488,"style":1872},[499],[59,29490,5777],{"className":29491},[553],[59,29493],{"className":29494,"style":1872},[499],[59,29496,27460],{"className":29497},[119,192],[59,29499],{"className":29500,"style":1872},[499],[59,29502,5777],{"className":29503},[553],[59,29505],{"className":29506,"style":1872},[499],[59,29508,29510,29513],{"className":29509},[119],[59,29511,27460],{"className":29512},[119,192],[59,29514,29516],{"className":29515},[196],[59,29517,29519],{"className":29518},[131],[59,29520,29522],{"className":29521},[136],[59,29523,29525],{"className":29524,"style":23310},[140],[59,29526,29527,29530],{"style":16426},[59,29528],{"className":29529,"style":540},[148],[59,29531,29533],{"className":29532},[153,154,155,156],[59,29534,2821],{"className":29535},[119,156],[59,29537],{"className":29538,"style":1872},[499],[59,29540,5777],{"className":29541},[553],[59,29543],{"className":29544,"style":1872},[499],[59,29546,23244],{"className":29547},[962],[59,29549],{"className":29550,"style":916},[499],[59,29552,812],{"className":29553},[313],[59,29555],{"className":29556,"style":1872},[499],[59,29558,5777],{"className":29559},[553],[59,29561],{"className":29562,"style":1872},[499],[59,29564,18995],{"className":29565,"style":19242},[119,192],[59,29567,29569,29572],{"className":29568},[119],[59,29570,25477],{"className":29571,"style":25498},[119,192],[59,29573,29575],{"className":29574},[196],[59,29576,29578,29618],{"className":29577},[131,132],[59,29579,29581,29615],{"className":29580},[136],[59,29582,29584,29604],{"className":29583,"style":24796},[140],[59,29585,29586,29589],{"style":25926},[59,29587],{"className":29588,"style":540},[148],[59,29590,29592],{"className":29591},[153,154,155,156],[59,29593,29595,29598,29601],{"className":29594},[119,156],[59,29596,777],{"className":29597},[119,192,156],[59,29599,5777],{"className":29600},[553,156],[59,29602,83],{"className":29603},[119,156],[59,29605,29606,29609],{"style":16426},[59,29607],{"className":29608,"style":540},[148],[59,29610,29612],{"className":29611},[153,154,155,156],[59,29613,672],{"className":29614,"style":702},[119,192,156],[59,29616,227],{"className":29617},[226],[59,29619,29621],{"className":29620},[136],[59,29622,29624],{"className":29623,"style":26310},[140],[59,29625],{},[59,29627],{"className":29628,"style":1872},[499],[59,29630,23049],{"className":29631},[553],[59,29633],{"className":29634,"style":1872},[499],[59,29636,798],{"className":29637},[123],[59,29639,27460],{"className":29640},[119,192],[59,29642],{"className":29643,"style":1872},[499],[59,29645,5777],{"className":29646},[553],[59,29648],{"className":29649,"style":1872},[499],[59,29651,29653,29656],{"className":29652},[119],[59,29654,27460],{"className":29655},[119,192],[59,29657,29659],{"className":29658},[196],[59,29660,29662],{"className":29661},[131],[59,29663,29665],{"className":29664},[136],[59,29666,29668],{"className":29667,"style":23310},[140],[59,29669,29670,29673],{"style":16426},[59,29671],{"className":29672,"style":540},[148],[59,29674,29676],{"className":29675},[153,154,155,156],[59,29677,2821],{"className":29678},[119,156],[59,29680],{"className":29681,"style":1872},[499],[59,29683,5777],{"className":29684},[553],[59,29686],{"className":29687,"style":1872},[499],[59,29689,23244],{"className":29690},[962],[59,29692],{"className":29693,"style":916},[499],[59,29695,812],{"className":29696},[313],[59,29698],{"className":29699,"style":1872},[499],[59,29701,5777],{"className":29702},[553],[59,29704],{"className":29705,"style":1872},[499],[59,29707,23244],{"className":29708},[962],[59,29710],{"className":29711,"style":916},[499],[59,29713,812],{"className":29714},[313],[59,29716,29717,29720],{"style":28248},[59,29718],{"className":29719,"style":28109},[148],[59,29721,29723,29726,29729,29732,29735,29738,29741,29744,29747,29750,29753,29756,29759],{"className":29722},[119],[59,29724],{"className":29725},[119],[59,29727],{"className":29728,"style":500},[499],[59,29730,815],{"className":29731},[504],[59,29733],{"className":29734,"style":500},[499],[59,29736,798],{"className":29737},[123],[59,29739,83],{"className":29740},[119],[59,29742],{"className":29743,"style":1872},[499],[59,29745,18365],{"className":29746},[553],[59,29748],{"className":29749,"style":1872},[499],[59,29751,27460],{"className":29752},[119,192],[59,29754,812],{"className":29755},[313],[59,29757],{"className":29758,"style":916},[499],[59,29760,29762,29768,29819,29894,29897,29900,29903,29906,29966,30041,30044,30047,30050,30053,30056],{"className":29761},[962],[59,29763,29765],{"className":29764,"style":967},[123,966],[59,29766,798],{"className":29767},[971,155],[59,29769,29771,29774],{"className":29770},[119],[59,29772,25477],{"className":29773,"style":25498},[119,192],[59,29775,29777],{"className":29776},[196],[59,29778,29780,29811],{"className":29779},[131,132],[59,29781,29783,29808],{"className":29782},[136],[59,29784,29786,29797],{"className":29785,"style":24796},[140],[59,29787,29788,29791],{"style":25926},[59,29789],{"className":29790,"style":540},[148],[59,29792,29794],{"className":29793},[153,154,155,156],[59,29795,777],{"className":29796},[119,192,156],[59,29798,29799,29802],{"style":16426},[59,29800],{"className":29801,"style":540},[148],[59,29803,29805],{"className":29804},[153,154,155,156],[59,29806,672],{"className":29807,"style":702},[119,192,156],[59,29809,227],{"className":29810},[226],[59,29812,29814],{"className":29813},[136],[59,29815,29817],{"className":29816,"style":17341},[140],[59,29818],{},[59,29820,29822,29825,29891],{"className":29821},[119],[59,29823],{"className":29824},[123,124],[59,29826,29828],{"className":29827},[78],[59,29829,29831,29882],{"className":29830},[131,132],[59,29832,29834,29879],{"className":29833},[136],[59,29835,29837,29860,29868],{"className":29836,"style":16279},[140],[59,29838,29839,29842],{"style":15819},[59,29840],{"className":29841,"style":149},[148],[59,29843,29845,29848,29851,29854,29857],{"className":29844},[119],[59,29846,83],{"className":29847},[119],[59,29849],{"className":29850,"style":1872},[499],[59,29852,18365],{"className":29853},[553],[59,29855],{"className":29856,"style":1872},[499],[59,29858,27460],{"className":29859},[119,192],[59,29861,29862,29865],{"style":274},[59,29863],{"className":29864,"style":149},[148],[59,29866],{"className":29867,"style":282},[281],[59,29869,29870,29873],{"style":15885},[59,29871],{"className":29872,"style":149},[148],[59,29874,29876],{"className":29875},[119],[59,29877,83],{"className":29878},[119],[59,29880,227],{"className":29881},[226],[59,29883,29885],{"className":29884},[136],[59,29886,29889],{"className":29887,"style":29888},[140],"height:0.7693em;",[59,29890],{},[59,29892],{"className":29893},[313,124],[59,29895],{"className":29896,"style":1872},[499],[59,29898,5777],{"className":29899},[553],[59,29901],{"className":29902,"style":1872},[499],[59,29904,18995],{"className":29905,"style":19242},[119,192],[59,29907,29909,29912],{"className":29908},[119],[59,29910,25477],{"className":29911,"style":25498},[119,192],[59,29913,29915],{"className":29914},[196],[59,29916,29918,29958],{"className":29917},[131,132],[59,29919,29921,29955],{"className":29920},[136],[59,29922,29924,29944],{"className":29923,"style":24796},[140],[59,29925,29926,29929],{"style":25926},[59,29927],{"className":29928,"style":540},[148],[59,29930,29932],{"className":29931},[153,154,155,156],[59,29933,29935,29938,29941],{"className":29934},[119,156],[59,29936,777],{"className":29937},[119,192,156],[59,29939,5777],{"className":29940},[553,156],[59,29942,83],{"className":29943},[119,156],[59,29945,29946,29949],{"style":16426},[59,29947],{"className":29948,"style":540},[148],[59,29950,29952],{"className":29951},[153,154,155,156],[59,29953,672],{"className":29954,"style":702},[119,192,156],[59,29956,227],{"className":29957},[226],[59,29959,29961],{"className":29960},[136],[59,29962,29964],{"className":29963,"style":26310},[140],[59,29965],{},[59,29967,29969,29972,30038],{"className":29968},[119],[59,29970],{"className":29971},[123,124],[59,29973,29975],{"className":29974},[78],[59,29976,29978,30030],{"className":29977},[131,132],[59,29979,29981,30027],{"className":29980},[136],[59,29982,29985,30008,30016],{"className":29983,"style":29984},[140],"height:1.3714em;",[59,29986,29987,29990],{"style":15819},[59,29988],{"className":29989,"style":149},[148],[59,29991,29993,29996,29999,30002,30005],{"className":29992},[119],[59,29994,83],{"className":29995},[119],[59,29997],{"className":29998,"style":1872},[499],[59,30000,18365],{"className":30001},[553],[59,30003],{"className":30004,"style":1872},[499],[59,30006,27460],{"className":30007},[119,192],[59,30009,30010,30013],{"style":274},[59,30011],{"className":30012,"style":149},[148],[59,30014],{"className":30015,"style":282},[281],[59,30017,30018,30021],{"style":15885},[59,30019],{"className":30020,"style":149},[148],[59,30022,30024],{"className":30023},[119],[59,30025,27460],{"className":30026},[119,192],[59,30028,227],{"className":30029},[226],[59,30031,30033],{"className":30032},[136],[59,30034,30036],{"className":30035,"style":29888},[140],[59,30037],{},[59,30039],{"className":30040},[313,124],[59,30042],{"className":30043,"style":1872},[499],[59,30045,5777],{"className":30046},[553],[59,30048],{"className":30049,"style":1872},[499],[59,30051,23244],{"className":30052},[962],[59,30054],{"className":30055,"style":916},[499],[59,30057,30059],{"className":30058,"style":967},[313,966],[59,30060,812],{"className":30061},[971,155],[59,30063,30064,30067],{"style":28257},[59,30065],{"className":30066,"style":28109},[148],[59,30068,30070,30073,30076,30079,30082,30149,30152,30155,30158,30188],{"className":30069},[119],[59,30071],{"className":30072},[119],[59,30074],{"className":30075,"style":500},[499],[59,30077,815],{"className":30078},[504],[59,30080],{"className":30081,"style":500},[499],[59,30083,30085],{"className":30084},[951,14238],[59,30086,30088,30140],{"className":30087},[131,132],[59,30089,30091,30137],{"className":30090},[136],[59,30092,30095,30116,30126],{"className":30093,"style":30094},[140],"height:1.6514em;",[59,30096,30098,30101],{"style":30097},"top:-1.8479em;margin-left:0em;",[59,30099],{"className":30100,"style":15432},[148],[59,30102,30104],{"className":30103},[153,154,155,156],[59,30105,30107,30110,30113],{"className":30106},[119,156],[59,30108,7357],{"className":30109},[119,192,156],[59,30111,815],{"className":30112},[504,156],[59,30114,1754],{"className":30115},[119,156],[59,30117,30118,30121],{"style":15441},[59,30119],{"className":30120,"style":15432},[148],[59,30122,30123],{},[59,30124,1826],{"className":30125},[951,1910,15450],[59,30127,30128,30131],{"style":16374},[59,30129],{"className":30130,"style":15432},[148],[59,30132,30134],{"className":30133},[153,154,155,156],[59,30135,28044],{"className":30136},[119,156],[59,30138,227],{"className":30139},[226],[59,30141,30143],{"className":30142},[136],[59,30144,30147],{"className":30145,"style":30146},[140],"height:1.3021em;",[59,30148],{},[59,30150,798],{"className":30151},[123],[59,30153,18995],{"className":30154,"style":19242},[119,192],[59,30156,27460],{"className":30157},[119,192],[59,30159,30161,30164],{"className":30160},[313],[59,30162,812],{"className":30163},[313],[59,30165,30167],{"className":30166},[196],[59,30168,30170],{"className":30169},[131],[59,30171,30173],{"className":30172},[136],[59,30174,30177],{"className":30175,"style":30176},[140],"height:0.8991em;",[59,30178,30179,30182],{"style":16426},[59,30180],{"className":30181,"style":540},[148],[59,30183,30185],{"className":30184},[153,154,155,156],[59,30186,7357],{"className":30187},[119,192,156],[59,30189,30191,30194],{"className":30190},[119],[59,30192,25477],{"className":30193,"style":25498},[119,192],[59,30195,30197],{"className":30196},[196],[59,30198,30200,30240],{"className":30199},[131,132],[59,30201,30203,30237],{"className":30202},[136],[59,30204,30206,30226],{"className":30205,"style":24796},[140],[59,30207,30208,30211],{"style":25926},[59,30209],{"className":30210,"style":540},[148],[59,30212,30214],{"className":30213},[153,154,155,156],[59,30215,30217,30220,30223],{"className":30216},[119,156],[59,30218,777],{"className":30219},[119,192,156],[59,30221,5777],{"className":30222},[553,156],[59,30224,7357],{"className":30225},[119,192,156],[59,30227,30228,30231],{"style":16426},[59,30229],{"className":30230,"style":540},[148],[59,30232,30234],{"className":30233},[153,154,155,156],[59,30235,672],{"className":30236,"style":702},[119,192,156],[59,30238,227],{"className":30239},[226],[59,30241,30243],{"className":30242},[136],[59,30244,30246],{"className":30245,"style":26310},[140],[59,30247],{},[59,30249,227],{"className":30250},[226],[59,30252,30254],{"className":30253},[136],[59,30255,30257],{"className":30256,"style":28273},[140],[59,30258],{},[15,30260,30261],{},"整理后发现其中有等比数列，利用等比数列求和得到最终的 GAE 优势函数的表达式。",[15,30263,30264],{},"GAE 优势函数平衡了采样不同步带来的方差和偏差的问题。",[15,30266,30267],{},"现在我们来看上面得之不易的三个表达式：",[59,30269,30271],{"className":30270},[14350],[59,30272,30274,30530],{"className":30273},[62],[59,30275,30277],{"className":30276},[66],[68,30278,30279],{"xmlns":70,"display":9481},[72,30280,30281,30527],{},[75,30282,30283,30291,30293,30299,30301,30303,30309,30311,30323,30325,30327,30333,30335,30341,30343,30345,30359,30361,30367,30369,30371,30373,30375,30389,30391,30393,30395,30401,30415,30417,30423,30437,30455,30469,30471,30479,30481,30489,30491,30493,30495,30497,30499,30505,30507,30515,30517,30525],{},[1822,30284,30285,30287,30289],{},[91,30286,25477],{},[91,30288,777],{},[91,30290,672],{},[461,30292,815],{},[88,30294,30295,30297],{},[91,30296,7826],{},[91,30298,777],{},[461,30300,5777],{},[91,30302,18995],{},[88,30304,30305,30307],{},[91,30306,672],{},[91,30308,14051],{},[461,30310,798],{"stretchy":797},[88,30312,30313,30315],{},[91,30314,12802],{},[75,30316,30317,30319,30321],{},[91,30318,777],{},[461,30320,5777],{},[81,30322,83],{},[461,30324,812],{"stretchy":797},[461,30326,18365],{},[88,30328,30329,30331],{},[91,30330,672],{},[91,30332,14051],{},[461,30334,798],{"stretchy":797},[88,30336,30337,30339],{},[91,30338,12802],{},[91,30340,777],{},[461,30342,812],{"stretchy":797},[499,30344],{"linebreak":21653},[1822,30346,30347,30349,30351],{},[91,30348,774],{},[91,30350,14051],{},[75,30352,30353,30355,30357],{},[91,30354,27510],{},[91,30356,774],{},[91,30358,5607],{},[461,30360,798],{"stretchy":797},[88,30362,30363,30365],{},[91,30364,12802],{},[91,30366,777],{},[461,30368,803],{"separator":106},[91,30370,22],{},[461,30372,812],{"stretchy":797},[461,30374,815],{},[14683,30376,30377,30379,30387],{},[461,30378,1826],{},[75,30380,30381,30383,30385],{},[91,30382,7357],{},[461,30384,815],{},[81,30386,1754],{},[91,30388,28044],{"mathvariant":818},[461,30390,798],{"stretchy":797},[91,30392,18995],{},[91,30394,27460],{},[465,30396,30397,30399],{},[461,30398,812],{"stretchy":797},[91,30400,7357],{},[1822,30402,30403,30405,30413],{},[91,30404,25477],{},[75,30406,30407,30409,30411],{},[91,30408,777],{},[461,30410,5777],{},[91,30412,7357],{},[91,30414,672],{},[499,30416],{"linebreak":21653},[78,30418,30419,30421],{},[81,30420,83],{},[91,30422,7322],{},[14683,30424,30425,30427,30435],{},[461,30426,1826],{},[75,30428,30429,30431,30433],{},[91,30430,335],{},[461,30432,815],{},[81,30434,83],{},[91,30436,7322],{},[14683,30438,30439,30441,30449],{},[461,30440,1826],{},[75,30442,30443,30445,30447],{},[91,30444,777],{},[461,30446,815],{},[81,30448,83],{},[88,30450,30451,30453],{},[91,30452,13242],{},[91,30454,335],{},[1822,30456,30457,30459,30461],{},[91,30458,774],{},[91,30460,14051],{},[75,30462,30463,30465,30467],{},[91,30464,27510],{},[91,30466,774],{},[91,30468,5607],{},[461,30470,798],{"stretchy":797},[1822,30472,30473,30475,30477],{},[91,30474,12802],{},[91,30476,335],{},[91,30478,777],{},[461,30480,803],{"separator":106},[1822,30482,30483,30485,30487],{},[91,30484,22],{},[91,30486,335],{},[91,30488,777],{},[461,30490,812],{"stretchy":797},[3277,30492,27595],{},[91,30494,14380],{"mathvariant":818},[91,30496,14805],{},[461,30498,822],{},[88,30500,30501,30503],{},[91,30502,5604],{},[91,30504,14051],{},[461,30506,798],{"stretchy":797},[1822,30508,30509,30511,30513],{},[91,30510,22],{},[91,30512,335],{},[91,30514,777],{},[91,30516,12941],{"mathvariant":818},[1822,30518,30519,30521,30523],{},[91,30520,12802],{},[91,30522,335],{},[91,30524,777],{},[461,30526,812],{"stretchy":797},[98,30528,30529],{"encoding":100},"\\delta_t^V = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t) \\\\\nA_\\theta^{GAE}(s_t, a) = \\sum_{b=0}^{\\infty} (\\gamma\\lambda)^b \\delta_{t+b}^V \\\\\n\\frac{1}{N}\\sum_{n=1}^N\\sum_{t=1}^{T_n} A_\\theta^{GAE}(s_n^t, a_n^t)\\, \\nabla \\log P_\\theta(a_n^t|s_n^t)",[59,30531,30533,30599,30654,30767,30859,30862,30992,31164,31167],{"className":30532,"ariaHidden":106},[105],[59,30534,30536,30539,30590,30593,30596],{"className":30535},[110],[59,30537],{"className":30538,"style":25905},[114],[59,30540,30542,30545],{"className":30541},[119],[59,30543,25477],{"className":30544,"style":25498},[119,192],[59,30546,30548],{"className":30547},[196],[59,30549,30551,30582],{"className":30550},[131,132],[59,30552,30554,30579],{"className":30553},[136],[59,30555,30557,30568],{"className":30556,"style":24796},[140],[59,30558,30559,30562],{"style":25926},[59,30560],{"className":30561,"style":540},[148],[59,30563,30565],{"className":30564},[153,154,155,156],[59,30566,777],{"className":30567},[119,192,156],[59,30569,30570,30573],{"style":16426},[59,30571],{"className":30572,"style":540},[148],[59,30574,30576],{"className":30575},[153,154,155,156],[59,30577,672],{"className":30578,"style":702},[119,192,156],[59,30580,227],{"className":30581},[226],[59,30583,30585],{"className":30584},[136],[59,30586,30588],{"className":30587,"style":17341},[140],[59,30589],{},[59,30591],{"className":30592,"style":500},[499],[59,30594,815],{"className":30595},[504],[59,30597],{"className":30598,"style":500},[499],[59,30600,30602,30605,30645,30648,30651],{"className":30601},[110],[59,30603],{"className":30604,"style":21912},[114],[59,30606,30608,30611],{"className":30607},[119],[59,30609,7826],{"className":30610,"style":3210},[119,192],[59,30612,30614],{"className":30613},[196],[59,30615,30617,30637],{"className":30616},[131,132],[59,30618,30620,30634],{"className":30619},[136],[59,30621,30623],{"className":30622,"style":12732},[140],[59,30624,30625,30628],{"style":13105},[59,30626],{"className":30627,"style":540},[148],[59,30629,30631],{"className":30630},[153,154,155,156],[59,30632,777],{"className":30633},[119,192,156],[59,30635,227],{"className":30636},[226],[59,30638,30640],{"className":30639},[136],[59,30641,30643],{"className":30642,"style":1270},[140],[59,30644],{},[59,30646],{"className":30647,"style":1872},[499],[59,30649,5777],{"className":30650},[553],[59,30652],{"className":30653,"style":1872},[499],[59,30655,30657,30660,30663,30703,30706,30755,30758,30761,30764],{"className":30656},[110],[59,30658],{"className":30659,"style":877},[114],[59,30661,18995],{"className":30662,"style":19242},[119,192],[59,30664,30666,30669],{"className":30665},[119],[59,30667,672],{"className":30668,"style":702},[119,192],[59,30670,30672],{"className":30671},[196],[59,30673,30675,30695],{"className":30674},[131,132],[59,30676,30678,30692],{"className":30677},[136],[59,30679,30681],{"className":30680,"style":1248},[140],[59,30682,30683,30686],{"style":20505},[59,30684],{"className":30685,"style":540},[148],[59,30687,30689],{"className":30688},[153,154,155,156],[59,30690,14051],{"className":30691,"style":3210},[119,192,156],[59,30693,227],{"className":30694},[226],[59,30696,30698],{"className":30697},[136],[59,30699,30701],{"className":30700,"style":1270},[140],[59,30702],{},[59,30704,798],{"className":30705},[123],[59,30707,30709,30712],{"className":30708},[119],[59,30710,12802],{"className":30711},[119,192],[59,30713,30715],{"className":30714},[196],[59,30716,30718,30747],{"className":30717},[131,132],[59,30719,30721,30744],{"className":30720},[136],[59,30722,30724],{"className":30723,"style":3091},[140],[59,30725,30726,30729],{"style":1251},[59,30727],{"className":30728,"style":540},[148],[59,30730,30732],{"className":30731},[153,154,155,156],[59,30733,30735,30738,30741],{"className":30734},[119,156],[59,30736,777],{"className":30737},[119,192,156],[59,30739,5777],{"className":30740},[553,156],[59,30742,83],{"className":30743},[119,156],[59,30745,227],{"className":30746},[226],[59,30748,30750],{"className":30749},[136],[59,30751,30753],{"className":30752,"style":6938},[140],[59,30754],{},[59,30756,812],{"className":30757},[313],[59,30759],{"className":30760,"style":1872},[499],[59,30762,18365],{"className":30763},[553],[59,30765],{"className":30766,"style":1872},[499],[59,30768,30770,30773,30813,30816,30856],{"className":30769},[110],[59,30771],{"className":30772,"style":877},[114],[59,30774,30776,30779],{"className":30775},[119],[59,30777,672],{"className":30778,"style":702},[119,192],[59,30780,30782],{"className":30781},[196],[59,30783,30785,30805],{"className":30784},[131,132],[59,30786,30788,30802],{"className":30787},[136],[59,30789,30791],{"className":30790,"style":1248},[140],[59,30792,30793,30796],{"style":20505},[59,30794],{"className":30795,"style":540},[148],[59,30797,30799],{"className":30798},[153,154,155,156],[59,30800,14051],{"className":30801,"style":3210},[119,192,156],[59,30803,227],{"className":30804},[226],[59,30806,30808],{"className":30807},[136],[59,30809,30811],{"className":30810,"style":1270},[140],[59,30812],{},[59,30814,798],{"className":30815},[123],[59,30817,30819,30822],{"className":30818},[119],[59,30820,12802],{"className":30821},[119,192],[59,30823,30825],{"className":30824},[196],[59,30826,30828,30848],{"className":30827},[131,132],[59,30829,30831,30845],{"className":30830},[136],[59,30832,30834],{"className":30833,"style":12732},[140],[59,30835,30836,30839],{"style":1251},[59,30837],{"className":30838,"style":540},[148],[59,30840,30842],{"className":30841},[153,154,155,156],[59,30843,777],{"className":30844},[119,192,156],[59,30846,227],{"className":30847},[226],[59,30849,30851],{"className":30850},[136],[59,30852,30854],{"className":30853,"style":1270},[140],[59,30855],{},[59,30857,812],{"className":30858},[313],[59,30860],{"className":30861},[499,21653],[59,30863,30865,30868,30928,30931,30971,30974,30977,30980,30983,30986,30989],{"className":30864},[110],[59,30866],{"className":30867,"style":24777},[114],[59,30869,30871,30874],{"className":30870},[119],[59,30872,774],{"className":30873},[119,192],[59,30875,30877],{"className":30876},[196],[59,30878,30880,30920],{"className":30879},[131,132],[59,30881,30883,30917],{"className":30882},[136],[59,30884,30886,30897],{"className":30885,"style":24796},[140],[59,30887,30888,30891],{"style":17311},[59,30889],{"className":30890,"style":540},[148],[59,30892,30894],{"className":30893},[153,154,155,156],[59,30895,14051],{"className":30896,"style":3210},[119,192,156],[59,30898,30899,30902],{"style":16426},[59,30900],{"className":30901,"style":540},[148],[59,30903,30905],{"className":30904},[153,154,155,156],[59,30906,30908,30911,30914],{"className":30907},[119,156],[59,30909,27510],{"className":30910},[119,192,156],[59,30912,774],{"className":30913},[119,192,156],[59,30915,5607],{"className":30916,"style":6050},[119,192,156],[59,30918,227],{"className":30919},[226],[59,30921,30923],{"className":30922},[136],[59,30924,30926],{"className":30925,"style":17341},[140],[59,30927],{},[59,30929,798],{"className":30930},[123],[59,30932,30934,30937],{"className":30933},[119],[59,30935,12802],{"className":30936},[119,192],[59,30938,30940],{"className":30939},[196],[59,30941,30943,30963],{"className":30942},[131,132],[59,30944,30946,30960],{"className":30945},[136],[59,30947,30949],{"className":30948,"style":12732},[140],[59,30950,30951,30954],{"style":1251},[59,30952],{"className":30953,"style":540},[148],[59,30955,30957],{"className":30956},[153,154,155,156],[59,30958,777],{"className":30959},[119,192,156],[59,30961,227],{"className":30962},[226],[59,30964,30966],{"className":30965},[136],[59,30967,30969],{"className":30968,"style":1270},[140],[59,30970],{},[59,30972,803],{"className":30973},[912],[59,30975],{"className":30976,"style":916},[499],[59,30978,22],{"className":30979},[119,192],[59,30981,812],{"className":30982},[313],[59,30984],{"className":30985,"style":500},[499],[59,30987,815],{"className":30988},[504],[59,30990],{"className":30991,"style":500},[499],[59,30993,30995,30999,31066,31069,31072,31075,31104],{"className":30994},[110],[59,30996],{"className":30997,"style":30998},[114],"height:2.9535em;vertical-align:-1.3021em;",[59,31000,31002],{"className":31001},[951,14238],[59,31003,31005,31058],{"className":31004},[131,132],[59,31006,31008,31055],{"className":31007},[136],[59,31009,31011,31031,31041],{"className":31010,"style":30094},[140],[59,31012,31013,31016],{"style":30097},[59,31014],{"className":31015,"style":15432},[148],[59,31017,31019],{"className":31018},[153,154,155,156],[59,31020,31022,31025,31028],{"className":31021},[119,156],[59,31023,7357],{"className":31024},[119,192,156],[59,31026,815],{"className":31027},[504,156],[59,31029,1754],{"className":31030},[119,156],[59,31032,31033,31036],{"style":15441},[59,31034],{"className":31035,"style":15432},[148],[59,31037,31038],{},[59,31039,1826],{"className":31040},[951,1910,15450],[59,31042,31043,31046],{"style":16374},[59,31044],{"className":31045,"style":15432},[148],[59,31047,31049],{"className":31048},[153,154,155,156],[59,31050,31052],{"className":31051},[119,156],[59,31053,28044],{"className":31054},[119,156],[59,31056,227],{"className":31057},[226],[59,31059,31061],{"className":31060},[136],[59,31062,31064],{"className":31063,"style":30146},[140],[59,31065],{},[59,31067,798],{"className":31068},[123],[59,31070,18995],{"className":31071,"style":19242},[119,192],[59,31073,27460],{"className":31074},[119,192],[59,31076,31078,31081],{"className":31077},[313],[59,31079,812],{"className":31080},[313],[59,31082,31084],{"className":31083},[196],[59,31085,31087],{"className":31086},[131],[59,31088,31090],{"className":31089},[136],[59,31091,31093],{"className":31092,"style":30176},[140],[59,31094,31095,31098],{"style":16426},[59,31096],{"className":31097,"style":540},[148],[59,31099,31101],{"className":31100},[153,154,155,156],[59,31102,7357],{"className":31103},[119,192,156],[59,31105,31107,31110],{"className":31106},[119],[59,31108,25477],{"className":31109,"style":25498},[119,192],[59,31111,31113],{"className":31112},[196],[59,31114,31116,31156],{"className":31115},[131,132],[59,31117,31119,31153],{"className":31118},[136],[59,31120,31122,31142],{"className":31121,"style":24796},[140],[59,31123,31124,31127],{"style":25926},[59,31125],{"className":31126,"style":540},[148],[59,31128,31130],{"className":31129},[153,154,155,156],[59,31131,31133,31136,31139],{"className":31132},[119,156],[59,31134,777],{"className":31135},[119,192,156],[59,31137,5777],{"className":31138},[553,156],[59,31140,7357],{"className":31141},[119,192,156],[59,31143,31144,31147],{"style":16426},[59,31145],{"className":31146,"style":540},[148],[59,31148,31150],{"className":31149},[153,154,155,156],[59,31151,672],{"className":31152,"style":702},[119,192,156],[59,31154,227],{"className":31155},[226],[59,31157,31159],{"className":31158},[136],[59,31160,31162],{"className":31161,"style":26310},[140],[59,31163],{},[59,31165],{"className":31166},[499,21653],[59,31168,31170,31174,31236,31239,31303,31306,31410,31413,31473,31476,31527,31530,31533,31584,31587,31590,31593,31596,31601,31604,31644,31647,31698,31701,31752],{"className":31169},[110],[59,31171],{"className":31172,"style":31173},[114],"height:3.1065em;vertical-align:-1.2671em;",[59,31175,31177,31180,31233],{"className":31176},[119],[59,31178],{"className":31179},[123,124],[59,31181,31183],{"className":31182},[78],[59,31184,31186,31225],{"className":31185},[131,132],[59,31187,31189,31222],{"className":31188},[136],[59,31190,31192,31203,31211],{"className":31191,"style":16279},[140],[59,31193,31194,31197],{"style":15819},[59,31195],{"className":31196,"style":149},[148],[59,31198,31200],{"className":31199},[119],[59,31201,7322],{"className":31202,"style":7397},[119,192],[59,31204,31205,31208],{"style":274},[59,31206],{"className":31207,"style":149},[148],[59,31209],{"className":31210,"style":282},[281],[59,31212,31213,31216],{"style":15885},[59,31214],{"className":31215,"style":149},[148],[59,31217,31219],{"className":31218},[119],[59,31220,83],{"className":31221},[119],[59,31223,227],{"className":31224},[226],[59,31226,31228],{"className":31227},[136],[59,31229,31231],{"className":31230,"style":16319},[140],[59,31232],{},[59,31234],{"className":31235},[313,124],[59,31237],{"className":31238,"style":916},[499],[59,31240,31242],{"className":31241},[951,14238],[59,31243,31245,31295],{"className":31244},[131,132],[59,31246,31248,31292],{"className":31247},[136],[59,31249,31251,31271,31281],{"className":31250,"style":16340},[140],[59,31252,31253,31256],{"style":16343},[59,31254],{"className":31255,"style":15432},[148],[59,31257,31259],{"className":31258},[153,154,155,156],[59,31260,31262,31265,31268],{"className":31261},[119,156],[59,31263,335],{"className":31264},[119,192,156],[59,31266,815],{"className":31267},[504,156],[59,31269,83],{"className":31270},[119,156],[59,31272,31273,31276],{"style":15441},[59,31274],{"className":31275,"style":15432},[148],[59,31277,31278],{},[59,31279,1826],{"className":31280},[951,1910,15450],[59,31282,31283,31286],{"style":16374},[59,31284],{"className":31285,"style":15432},[148],[59,31287,31289],{"className":31288},[153,154,155,156],[59,31290,7322],{"className":31291,"style":7397},[119,192,156],[59,31293,227],{"className":31294},[226],[59,31296,31298],{"className":31297},[136],[59,31299,31301],{"className":31300,"style":16393},[140],[59,31302],{},[59,31304],{"className":31305,"style":916},[499],[59,31307,31309],{"className":31308},[951,14238],[59,31310,31312,31402],{"className":31311},[131,132],[59,31313,31315,31399],{"className":31314},[136],[59,31316,31318,31338,31348],{"className":31317,"style":17149},[140],[59,31319,31320,31323],{"style":16343},[59,31321],{"className":31322,"style":15432},[148],[59,31324,31326],{"className":31325},[153,154,155,156],[59,31327,31329,31332,31335],{"className":31328},[119,156],[59,31330,777],{"className":31331},[119,192,156],[59,31333,815],{"className":31334},[504,156],[59,31336,83],{"className":31337},[119,156],[59,31339,31340,31343],{"style":15441},[59,31341],{"className":31342,"style":15432},[148],[59,31344,31345],{},[59,31346,1826],{"className":31347},[951,1910,15450],[59,31349,31350,31353],{"style":17182},[59,31351],{"className":31352,"style":15432},[148],[59,31354,31356],{"className":31355},[153,154,155,156],[59,31357,31359],{"className":31358},[119,156],[59,31360,31362,31365],{"className":31361},[119,156],[59,31363,13242],{"className":31364,"style":3185},[119,192,156],[59,31366,31368],{"className":31367},[196],[59,31369,31371,31391],{"className":31370},[131,132],[59,31372,31374,31388],{"className":31373},[136],[59,31375,31377],{"className":31376,"style":4302},[140],[59,31378,31379,31382],{"style":17212},[59,31380],{"className":31381,"style":213},[148],[59,31383,31385],{"className":31384},[153,217,218,156],[59,31386,335],{"className":31387},[119,192,156],[59,31389,227],{"className":31390},[226],[59,31392,31394],{"className":31393},[136],[59,31395,31397],{"className":31396,"style":4324},[140],[59,31398],{},[59,31400,227],{"className":31401},[226],[59,31403,31405],{"className":31404},[136],[59,31406,31408],{"className":31407,"style":16393},[140],[59,31409],{},[59,31411],{"className":31412,"style":916},[499],[59,31414,31416,31419],{"className":31415},[119],[59,31417,774],{"className":31418},[119,192],[59,31420,31422],{"className":31421},[196],[59,31423,31425,31465],{"className":31424},[131,132],[59,31426,31428,31462],{"className":31427},[136],[59,31429,31431,31442],{"className":31430,"style":24796},[140],[59,31432,31433,31436],{"style":17311},[59,31434],{"className":31435,"style":540},[148],[59,31437,31439],{"className":31438},[153,154,155,156],[59,31440,14051],{"className":31441,"style":3210},[119,192,156],[59,31443,31444,31447],{"style":16426},[59,31445],{"className":31446,"style":540},[148],[59,31448,31450],{"className":31449},[153,154,155,156],[59,31451,31453,31456,31459],{"className":31452},[119,156],[59,31454,27510],{"className":31455},[119,192,156],[59,31457,774],{"className":31458},[119,192,156],[59,31460,5607],{"className":31461,"style":6050},[119,192,156],[59,31463,227],{"className":31464},[226],[59,31466,31468],{"className":31467},[136],[59,31469,31471],{"className":31470,"style":17341},[140],[59,31472],{},[59,31474,798],{"className":31475},[123],[59,31477,31479,31482],{"className":31478},[119],[59,31480,12802],{"className":31481},[119,192],[59,31483,31485],{"className":31484},[196],[59,31486,31488,31519],{"className":31487},[131,132],[59,31489,31491,31516],{"className":31490},[136],[59,31492,31494,31505],{"className":31493,"style":17308},[140],[59,31495,31496,31499],{"style":17311},[59,31497],{"className":31498,"style":540},[148],[59,31500,31502],{"className":31501},[153,154,155,156],[59,31503,335],{"className":31504},[119,192,156],[59,31506,31507,31510],{"style":16426},[59,31508],{"className":31509,"style":540},[148],[59,31511,31513],{"className":31512},[153,154,155,156],[59,31514,777],{"className":31515},[119,192,156],[59,31517,227],{"className":31518},[226],[59,31520,31522],{"className":31521},[136],[59,31523,31525],{"className":31524,"style":17341},[140],[59,31526],{},[59,31528,803],{"className":31529},[912],[59,31531],{"className":31532,"style":916},[499],[59,31534,31536,31539],{"className":31535},[119],[59,31537,22],{"className":31538},[119,192],[59,31540,31542],{"className":31541},[196],[59,31543,31545,31576],{"className":31544},[131,132],[59,31546,31548,31573],{"className":31547},[136],[59,31549,31551,31562],{"className":31550,"style":17308},[140],[59,31552,31553,31556],{"style":17311},[59,31554],{"className":31555,"style":540},[148],[59,31557,31559],{"className":31558},[153,154,155,156],[59,31560,335],{"className":31561},[119,192,156],[59,31563,31564,31567],{"style":16426},[59,31565],{"className":31566,"style":540},[148],[59,31568,31570],{"className":31569},[153,154,155,156],[59,31571,777],{"className":31572},[119,192,156],[59,31574,227],{"className":31575},[226],[59,31577,31579],{"className":31578},[136],[59,31580,31582],{"className":31581,"style":17341},[140],[59,31583],{},[59,31585,812],{"className":31586},[313],[59,31588],{"className":31589,"style":916},[499],[59,31591,14380],{"className":31592},[119],[59,31594],{"className":31595,"style":916},[499],[59,31597,16850,31599],{"className":31598},[951],[59,31600,16854],{"style":16853},[59,31602],{"className":31603,"style":916},[499],[59,31605,31607,31610],{"className":31606},[119],[59,31608,5604],{"className":31609,"style":3185},[119,192],[59,31611,31613],{"className":31612},[196],[59,31614,31616,31636],{"className":31615},[131,132],[59,31617,31619,31633],{"className":31618},[136],[59,31620,31622],{"className":31621,"style":1248},[140],[59,31623,31624,31627],{"style":7460},[59,31625],{"className":31626,"style":540},[148],[59,31628,31630],{"className":31629},[153,154,155,156],[59,31631,14051],{"className":31632,"style":3210},[119,192,156],[59,31634,227],{"className":31635},[226],[59,31637,31639],{"className":31638},[136],[59,31640,31642],{"className":31641,"style":1270},[140],[59,31643],{},[59,31645,798],{"className":31646},[123],[59,31648,31650,31653],{"className":31649},[119],[59,31651,22],{"className":31652},[119,192],[59,31654,31656],{"className":31655},[196],[59,31657,31659,31690],{"className":31658},[131,132],[59,31660,31662,31687],{"className":31661},[136],[59,31663,31665,31676],{"className":31664,"style":17308},[140],[59,31666,31667,31670],{"style":17311},[59,31668],{"className":31669,"style":540},[148],[59,31671,31673],{"className":31672},[153,154,155,156],[59,31674,335],{"className":31675},[119,192,156],[59,31677,31678,31681],{"style":16426},[59,31679],{"className":31680,"style":540},[148],[59,31682,31684],{"className":31683},[153,154,155,156],[59,31685,777],{"className":31686},[119,192,156],[59,31688,227],{"className":31689},[226],[59,31691,31693],{"className":31692},[136],[59,31694,31696],{"className":31695,"style":17341},[140],[59,31697],{},[59,31699,12941],{"className":31700},[119],[59,31702,31704,31707],{"className":31703},[119],[59,31705,12802],{"className":31706},[119,192],[59,31708,31710],{"className":31709},[196],[59,31711,31713,31744],{"className":31712},[131,132],[59,31714,31716,31741],{"className":31715},[136],[59,31717,31719,31730],{"className":31718,"style":17308},[140],[59,31720,31721,31724],{"style":17311},[59,31722],{"className":31723,"style":540},[148],[59,31725,31727],{"className":31726},[153,154,155,156],[59,31728,335],{"className":31729},[119,192,156],[59,31731,31732,31735],{"style":16426},[59,31733],{"className":31734,"style":540},[148],[59,31736,31738],{"className":31737},[153,154,155,156],[59,31739,777],{"className":31740},[119,192,156],[59,31742,227],{"className":31743},[226],[59,31745,31747],{"className":31746},[136],[59,31748,31750],{"className":31749,"style":17341},[140],[59,31751],{},[59,31753,812],{"className":31754},[313],[15,31756,31757],{},"这里的价值状态函数我们一般用一个神经网路进行拟合，一般可以和策略网络公用参数，只是最后一层不同。",[11,31759,31761],{"id":31760},"proximal-policy-optimization-ppo-近邻策略优化算法","Proximal Policy Optimization (PPO) 近邻策略优化算法",[15,31763,31764],{},"在经典的强化学习环境中，我们一般是一边采集数据，一边进行训练模型，采集完的数据用完就会丢掉。这导致我们采集的数据发生了一定的浪费，且模型训练的成本也会增高。但如果我们有一种方法，能让模型在其他模型的表现中学习自己的表现，参考其他数据就能实现自身的更新，这样我们的训练成本就会大大减少。",[49,31766,31767],{"id":31767},"重要性采样",[15,31769,31770,31771,31815,31816,31844,31845,31873,31874,31902],{},"将 ",[59,31772,31774,31794],{"className":31773},[62],[59,31775,31777],{"className":31776},[66],[68,31778,31779],{"xmlns":70},[72,31780,31781,31791],{},[75,31782,31783,31785,31787,31789],{},[91,31784,11657],{},[461,31786,798],{"stretchy":797},[91,31788,7327],{},[461,31790,812],{"stretchy":797},[98,31792,31793],{"encoding":100},"f(x)",[59,31795,31797],{"className":31796,"ariaHidden":106},[105],[59,31798,31800,31803,31806,31809,31812],{"className":31799},[110],[59,31801],{"className":31802,"style":877},[114],[59,31804,11657],{"className":31805,"style":13619},[119,192],[59,31807,798],{"className":31808},[123],[59,31810,7327],{"className":31811},[119,192],[59,31813,812],{"className":31814},[313]," 在分布 ",[59,31817,31819,31832],{"className":31818},[62],[59,31820,31822],{"className":31821},[66],[68,31823,31824],{"xmlns":70},[72,31825,31826,31830],{},[75,31827,31828],{},[91,31829,15],{},[98,31831,15],{"encoding":100},[59,31833,31835],{"className":31834,"ariaHidden":106},[105],[59,31836,31838,31841],{"className":31837},[110],[59,31839],{"className":31840,"style":1706},[114],[59,31842,15],{"className":31843},[119,192]," 下的期望改写为在另一分布 ",[59,31846,31848,31861],{"className":31847},[62],[59,31849,31851],{"className":31850},[66],[68,31852,31853],{"xmlns":70},[72,31854,31855,31859],{},[75,31856,31857],{},[91,31858,1694],{},[98,31860,1694],{"encoding":100},[59,31862,31864],{"className":31863,"ariaHidden":106},[105],[59,31865,31867,31870],{"className":31866},[110],[59,31868],{"className":31869,"style":1706},[114],[59,31871,1694],{"className":31872,"style":441},[119,192],"（proposal）下的期望，从而可用从 ",[59,31875,31877,31890],{"className":31876},[62],[59,31878,31880],{"className":31879},[66],[68,31881,31882],{"xmlns":70},[72,31883,31884,31888],{},[75,31885,31886],{},[91,31887,1694],{},[98,31889,1694],{"encoding":100},[59,31891,31893],{"className":31892,"ariaHidden":106},[105],[59,31894,31896,31899],{"className":31895},[110],[59,31897],{"className":31898,"style":1706},[114],[59,31900,1694],{"className":31901,"style":441},[119,192]," 采样的数据估计：",[59,31904,31906],{"className":31905},[14350],[59,31907,31909,32293],{"className":31908},[62],[59,31910,31912],{"className":31911},[66],[68,31913,31914],{"xmlns":70,"display":9481},[72,31915,31916,32290],{},[14362,31917,31918,31990,32054,32118,32190],{"rowspacing":14364,"columnalign":14365,"columnspacing":14366},[14368,31919,31920,31956],{},[14371,31921,31922],{},[14374,31923,31924],{"scriptlevel":1754,"displaystyle":106},[75,31925,31926,31928,31930,31932,31934,31936,31938],{},[91,31927,5607],{"mathvariant":469},[461,31929,798],{"stretchy":797},[91,31931,11657],{},[461,31933,798],{"stretchy":797},[91,31935,7327],{},[461,31937,812],{"stretchy":797},[88,31939,31940,31942],{},[461,31941,812],{"stretchy":797},[75,31943,31944,31946,31948,31950,31952,31954],{},[91,31945,7327],{},[461,31947,14044],{},[91,31949,15],{},[461,31951,798],{"stretchy":797},[91,31953,7327],{},[461,31955,812],{"stretchy":797},[14371,31957,31958],{},[14374,31959,31960],{"scriptlevel":1754,"displaystyle":106},[75,31961,31962,31964,31966,31972,31974,31976,31978,31980,31982,31984,31986,31988],{},[75,31963],{},[461,31965,815],{},[14061,31967,31968,31970],{},[461,31969,1826],{},[91,31971,7327],{},[91,31973,11657],{},[461,31975,798],{"stretchy":797},[91,31977,7327],{},[461,31979,812],{"stretchy":797},[461,31981,1816],{},[91,31983,15],{},[461,31985,798],{"stretchy":797},[91,31987,7327],{},[461,31989,812],{"stretchy":797},[14368,31991,31992,31998],{},[14371,31993,31994],{},[14374,31995,31996],{"scriptlevel":1754,"displaystyle":106},[75,31997],{},[14371,31999,32000],{},[14374,32001,32002],{"scriptlevel":1754,"displaystyle":106},[75,32003,32004,32006,32008,32014,32016,32018,32020,32022,32024,32026,32028,32030,32032],{},[75,32005],{},[461,32007,815],{},[14061,32009,32010,32012],{},[461,32011,1826],{},[91,32013,7327],{},[91,32015,11657],{},[461,32017,798],{"stretchy":797},[91,32019,7327],{},[461,32021,812],{"stretchy":797},[461,32023,1816],{},[91,32025,15],{},[461,32027,798],{"stretchy":797},[91,32029,7327],{},[461,32031,812],{"stretchy":797},[78,32033,32034,32044],{},[75,32035,32036,32038,32040,32042],{},[91,32037,1694],{},[461,32039,798],{"stretchy":797},[91,32041,7327],{},[461,32043,812],{"stretchy":797},[75,32045,32046,32048,32050,32052],{},[91,32047,1694],{},[461,32049,798],{"stretchy":797},[91,32051,7327],{},[461,32053,812],{"stretchy":797},[14368,32055,32056,32062],{},[14371,32057,32058],{},[14374,32059,32060],{"scriptlevel":1754,"displaystyle":106},[75,32061],{},[14371,32063,32064],{},[14374,32065,32066],{"scriptlevel":1754,"displaystyle":106},[75,32067,32068,32070,32072,32078,32080,32082,32084,32086,32108,32110,32112,32114,32116],{},[75,32069],{},[461,32071,815],{},[14061,32073,32074,32076],{},[461,32075,1826],{},[91,32077,7327],{},[91,32079,11657],{},[461,32081,798],{"stretchy":797},[91,32083,7327],{},[461,32085,812],{"stretchy":797},[78,32087,32088,32098],{},[75,32089,32090,32092,32094,32096],{},[91,32091,15],{},[461,32093,798],{"stretchy":797},[91,32095,7327],{},[461,32097,812],{"stretchy":797},[75,32099,32100,32102,32104,32106],{},[91,32101,1694],{},[461,32103,798],{"stretchy":797},[91,32105,7327],{},[461,32107,812],{"stretchy":797},[461,32109,1816],{},[91,32111,1694],{},[461,32113,798],{"stretchy":797},[91,32115,7327],{},[461,32117,812],{"stretchy":797},[14368,32119,32120,32126],{},[14371,32121,32122],{},[14374,32123,32124],{"scriptlevel":1754,"displaystyle":106},[75,32125],{},[14371,32127,32128],{},[14374,32129,32130],{"scriptlevel":1754,"displaystyle":106},[75,32131,32132,32134,32136,32138],{},[75,32133],{},[461,32135,815],{},[91,32137,5607],{"mathvariant":469},[88,32139,32140,32176],{},[75,32141,32142,32144,32146,32148,32150,32152,32174],{},[461,32143,798],{"fence":106},[91,32145,11657],{},[461,32147,798],{"stretchy":797},[91,32149,7327],{},[461,32151,812],{"stretchy":797},[78,32153,32154,32164],{},[75,32155,32156,32158,32160,32162],{},[91,32157,15],{},[461,32159,798],{"stretchy":797},[91,32161,7327],{},[461,32163,812],{"stretchy":797},[75,32165,32166,32168,32170,32172],{},[91,32167,1694],{},[461,32169,798],{"stretchy":797},[91,32171,7327],{},[461,32173,812],{"stretchy":797},[461,32175,812],{"fence":106},[75,32177,32178,32180,32182,32184,32186,32188],{},[91,32179,7327],{},[461,32181,14044],{},[91,32183,1694],{},[461,32185,798],{"stretchy":797},[91,32187,7327],{},[461,32189,812],{"stretchy":797},[14368,32191,32192,32198],{},[14371,32193,32194],{},[14374,32195,32196],{"scriptlevel":1754,"displaystyle":106},[75,32197],{},[14371,32199,32200],{},[14374,32201,32202],{"scriptlevel":1754,"displaystyle":106},[75,32203,32204,32206,32208,32214,32228,32230,32232,32238,32240,32270,32272,32274,32280,32282,32284,32286,32288],{},[75,32205],{},[461,32207,14675],{},[78,32209,32210,32212],{},[81,32211,83],{},[91,32213,7322],{},[14683,32215,32216,32218,32226],{},[461,32217,1826],{},[75,32219,32220,32222,32224],{},[91,32221,335],{},[461,32223,815],{},[81,32225,83],{},[91,32227,7322],{},[91,32229,11657],{},[461,32231,798],{"stretchy":797},[88,32233,32234,32236],{},[91,32235,7327],{},[91,32237,335],{},[461,32239,812],{"stretchy":797},[78,32241,32242,32256],{},[75,32243,32244,32246,32248,32254],{},[91,32245,15],{},[461,32247,798],{"stretchy":797},[88,32249,32250,32252],{},[91,32251,7327],{},[91,32253,335],{},[461,32255,812],{"stretchy":797},[75,32257,32258,32260,32262,32268],{},[91,32259,1694],{},[461,32261,798],{"stretchy":797},[88,32263,32264,32266],{},[91,32265,7327],{},[91,32267,335],{},[461,32269,812],{"stretchy":797},[461,32271,803],{"separator":106},[499,32273],{"width":27600},[88,32275,32276,32278],{},[91,32277,7327],{},[91,32279,335],{},[461,32281,14044],{},[91,32283,1694],{},[461,32285,798],{"stretchy":797},[91,32287,7327],{},[461,32289,812],{"stretchy":797},[98,32291,32292],{"encoding":100},"\\begin{aligned}\n\\mathbb{E}(f(x))_{x \\sim p(x)} &= \\sum_{x} f(x) \\cdot p(x) \\\\\n&= \\sum_{x} f(x) \\cdot p(x) \\frac{q(x)}{q(x)} \\\\\n&= \\sum_{x} f(x) \\frac{p(x)}{q(x)} \\cdot q(x) \\\\\n&= \\mathbb{E}\\left(f(x) \\frac{p(x)}{q(x)}\\right)_{x \\sim q(x)} \\\\\n&\\approx \\frac{1}{N} \\sum_{n=1}^{N} f(x_n) \\frac{p(x_n)}{q(x_n)}, \\quad x_n \\sim q(x)\n\\end{aligned}",[59,32294,32296],{"className":32295,"ariaHidden":106},[105],[59,32297,32299,32303],{"className":32298},[110],[59,32300],{"className":32301,"style":32302},[114],"height:14.8742em;vertical-align:-7.1871em;",[59,32304,32306],{"className":32305},[119],[59,32307,32309,32456],{"className":32308},[14362],[59,32310,32312],{"className":32311},[15153],[59,32313,32315,32447],{"className":32314},[131,132],[59,32316,32318,32444],{"className":32317},[136],[59,32319,32322,32408,32417,32426,32435],{"className":32320,"style":32321},[140],"height:7.6871em;",[59,32323,32325,32329],{"style":32324},"top:-10.4654em;",[59,32326],{"className":32327,"style":32328},[148],"height:3.8283em;",[59,32330,32332,32335,32338,32341,32344,32347,32350],{"className":32331},[119],[59,32333,5607],{"className":32334},[119,521],[59,32336,798],{"className":32337},[123],[59,32339,11657],{"className":32340,"style":13619},[119,192],[59,32342,798],{"className":32343},[123],[59,32345,7327],{"className":32346},[119,192],[59,32348,812],{"className":32349},[313],[59,32351,32353,32356],{"className":32352},[313],[59,32354,812],{"className":32355},[313],[59,32357,32359],{"className":32358},[196],[59,32360,32362,32400],{"className":32361},[131,132],[59,32363,32365,32397],{"className":32364},[136],[59,32366,32368],{"className":32367,"style":206},[140],[59,32369,32370,32373],{"style":14139},[59,32371],{"className":32372,"style":540},[148],[59,32374,32376],{"className":32375},[153,154,155,156],[59,32377,32379,32382,32385,32388,32391,32394],{"className":32378},[119,156],[59,32380,7327],{"className":32381},[119,192,156],[59,32383,14044],{"className":32384},[504,156],[59,32386,15],{"className":32387},[119,192,156],[59,32389,798],{"className":32390},[123,156],[59,32392,7327],{"className":32393},[119,192,156],[59,32395,812],{"className":32396},[313,156],[59,32398,227],{"className":32399},[226],[59,32401,32403],{"className":32402},[136],[59,32404,32406],{"className":32405,"style":6102},[140],[59,32407],{},[59,32409,32411,32414],{"style":32410},"top:-7.4884em;",[59,32412],{"className":32413,"style":32328},[148],[59,32415],{"className":32416},[119],[59,32418,32420,32423],{"style":32419},"top:-4.5114em;",[59,32421],{"className":32422,"style":32328},[148],[59,32424],{"className":32425},[119],[59,32427,32429,32432],{"style":32428},"top:-1.5114em;",[59,32430],{"className":32431,"style":32328},[148],[59,32433],{"className":32434},[119],[59,32436,32438,32441],{"style":32437},"top:1.7917em;",[59,32439],{"className":32440,"style":32328},[148],[59,32442],{"className":32443},[119],[59,32445,227],{"className":32446},[226],[59,32448,32450],{"className":32449},[136],[59,32451,32454],{"className":32452,"style":32453},[140],"height:7.1871em;",[59,32455],{},[59,32457,32459],{"className":32458},[15377],[59,32460,32462,33559],{"className":32461},[131,132],[59,32463,32465,33556],{"className":32464},[136],[59,32466,32468,32571,32754,32937,33128],{"className":32467,"style":32321},[140],[59,32469,32470,32473],{"style":32324},[59,32471],{"className":32472,"style":32328},[148],[59,32474,32476,32479,32482,32485,32488,32535,32538,32541,32544,32547,32550,32553,32556,32559,32562,32565,32568],{"className":32475},[119],[59,32477],{"className":32478},[119],[59,32480],{"className":32481,"style":500},[499],[59,32483,815],{"className":32484},[504],[59,32486],{"className":32487,"style":500},[499],[59,32489,32491],{"className":32490},[951,14238],[59,32492,32494,32527],{"className":32493},[131,132],[59,32495,32497,32524],{"className":32496},[136],[59,32498,32500,32514],{"className":32499,"style":15425},[140],[59,32501,32502,32505],{"style":15428},[59,32503],{"className":32504,"style":15432},[148],[59,32506,32508],{"className":32507},[153,154,155,156],[59,32509,32511],{"className":32510},[119,156],[59,32512,7327],{"className":32513},[119,192,156],[59,32515,32516,32519],{"style":15441},[59,32517],{"className":32518,"style":15432},[148],[59,32520,32521],{},[59,32522,1826],{"className":32523},[951,1910,15450],[59,32525,227],{"className":32526},[226],[59,32528,32530],{"className":32529},[136],[59,32531,32533],{"className":32532,"style":15460},[140],[59,32534],{},[59,32536],{"className":32537,"style":916},[499],[59,32539,11657],{"className":32540,"style":13619},[119,192],[59,32542,798],{"className":32543},[123],[59,32545,7327],{"className":32546},[119,192],[59,32548,812],{"className":32549},[313],[59,32551],{"className":32552,"style":1872},[499],[59,32554,1816],{"className":32555},[553],[59,32557],{"className":32558,"style":1872},[499],[59,32560,15],{"className":32561},[119,192],[59,32563,798],{"className":32564},[123],[59,32566,7327],{"className":32567},[119,192],[59,32569,812],{"className":32570},[313],[59,32572,32573,32576],{"style":32410},[59,32574],{"className":32575,"style":32328},[148],[59,32577,32579,32582,32585,32588,32591,32638,32641,32644,32647,32650,32653,32656,32659,32662,32665,32668,32671,32674],{"className":32578},[119],[59,32580],{"className":32581},[119],[59,32583],{"className":32584,"style":500},[499],[59,32586,815],{"className":32587},[504],[59,32589],{"className":32590,"style":500},[499],[59,32592,32594],{"className":32593},[951,14238],[59,32595,32597,32630],{"className":32596},[131,132],[59,32598,32600,32627],{"className":32599},[136],[59,32601,32603,32617],{"className":32602,"style":15425},[140],[59,32604,32605,32608],{"style":15428},[59,32606],{"className":32607,"style":15432},[148],[59,32609,32611],{"className":32610},[153,154,155,156],[59,32612,32614],{"className":32613},[119,156],[59,32615,7327],{"className":32616},[119,192,156],[59,32618,32619,32622],{"style":15441},[59,32620],{"className":32621,"style":15432},[148],[59,32623,32624],{},[59,32625,1826],{"className":32626},[951,1910,15450],[59,32628,227],{"className":32629},[226],[59,32631,32633],{"className":32632},[136],[59,32634,32636],{"className":32635,"style":15460},[140],[59,32637],{},[59,32639],{"className":32640,"style":916},[499],[59,32642,11657],{"className":32643,"style":13619},[119,192],[59,32645,798],{"className":32646},[123],[59,32648,7327],{"className":32649},[119,192],[59,32651,812],{"className":32652},[313],[59,32654],{"className":32655,"style":1872},[499],[59,32657,1816],{"className":32658},[553],[59,32660],{"className":32661,"style":1872},[499],[59,32663,15],{"className":32664},[119,192],[59,32666,798],{"className":32667},[123],[59,32669,7327],{"className":32670},[119,192],[59,32672,812],{"className":32673},[313],[59,32675,32677,32680,32751],{"className":32676},[119],[59,32678],{"className":32679},[123,124],[59,32681,32683],{"className":32682},[78],[59,32684,32686,32743],{"className":32685},[131,132],[59,32687,32689,32740],{"className":32688},[136],[59,32690,32692,32712,32720],{"className":32691,"style":15816},[140],[59,32693,32694,32697],{"style":15819},[59,32695],{"className":32696,"style":149},[148],[59,32698,32700,32703,32706,32709],{"className":32699},[119],[59,32701,1694],{"className":32702,"style":441},[119,192],[59,32704,798],{"className":32705},[123],[59,32707,7327],{"className":32708},[119,192],[59,32710,812],{"className":32711},[313],[59,32713,32714,32717],{"style":274},[59,32715],{"className":32716,"style":149},[148],[59,32718],{"className":32719,"style":282},[281],[59,32721,32722,32725],{"style":15885},[59,32723],{"className":32724,"style":149},[148],[59,32726,32728,32731,32734,32737],{"className":32727},[119],[59,32729,1694],{"className":32730,"style":441},[119,192],[59,32732,798],{"className":32733},[123],[59,32735,7327],{"className":32736},[119,192],[59,32738,812],{"className":32739},[313],[59,32741,227],{"className":32742},[226],[59,32744,32746],{"className":32745},[136],[59,32747,32749],{"className":32748,"style":15950},[140],[59,32750],{},[59,32752],{"className":32753},[313,124],[59,32755,32756,32759],{"style":32419},[59,32757],{"className":32758,"style":32328},[148],[59,32760,32762,32765,32768,32771,32774,32821,32824,32827,32830,32833,32836,32916,32919,32922,32925,32928,32931,32934],{"className":32761},[119],[59,32763],{"className":32764},[119],[59,32766],{"className":32767,"style":500},[499],[59,32769,815],{"className":32770},[504],[59,32772],{"className":32773,"style":500},[499],[59,32775,32777],{"className":32776},[951,14238],[59,32778,32780,32813],{"className":32779},[131,132],[59,32781,32783,32810],{"className":32782},[136],[59,32784,32786,32800],{"className":32785,"style":15425},[140],[59,32787,32788,32791],{"style":15428},[59,32789],{"className":32790,"style":15432},[148],[59,32792,32794],{"className":32793},[153,154,155,156],[59,32795,32797],{"className":32796},[119,156],[59,32798,7327],{"className":32799},[119,192,156],[59,32801,32802,32805],{"style":15441},[59,32803],{"className":32804,"style":15432},[148],[59,32806,32807],{},[59,32808,1826],{"className":32809},[951,1910,15450],[59,32811,227],{"className":32812},[226],[59,32814,32816],{"className":32815},[136],[59,32817,32819],{"className":32818,"style":15460},[140],[59,32820],{},[59,32822],{"className":32823,"style":916},[499],[59,32825,11657],{"className":32826,"style":13619},[119,192],[59,32828,798],{"className":32829},[123],[59,32831,7327],{"className":32832},[119,192],[59,32834,812],{"className":32835},[313],[59,32837,32839,32842,32913],{"className":32838},[119],[59,32840],{"className":32841},[123,124],[59,32843,32845],{"className":32844},[78],[59,32846,32848,32905],{"className":32847},[131,132],[59,32849,32851,32902],{"className":32850},[136],[59,32852,32854,32874,32882],{"className":32853,"style":15816},[140],[59,32855,32856,32859],{"style":15819},[59,32857],{"className":32858,"style":149},[148],[59,32860,32862,32865,32868,32871],{"className":32861},[119],[59,32863,1694],{"className":32864,"style":441},[119,192],[59,32866,798],{"className":32867},[123],[59,32869,7327],{"className":32870},[119,192],[59,32872,812],{"className":32873},[313],[59,32875,32876,32879],{"style":274},[59,32877],{"className":32878,"style":149},[148],[59,32880],{"className":32881,"style":282},[281],[59,32883,32884,32887],{"style":15885},[59,32885],{"className":32886,"style":149},[148],[59,32888,32890,32893,32896,32899],{"className":32889},[119],[59,32891,15],{"className":32892},[119,192],[59,32894,798],{"className":32895},[123],[59,32897,7327],{"className":32898},[119,192],[59,32900,812],{"className":32901},[313],[59,32903,227],{"className":32904},[226],[59,32906,32908],{"className":32907},[136],[59,32909,32911],{"className":32910,"style":15950},[140],[59,32912],{},[59,32914],{"className":32915},[313,124],[59,32917],{"className":32918,"style":1872},[499],[59,32920,1816],{"className":32921},[553],[59,32923],{"className":32924,"style":1872},[499],[59,32926,1694],{"className":32927,"style":441},[119,192],[59,32929,798],{"className":32930},[123],[59,32932,7327],{"className":32933},[119,192],[59,32935,812],{"className":32936},[313],[59,32938,32939,32942],{"style":32428},[59,32940],{"className":32941,"style":32328},[148],[59,32943,32945,32948,32951,32954,32957,32960,32963],{"className":32944},[119],[59,32946],{"className":32947},[119],[59,32949],{"className":32950,"style":500},[499],[59,32952,815],{"className":32953},[504],[59,32955],{"className":32956,"style":500},[499],[59,32958,5607],{"className":32959},[119,521],[59,32961],{"className":32962,"style":916},[499],[59,32964,32966,33073],{"className":32965},[962],[59,32967,32969,32975,32978,32981,32984,32987,33067],{"className":32968},[962],[59,32970,32972],{"className":32971,"style":967},[123,966],[59,32973,798],{"className":32974},[971,155],[59,32976,11657],{"className":32977,"style":13619},[119,192],[59,32979,798],{"className":32980},[123],[59,32982,7327],{"className":32983},[119,192],[59,32985,812],{"className":32986},[313],[59,32988,32990,32993,33064],{"className":32989},[119],[59,32991],{"className":32992},[123,124],[59,32994,32996],{"className":32995},[78],[59,32997,32999,33056],{"className":32998},[131,132],[59,33000,33002,33053],{"className":33001},[136],[59,33003,33005,33025,33033],{"className":33004,"style":15816},[140],[59,33006,33007,33010],{"style":15819},[59,33008],{"className":33009,"style":149},[148],[59,33011,33013,33016,33019,33022],{"className":33012},[119],[59,33014,1694],{"className":33015,"style":441},[119,192],[59,33017,798],{"className":33018},[123],[59,33020,7327],{"className":33021},[119,192],[59,33023,812],{"className":33024},[313],[59,33026,33027,33030],{"style":274},[59,33028],{"className":33029,"style":149},[148],[59,33031],{"className":33032,"style":282},[281],[59,33034,33035,33038],{"style":15885},[59,33036],{"className":33037,"style":149},[148],[59,33039,33041,33044,33047,33050],{"className":33040},[119],[59,33042,15],{"className":33043},[119,192],[59,33045,798],{"className":33046},[123],[59,33048,7327],{"className":33049},[119,192],[59,33051,812],{"className":33052},[313],[59,33054,227],{"className":33055},[226],[59,33057,33059],{"className":33058},[136],[59,33060,33062],{"className":33061,"style":15950},[140],[59,33063],{},[59,33065],{"className":33066},[313,124],[59,33068,33070],{"className":33069,"style":967},[313,966],[59,33071,812],{"className":33072},[971,155],[59,33074,33076],{"className":33075},[196],[59,33077,33079,33119],{"className":33078},[131,132],[59,33080,33082,33116],{"className":33081},[136],[59,33083,33086],{"className":33084,"style":33085},[140],"height:-0.4747em;",[59,33087,33089,33092],{"style":33088},"top:-1.7003em;margin-right:0.05em;",[59,33090],{"className":33091,"style":540},[148],[59,33093,33095],{"className":33094},[153,154,155,156],[59,33096,33098,33101,33104,33107,33110,33113],{"className":33097},[119,156],[59,33099,7327],{"className":33100},[119,192,156],[59,33102,14044],{"className":33103},[504,156],[59,33105,1694],{"className":33106,"style":441},[119,192,156],[59,33108,798],{"className":33109},[123,156],[59,33111,7327],{"className":33112},[119,192,156],[59,33114,812],{"className":33115},[313,156],[59,33117,227],{"className":33118},[226],[59,33120,33122],{"className":33121},[136],[59,33123,33126],{"className":33124,"style":33125},[140],"height:1.1747em;",[59,33127],{},[59,33129,33130,33133],{"style":32437},[59,33131],{"className":33132,"style":32328},[148],[59,33134,33136,33139,33142,33145,33148,33210,33213,33280,33283,33286,33289,33329,33332,33486,33489,33492,33495,33535,33538,33541,33544,33547,33550,33553],{"className":33135},[119],[59,33137],{"className":33138},[119],[59,33140],{"className":33141,"style":500},[499],[59,33143,14675],{"className":33144},[504],[59,33146],{"className":33147,"style":500},[499],[59,33149,33151,33154,33207],{"className":33150},[119],[59,33152],{"className":33153},[123,124],[59,33155,33157],{"className":33156},[78],[59,33158,33160,33199],{"className":33159},[131,132],[59,33161,33163,33196],{"className":33162},[136],[59,33164,33166,33177,33185],{"className":33165,"style":16279},[140],[59,33167,33168,33171],{"style":15819},[59,33169],{"className":33170,"style":149},[148],[59,33172,33174],{"className":33173},[119],[59,33175,7322],{"className":33176,"style":7397},[119,192],[59,33178,33179,33182],{"style":274},[59,33180],{"className":33181,"style":149},[148],[59,33183],{"className":33184,"style":282},[281],[59,33186,33187,33190],{"style":15885},[59,33188],{"className":33189,"style":149},[148],[59,33191,33193],{"className":33192},[119],[59,33194,83],{"className":33195},[119],[59,33197,227],{"className":33198},[226],[59,33200,33202],{"className":33201},[136],[59,33203,33205],{"className":33204,"style":16319},[140],[59,33206],{},[59,33208],{"className":33209},[313,124],[59,33211],{"className":33212,"style":916},[499],[59,33214,33216],{"className":33215},[951,14238],[59,33217,33219,33272],{"className":33218},[131,132],[59,33220,33222,33269],{"className":33221},[136],[59,33223,33225,33245,33255],{"className":33224,"style":16340},[140],[59,33226,33227,33230],{"style":16343},[59,33228],{"className":33229,"style":15432},[148],[59,33231,33233],{"className":33232},[153,154,155,156],[59,33234,33236,33239,33242],{"className":33235},[119,156],[59,33237,335],{"className":33238},[119,192,156],[59,33240,815],{"className":33241},[504,156],[59,33243,83],{"className":33244},[119,156],[59,33246,33247,33250],{"style":15441},[59,33248],{"className":33249,"style":15432},[148],[59,33251,33252],{},[59,33253,1826],{"className":33254},[951,1910,15450],[59,33256,33257,33260],{"style":16374},[59,33258],{"className":33259,"style":15432},[148],[59,33261,33263],{"className":33262},[153,154,155,156],[59,33264,33266],{"className":33265},[119,156],[59,33267,7322],{"className":33268,"style":7397},[119,192,156],[59,33270,227],{"className":33271},[226],[59,33273,33275],{"className":33274},[136],[59,33276,33278],{"className":33277,"style":16393},[140],[59,33279],{},[59,33281],{"className":33282,"style":916},[499],[59,33284,11657],{"className":33285,"style":13619},[119,192],[59,33287,798],{"className":33288},[123],[59,33290,33292,33295],{"className":33291},[119],[59,33293,7327],{"className":33294},[119,192],[59,33296,33298],{"className":33297},[196],[59,33299,33301,33321],{"className":33300},[131,132],[59,33302,33304,33318],{"className":33303},[136],[59,33305,33307],{"className":33306,"style":1322},[140],[59,33308,33309,33312],{"style":1251},[59,33310],{"className":33311,"style":540},[148],[59,33313,33315],{"className":33314},[153,154,155,156],[59,33316,335],{"className":33317},[119,192,156],[59,33319,227],{"className":33320},[226],[59,33322,33324],{"className":33323},[136],[59,33325,33327],{"className":33326,"style":1270},[140],[59,33328],{},[59,33330,812],{"className":33331},[313],[59,33333,33335,33338,33483],{"className":33334},[119],[59,33336],{"className":33337},[123,124],[59,33339,33341],{"className":33340},[78],[59,33342,33344,33475],{"className":33343},[131,132],[59,33345,33347,33472],{"className":33346},[136],[59,33348,33350,33407,33415],{"className":33349,"style":15816},[140],[59,33351,33352,33355],{"style":15819},[59,33353],{"className":33354,"style":149},[148],[59,33356,33358,33361,33364,33404],{"className":33357},[119],[59,33359,1694],{"className":33360,"style":441},[119,192],[59,33362,798],{"className":33363},[123],[59,33365,33367,33370],{"className":33366},[119],[59,33368,7327],{"className":33369},[119,192],[59,33371,33373],{"className":33372},[196],[59,33374,33376,33396],{"className":33375},[131,132],[59,33377,33379,33393],{"className":33378},[136],[59,33380,33382],{"className":33381,"style":1322},[140],[59,33383,33384,33387],{"style":1251},[59,33385],{"className":33386,"style":540},[148],[59,33388,33390],{"className":33389},[153,154,155,156],[59,33391,335],{"className":33392},[119,192,156],[59,33394,227],{"className":33395},[226],[59,33397,33399],{"className":33398},[136],[59,33400,33402],{"className":33401,"style":1270},[140],[59,33403],{},[59,33405,812],{"className":33406},[313],[59,33408,33409,33412],{"style":274},[59,33410],{"className":33411,"style":149},[148],[59,33413],{"className":33414,"style":282},[281],[59,33416,33417,33420],{"style":15885},[59,33418],{"className":33419,"style":149},[148],[59,33421,33423,33426,33429,33469],{"className":33422},[119],[59,33424,15],{"className":33425},[119,192],[59,33427,798],{"className":33428},[123],[59,33430,33432,33435],{"className":33431},[119],[59,33433,7327],{"className":33434},[119,192],[59,33436,33438],{"className":33437},[196],[59,33439,33441,33461],{"className":33440},[131,132],[59,33442,33444,33458],{"className":33443},[136],[59,33445,33447],{"className":33446,"style":1322},[140],[59,33448,33449,33452],{"style":1251},[59,33450],{"className":33451,"style":540},[148],[59,33453,33455],{"className":33454},[153,154,155,156],[59,33456,335],{"className":33457},[119,192,156],[59,33459,227],{"className":33460},[226],[59,33462,33464],{"className":33463},[136],[59,33465,33467],{"className":33466,"style":1270},[140],[59,33468],{},[59,33470,812],{"className":33471},[313],[59,33473,227],{"className":33474},[226],[59,33476,33478],{"className":33477},[136],[59,33479,33481],{"className":33480,"style":15950},[140],[59,33482],{},[59,33484],{"className":33485},[313,124],[59,33487,803],{"className":33488},[912],[59,33490],{"className":33491,"style":28565},[499],[59,33493],{"className":33494,"style":916},[499],[59,33496,33498,33501],{"className":33497},[119],[59,33499,7327],{"className":33500},[119,192],[59,33502,33504],{"className":33503},[196],[59,33505,33507,33527],{"className":33506},[131,132],[59,33508,33510,33524],{"className":33509},[136],[59,33511,33513],{"className":33512,"style":1322},[140],[59,33514,33515,33518],{"style":1251},[59,33516],{"className":33517,"style":540},[148],[59,33519,33521],{"className":33520},[153,154,155,156],[59,33522,335],{"className":33523},[119,192,156],[59,33525,227],{"className":33526},[226],[59,33528,33530],{"className":33529},[136],[59,33531,33533],{"className":33532,"style":1270},[140],[59,33534],{},[59,33536],{"className":33537,"style":500},[499],[59,33539,14044],{"className":33540},[504],[59,33542],{"className":33543,"style":500},[499],[59,33545,1694],{"className":33546,"style":441},[119,192],[59,33548,798],{"className":33549},[123],[59,33551,7327],{"className":33552},[119,192],[59,33554,812],{"className":33555},[313],[59,33557,227],{"className":33558},[226],[59,33560,33562],{"className":33561},[136],[59,33563,33565],{"className":33564,"style":32453},[140],[59,33566],{},[15,33568,33569],{},"这样我们可以用重要性采样更新我们的目标函数的梯度公式，这样我们可以把 on-policy 的训练改为 off-policy 的训练。",[49,33571,33573],{"id":33572},"off-policy","Off-policy",[15,33575,33576,33577,33640,33641,33670,33671,33808,33809,34000,34001,34216],{},"记 ",[59,33578,33580,33598],{"className":33579},[62],[59,33581,33583],{"className":33582},[66],[68,33584,33585],{"xmlns":70},[72,33586,33587,33595],{},[75,33588,33589],{},[465,33590,33591,33593],{},[91,33592,14051],{},[461,33594,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[98,33596,33597],{"encoding":100},"\\theta'",[59,33599,33601],{"className":33600,"ariaHidden":106},[105],[59,33602,33604,33608],{"className":33603},[110],[59,33605],{"className":33606,"style":33607},[114],"height:0.7519em;",[59,33609,33611,33614],{"className":33610},[119],[59,33612,14051],{"className":33613,"style":3210},[119,192],[59,33615,33617],{"className":33616},[196],[59,33618,33620],{"className":33619},[131],[59,33621,33623],{"className":33622},[136],[59,33624,33626],{"className":33625,"style":33607},[140],[59,33627,33628,33631],{"style":536},[59,33629],{"className":33630,"style":540},[148],[59,33632,33634],{"className":33633},[153,154,155,156],[59,33635,33637],{"className":33636},[119,156],[59,33638,18984],{"className":33639},[119,156]," 为采集数据时的旧策略，",[59,33642,33644,33658],{"className":33643},[62],[59,33645,33647],{"className":33646},[66],[68,33648,33649],{"xmlns":70},[72,33650,33651,33655],{},[75,33652,33653],{},[91,33654,14051],{},[98,33656,33657],{"encoding":100},"\\theta",[59,33659,33661],{"className":33660,"ariaHidden":106},[105],[59,33662,33664,33667],{"className":33663},[110],[59,33665],{"className":33666,"style":407},[114],[59,33668,14051],{"className":33669,"style":3210},[119,192]," 为当前要优化的策略；优势 ",[59,33672,33674,33704],{"className":33673},[62],[59,33675,33677],{"className":33676},[66],[68,33678,33679],{"xmlns":70},[72,33680,33681,33701],{},[75,33682,33683],{},[1822,33684,33685,33687,33693],{},[91,33686,774],{},[465,33688,33689,33691],{},[91,33690,14051],{},[461,33692,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,33694,33695,33697,33699],{},[91,33696,27510],{},[91,33698,774],{},[91,33700,5607],{},[98,33702,33703],{"encoding":100},"A_{\\theta'}^{GAE}",[59,33705,33707],{"className":33706,"ariaHidden":106},[105],[59,33708,33710,33714],{"className":33709},[110],[59,33711],{"className":33712,"style":33713},[114],"height:1.1244em;vertical-align:-0.2831em;",[59,33715,33717,33720],{"className":33716},[119],[59,33718,774],{"className":33719},[119,192],[59,33721,33723],{"className":33722},[196],[59,33724,33726,33799],{"className":33725},[131,132],[59,33727,33729,33796],{"className":33728},[136],[59,33730,33732,33776],{"className":33731,"style":3198},[140],[59,33733,33735,33738],{"style":33734},"top:-2.4169em;margin-left:0em;margin-right:0.05em;",[59,33736],{"className":33737,"style":540},[148],[59,33739,33741],{"className":33740},[153,154,155,156],[59,33742,33744],{"className":33743},[119,156],[59,33745,33747,33750],{"className":33746},[119,156],[59,33748,14051],{"className":33749,"style":3210},[119,192,156],[59,33751,33753],{"className":33752},[196],[59,33754,33756],{"className":33755},[131],[59,33757,33759],{"className":33758},[136],[59,33760,33762],{"className":33761,"style":19144},[140],[59,33763,33764,33767],{"style":19147},[59,33765],{"className":33766,"style":213},[148],[59,33768,33770],{"className":33769},[153,217,218,156],[59,33771,33773],{"className":33772},[119,156],[59,33774,18984],{"className":33775},[119,156],[59,33777,33778,33781],{"style":536},[59,33779],{"className":33780,"style":540},[148],[59,33782,33784],{"className":33783},[153,154,155,156],[59,33785,33787,33790,33793],{"className":33786},[119,156],[59,33788,27510],{"className":33789},[119,192,156],[59,33791,774],{"className":33792},[119,192,156],[59,33794,5607],{"className":33795,"style":6050},[119,192,156],[59,33797,227],{"className":33798},[226],[59,33800,33802],{"className":33801},[136],[59,33803,33806],{"className":33804,"style":33805},[140],"height:0.2831em;",[59,33807],{}," 由旧策略下的价值网络估计。利用恒等式 ",[59,33810,33812,33866],{"className":33811},[62],[59,33813,33815],{"className":33814},[66],[68,33816,33817],{"xmlns":70},[72,33818,33819,33863],{},[75,33820,33821,33823,33825,33827,33829,33831,33833,33835,33837],{},[91,33822,14380],{"mathvariant":818},[91,33824,14805],{},[461,33826,822],{},[91,33828,11657],{},[461,33830,798],{"stretchy":797},[91,33832,7327],{},[461,33834,812],{"stretchy":797},[461,33836,815],{},[14374,33838,33839],{"scriptlevel":1754,"displaystyle":106},[78,33840,33841,33853],{},[75,33842,33843,33845,33847,33849,33851],{},[91,33844,14380],{"mathvariant":818},[91,33846,11657],{},[461,33848,798],{"stretchy":797},[91,33850,7327],{},[461,33852,812],{"stretchy":797},[75,33854,33855,33857,33859,33861],{},[91,33856,11657],{},[461,33858,798],{"stretchy":797},[91,33860,7327],{},[461,33862,812],{"stretchy":797},[98,33864,33865],{"encoding":100},"\\nabla \\log f(x) = \\dfrac{\\nabla f(x)}{f(x)}",[59,33867,33869,33910],{"className":33868,"ariaHidden":106},[105],[59,33870,33872,33875,33878,33881,33886,33889,33892,33895,33898,33901,33904,33907],{"className":33871},[110],[59,33873],{"className":33874,"style":877},[114],[59,33876,14380],{"className":33877},[119],[59,33879],{"className":33880,"style":916},[499],[59,33882,16850,33884],{"className":33883},[951],[59,33885,16854],{"style":16853},[59,33887],{"className":33888,"style":916},[499],[59,33890,11657],{"className":33891,"style":13619},[119,192],[59,33893,798],{"className":33894},[123],[59,33896,7327],{"className":33897},[119,192],[59,33899,812],{"className":33900},[313],[59,33902],{"className":33903,"style":500},[499],[59,33905,815],{"className":33906},[504],[59,33908],{"className":33909,"style":500},[499],[59,33911,33913,33917],{"className":33912},[110],[59,33914],{"className":33915,"style":33916},[114],"height:2.363em;vertical-align:-0.936em;",[59,33918,33920,33923,33997],{"className":33919},[119],[59,33921],{"className":33922},[123,124],[59,33924,33926],{"className":33925},[78],[59,33927,33929,33989],{"className":33928},[131,132],[59,33930,33932,33986],{"className":33931},[136],[59,33933,33935,33955,33963],{"className":33934,"style":15816},[140],[59,33936,33937,33940],{"style":15819},[59,33938],{"className":33939,"style":149},[148],[59,33941,33943,33946,33949,33952],{"className":33942},[119],[59,33944,11657],{"className":33945,"style":13619},[119,192],[59,33947,798],{"className":33948},[123],[59,33950,7327],{"className":33951},[119,192],[59,33953,812],{"className":33954},[313],[59,33956,33957,33960],{"style":274},[59,33958],{"className":33959,"style":149},[148],[59,33961],{"className":33962,"style":282},[281],[59,33964,33965,33968],{"style":15885},[59,33966],{"className":33967,"style":149},[148],[59,33969,33971,33974,33977,33980,33983],{"className":33970},[119],[59,33972,14380],{"className":33973},[119],[59,33975,11657],{"className":33976,"style":13619},[119,192],[59,33978,798],{"className":33979},[123],[59,33981,7327],{"className":33982},[119,192],[59,33984,812],{"className":33985},[313],[59,33987,227],{"className":33988},[226],[59,33990,33992],{"className":33991},[136],[59,33993,33995],{"className":33994,"style":15950},[140],[59,33996],{},[59,33998],{"className":33999},[313,124],"，可将策略梯度写成重要性采样比 ",[59,34002,34004,34036],{"className":34003},[62],[59,34005,34007],{"className":34006},[66],[68,34008,34009],{"xmlns":70},[72,34010,34011,34033],{},[75,34012,34013],{},[14374,34014,34015],{"scriptlevel":1754,"displaystyle":106},[78,34016,34017,34023],{},[88,34018,34019,34021],{},[91,34020,5604],{},[91,34022,14051],{},[88,34024,34025,34027],{},[91,34026,5604],{},[465,34028,34029,34031],{},[91,34030,14051],{},[461,34032,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[98,34034,34035],{"encoding":100},"\\dfrac{P_\\theta}{P_{\\theta'}}",[59,34037,34039],{"className":34038,"ariaHidden":106},[105],[59,34040,34042,34046],{"className":34041},[110],[59,34043],{"className":34044,"style":34045},[114],"height:2.1963em;vertical-align:-0.836em;",[59,34047,34049,34052,34213],{"className":34048},[119],[59,34050],{"className":34051},[123,124],[59,34053,34055],{"className":34054},[78],[59,34056,34058,34204],{"className":34057},[131,132],[59,34059,34061,34201],{"className":34060},[136],[59,34062,34065,34145,34153],{"className":34063,"style":34064},[140],"height:1.3603em;",[59,34066,34067,34070],{"style":15819},[59,34068],{"className":34069,"style":149},[148],[59,34071,34073],{"className":34072},[119],[59,34074,34076,34079],{"className":34075},[119],[59,34077,5604],{"className":34078,"style":3185},[119,192],[59,34080,34082],{"className":34081},[196],[59,34083,34085,34137],{"className":34084},[131,132],[59,34086,34088,34134],{"className":34087},[136],[59,34089,34091],{"className":34090,"style":1248},[140],[59,34092,34093,34096],{"style":7460},[59,34094],{"className":34095,"style":540},[148],[59,34097,34099],{"className":34098},[153,154,155,156],[59,34100,34102],{"className":34101},[119,156],[59,34103,34105,34108],{"className":34104},[119,156],[59,34106,14051],{"className":34107,"style":3210},[119,192,156],[59,34109,34111],{"className":34110},[196],[59,34112,34114],{"className":34113},[131],[59,34115,34117],{"className":34116},[136],[59,34118,34120],{"className":34119,"style":19144},[140],[59,34121,34122,34125],{"style":19147},[59,34123],{"className":34124,"style":213},[148],[59,34126,34128],{"className":34127},[153,217,218,156],[59,34129,34131],{"className":34130},[119,156],[59,34132,18984],{"className":34133},[119,156],[59,34135,227],{"className":34136},[226],[59,34138,34140],{"className":34139},[136],[59,34141,34143],{"className":34142,"style":1270},[140],[59,34144],{},[59,34146,34147,34150],{"style":274},[59,34148],{"className":34149,"style":149},[148],[59,34151],{"className":34152,"style":282},[281],[59,34154,34155,34158],{"style":15885},[59,34156],{"className":34157,"style":149},[148],[59,34159,34161],{"className":34160},[119],[59,34162,34164,34167],{"className":34163},[119],[59,34165,5604],{"className":34166,"style":3185},[119,192],[59,34168,34170],{"className":34169},[196],[59,34171,34173,34193],{"className":34172},[131,132],[59,34174,34176,34190],{"className":34175},[136],[59,34177,34179],{"className":34178,"style":1248},[140],[59,34180,34181,34184],{"style":7460},[59,34182],{"className":34183,"style":540},[148],[59,34185,34187],{"className":34186},[153,154,155,156],[59,34188,14051],{"className":34189,"style":3210},[119,192,156],[59,34191,227],{"className":34192},[226],[59,34194,34196],{"className":34195},[136],[59,34197,34199],{"className":34198,"style":1270},[140],[59,34200],{},[59,34202,227],{"className":34203},[226],[59,34205,34207],{"className":34206},[136],[59,34208,34211],{"className":34209,"style":34210},[140],"height:0.836em;",[59,34212],{},[59,34214],{"className":34215},[313,124]," 的形式：",[59,34218,34220],{"className":34219},[14350],[59,34221,34223,34275],{"className":34222},[62],[59,34224,34226],{"className":34225},[66],[68,34227,34228],{"xmlns":70,"display":9481},[72,34229,34230,34272],{},[75,34231,34232,34234,34236,34238,34240,34242,34244,34246,34248],{},[91,34233,14380],{"mathvariant":818},[91,34235,14805],{},[461,34237,822],{},[91,34239,11657],{},[461,34241,798],{"stretchy":797},[91,34243,7327],{},[461,34245,812],{"stretchy":797},[461,34247,815],{},[78,34249,34250,34262],{},[75,34251,34252,34254,34256,34258,34260],{},[91,34253,14380],{"mathvariant":818},[91,34255,11657],{},[461,34257,798],{"stretchy":797},[91,34259,7327],{},[461,34261,812],{"stretchy":797},[75,34263,34264,34266,34268,34270],{},[91,34265,11657],{},[461,34267,798],{"stretchy":797},[91,34269,7327],{},[461,34271,812],{"stretchy":797},[98,34273,34274],{"encoding":100},"\\nabla \\log f(x) = \\frac{\\nabla f(x)}{f(x)}",[59,34276,34278,34319],{"className":34277,"ariaHidden":106},[105],[59,34279,34281,34284,34287,34290,34295,34298,34301,34304,34307,34310,34313,34316],{"className":34280},[110],[59,34282],{"className":34283,"style":877},[114],[59,34285,14380],{"className":34286},[119],[59,34288],{"className":34289,"style":916},[499],[59,34291,16850,34293],{"className":34292},[951],[59,34294,16854],{"style":16853},[59,34296],{"className":34297,"style":916},[499],[59,34299,11657],{"className":34300,"style":13619},[119,192],[59,34302,798],{"className":34303},[123],[59,34305,7327],{"className":34306},[119,192],[59,34308,812],{"className":34309},[313],[59,34311],{"className":34312,"style":500},[499],[59,34314,815],{"className":34315},[504],[59,34317],{"className":34318,"style":500},[499],[59,34320,34322,34325],{"className":34321},[110],[59,34323],{"className":34324,"style":33916},[114],[59,34326,34328,34331,34405],{"className":34327},[119],[59,34329],{"className":34330},[123,124],[59,34332,34334],{"className":34333},[78],[59,34335,34337,34397],{"className":34336},[131,132],[59,34338,34340,34394],{"className":34339},[136],[59,34341,34343,34363,34371],{"className":34342,"style":15816},[140],[59,34344,34345,34348],{"style":15819},[59,34346],{"className":34347,"style":149},[148],[59,34349,34351,34354,34357,34360],{"className":34350},[119],[59,34352,11657],{"className":34353,"style":13619},[119,192],[59,34355,798],{"className":34356},[123],[59,34358,7327],{"className":34359},[119,192],[59,34361,812],{"className":34362},[313],[59,34364,34365,34368],{"style":274},[59,34366],{"className":34367,"style":149},[148],[59,34369],{"className":34370,"style":282},[281],[59,34372,34373,34376],{"style":15885},[59,34374],{"className":34375,"style":149},[148],[59,34377,34379,34382,34385,34388,34391],{"className":34378},[119],[59,34380,14380],{"className":34381},[119],[59,34383,11657],{"className":34384,"style":13619},[119,192],[59,34386,798],{"className":34387},[123],[59,34389,7327],{"className":34390},[119,192],[59,34392,812],{"className":34393},[313],[59,34395,227],{"className":34396},[226],[59,34398,34400],{"className":34399},[136],[59,34401,34403],{"className":34402,"style":15950},[140],[59,34404],{},[59,34406],{"className":34407},[313,124],[59,34409,34411],{"className":34410},[14350],[59,34412,34414,35136],{"className":34413},[62],[59,34415,34417],{"className":34416},[66],[68,34418,34419],{"xmlns":70,"display":9481},[72,34420,34421,35133],{},[14362,34422,34423,34547,34743,34969],{"rowspacing":14364,"columnalign":14365,"columnspacing":14366},[14368,34424,34425,34431],{},[14371,34426,34427],{},[14374,34428,34429],{"scriptlevel":1754,"displaystyle":106},[75,34430],{},[14371,34432,34433],{},[14374,34434,34435],{"scriptlevel":1754,"displaystyle":106},[75,34436,34437,34439,34445,34459,34477,34491,34493,34501,34503,34511,34513,34515,34517,34519,34525,34527,34535,34537,34545],{},[75,34438],{},[78,34440,34441,34443],{},[81,34442,83],{},[91,34444,7322],{},[14683,34446,34447,34449,34457],{},[461,34448,1826],{},[75,34450,34451,34453,34455],{},[91,34452,335],{},[461,34454,815],{},[81,34456,83],{},[91,34458,7322],{},[14683,34460,34461,34463,34471],{},[461,34462,1826],{},[75,34464,34465,34467,34469],{},[91,34466,777],{},[461,34468,815],{},[81,34470,83],{},[88,34472,34473,34475],{},[91,34474,13242],{},[91,34476,335],{},[1822,34478,34479,34481,34483],{},[91,34480,774],{},[91,34482,14051],{},[75,34484,34485,34487,34489],{},[91,34486,27510],{},[91,34488,774],{},[91,34490,5607],{},[461,34492,798],{"stretchy":797},[1822,34494,34495,34497,34499],{},[91,34496,12802],{},[91,34498,335],{},[91,34500,777],{},[461,34502,803],{"separator":106},[1822,34504,34505,34507,34509],{},[91,34506,22],{},[91,34508,335],{},[91,34510,777],{},[461,34512,812],{"stretchy":797},[91,34514,14380],{"mathvariant":818},[91,34516,14805],{},[461,34518,822],{},[88,34520,34521,34523],{},[91,34522,5604],{},[91,34524,14051],{},[461,34526,798],{"stretchy":797},[1822,34528,34529,34531,34533],{},[91,34530,22],{},[91,34532,335],{},[91,34534,777],{},[91,34536,12941],{"mathvariant":818},[1822,34538,34539,34541,34543],{},[91,34540,12802],{},[91,34542,335],{},[91,34544,777],{},[461,34546,812],{"stretchy":797},[14368,34548,34549,34555],{},[14371,34550,34551],{},[14374,34552,34553],{"scriptlevel":1754,"displaystyle":106},[75,34554],{},[14371,34556,34557],{},[14374,34558,34559],{"scriptlevel":1754,"displaystyle":106},[75,34560,34561,34563,34565,34571,34585,34603,34621,34623,34631,34633,34641,34643,34709,34711,34713,34715,34721,34723,34731,34733,34741],{},[75,34562],{},[461,34564,815],{},[78,34566,34567,34569],{},[81,34568,83],{},[91,34570,7322],{},[14683,34572,34573,34575,34583],{},[461,34574,1826],{},[75,34576,34577,34579,34581],{},[91,34578,335],{},[461,34580,815],{},[81,34582,83],{},[91,34584,7322],{},[14683,34586,34587,34589,34597],{},[461,34588,1826],{},[75,34590,34591,34593,34595],{},[91,34592,777],{},[461,34594,815],{},[81,34596,83],{},[88,34598,34599,34601],{},[91,34600,13242],{},[91,34602,335],{},[1822,34604,34605,34607,34613],{},[91,34606,774],{},[465,34608,34609,34611],{},[91,34610,14051],{},[461,34612,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,34614,34615,34617,34619],{},[91,34616,27510],{},[91,34618,774],{},[91,34620,5607],{},[461,34622,798],{"stretchy":797},[1822,34624,34625,34627,34629],{},[91,34626,12802],{},[91,34628,335],{},[91,34630,777],{},[461,34632,803],{"separator":106},[1822,34634,34635,34637,34639],{},[91,34636,22],{},[91,34638,335],{},[91,34640,777],{},[461,34642,812],{"stretchy":797},[78,34644,34645,34675],{},[75,34646,34647,34653,34655,34663,34665,34673],{},[88,34648,34649,34651],{},[91,34650,5604],{},[91,34652,14051],{},[461,34654,798],{"stretchy":797},[1822,34656,34657,34659,34661],{},[91,34658,22],{},[91,34660,335],{},[91,34662,777],{},[91,34664,12941],{"mathvariant":818},[1822,34666,34667,34669,34671],{},[91,34668,12802],{},[91,34670,335],{},[91,34672,777],{},[461,34674,812],{"stretchy":797},[75,34676,34677,34687,34689,34697,34699,34707],{},[88,34678,34679,34681],{},[91,34680,5604],{},[465,34682,34683,34685],{},[91,34684,14051],{},[461,34686,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,34688,798],{"stretchy":797},[1822,34690,34691,34693,34695],{},[91,34692,22],{},[91,34694,335],{},[91,34696,777],{},[91,34698,12941],{"mathvariant":818},[1822,34700,34701,34703,34705],{},[91,34702,12802],{},[91,34704,335],{},[91,34706,777],{},[461,34708,812],{"stretchy":797},[91,34710,14380],{"mathvariant":818},[91,34712,14805],{},[461,34714,822],{},[88,34716,34717,34719],{},[91,34718,5604],{},[91,34720,14051],{},[461,34722,798],{"stretchy":797},[1822,34724,34725,34727,34729],{},[91,34726,22],{},[91,34728,335],{},[91,34730,777],{},[91,34732,12941],{"mathvariant":818},[1822,34734,34735,34737,34739],{},[91,34736,12802],{},[91,34738,335],{},[91,34740,777],{},[461,34742,812],{"stretchy":797},[14368,34744,34745,34751],{},[14371,34746,34747],{},[14374,34748,34749],{"scriptlevel":1754,"displaystyle":106},[75,34750],{},[14371,34752,34753],{},[14374,34754,34755],{"scriptlevel":1754,"displaystyle":106},[75,34756,34757,34759,34761,34767,34781,34799,34817,34819,34827,34829,34837,34839,34905],{},[75,34758],{},[461,34760,815],{},[78,34762,34763,34765],{},[81,34764,83],{},[91,34766,7322],{},[14683,34768,34769,34771,34779],{},[461,34770,1826],{},[75,34772,34773,34775,34777],{},[91,34774,335],{},[461,34776,815],{},[81,34778,83],{},[91,34780,7322],{},[14683,34782,34783,34785,34793],{},[461,34784,1826],{},[75,34786,34787,34789,34791],{},[91,34788,777],{},[461,34790,815],{},[81,34792,83],{},[88,34794,34795,34797],{},[91,34796,13242],{},[91,34798,335],{},[1822,34800,34801,34803,34809],{},[91,34802,774],{},[465,34804,34805,34807],{},[91,34806,14051],{},[461,34808,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,34810,34811,34813,34815],{},[91,34812,27510],{},[91,34814,774],{},[91,34816,5607],{},[461,34818,798],{"stretchy":797},[1822,34820,34821,34823,34825],{},[91,34822,12802],{},[91,34824,335],{},[91,34826,777],{},[461,34828,803],{"separator":106},[1822,34830,34831,34833,34835],{},[91,34832,22],{},[91,34834,335],{},[91,34836,777],{},[461,34838,812],{"stretchy":797},[78,34840,34841,34871],{},[75,34842,34843,34849,34851,34859,34861,34869],{},[88,34844,34845,34847],{},[91,34846,5604],{},[91,34848,14051],{},[461,34850,798],{"stretchy":797},[1822,34852,34853,34855,34857],{},[91,34854,22],{},[91,34856,335],{},[91,34858,777],{},[91,34860,12941],{"mathvariant":818},[1822,34862,34863,34865,34867],{},[91,34864,12802],{},[91,34866,335],{},[91,34868,777],{},[461,34870,812],{"stretchy":797},[75,34872,34873,34883,34885,34893,34895,34903],{},[88,34874,34875,34877],{},[91,34876,5604],{},[465,34878,34879,34881],{},[91,34880,14051],{},[461,34882,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,34884,798],{"stretchy":797},[1822,34886,34887,34889,34891],{},[91,34888,22],{},[91,34890,335],{},[91,34892,777],{},[91,34894,12941],{"mathvariant":818},[1822,34896,34897,34899,34901],{},[91,34898,12802],{},[91,34900,335],{},[91,34902,777],{},[461,34904,812],{"stretchy":797},[78,34906,34907,34939],{},[75,34908,34909,34911,34917,34919,34927,34929,34937],{},[91,34910,14380],{"mathvariant":818},[88,34912,34913,34915],{},[91,34914,5604],{},[91,34916,14051],{},[461,34918,798],{"stretchy":797},[1822,34920,34921,34923,34925],{},[91,34922,22],{},[91,34924,335],{},[91,34926,777],{},[91,34928,12941],{"mathvariant":818},[1822,34930,34931,34933,34935],{},[91,34932,12802],{},[91,34934,335],{},[91,34936,777],{},[461,34938,812],{"stretchy":797},[75,34940,34941,34947,34949,34957,34959,34967],{},[88,34942,34943,34945],{},[91,34944,5604],{},[91,34946,14051],{},[461,34948,798],{"stretchy":797},[1822,34950,34951,34953,34955],{},[91,34952,22],{},[91,34954,335],{},[91,34956,777],{},[91,34958,12941],{"mathvariant":818},[1822,34960,34961,34963,34965],{},[91,34962,12802],{},[91,34964,335],{},[91,34966,777],{},[461,34968,812],{"stretchy":797},[14368,34970,34971,34977],{},[14371,34972,34973],{},[14374,34974,34975],{"scriptlevel":1754,"displaystyle":106},[75,34976],{},[14371,34978,34979],{},[14374,34980,34981],{"scriptlevel":1754,"displaystyle":106},[75,34982,34983,34985,34987,34993,35007,35025,35043,35045,35053,35055,35063,35065],{},[75,34984],{},[461,34986,815],{},[78,34988,34989,34991],{},[81,34990,83],{},[91,34992,7322],{},[14683,34994,34995,34997,35005],{},[461,34996,1826],{},[75,34998,34999,35001,35003],{},[91,35000,335],{},[461,35002,815],{},[81,35004,83],{},[91,35006,7322],{},[14683,35008,35009,35011,35019],{},[461,35010,1826],{},[75,35012,35013,35015,35017],{},[91,35014,777],{},[461,35016,815],{},[81,35018,83],{},[88,35020,35021,35023],{},[91,35022,13242],{},[91,35024,335],{},[1822,35026,35027,35029,35035],{},[91,35028,774],{},[465,35030,35031,35033],{},[91,35032,14051],{},[461,35034,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,35036,35037,35039,35041],{},[91,35038,27510],{},[91,35040,774],{},[91,35042,5607],{},[461,35044,798],{"stretchy":797},[1822,35046,35047,35049,35051],{},[91,35048,12802],{},[91,35050,335],{},[91,35052,777],{},[461,35054,803],{"separator":106},[1822,35056,35057,35059,35061],{},[91,35058,22],{},[91,35060,335],{},[91,35062,777],{},[461,35064,812],{"stretchy":797},[78,35066,35067,35099],{},[75,35068,35069,35071,35077,35079,35087,35089,35097],{},[91,35070,14380],{"mathvariant":818},[88,35072,35073,35075],{},[91,35074,5604],{},[91,35076,14051],{},[461,35078,798],{"stretchy":797},[1822,35080,35081,35083,35085],{},[91,35082,22],{},[91,35084,335],{},[91,35086,777],{},[91,35088,12941],{"mathvariant":818},[1822,35090,35091,35093,35095],{},[91,35092,12802],{},[91,35094,335],{},[91,35096,777],{},[461,35098,812],{"stretchy":797},[75,35100,35101,35111,35113,35121,35123,35131],{},[88,35102,35103,35105],{},[91,35104,5604],{},[465,35106,35107,35109],{},[91,35108,14051],{},[461,35110,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,35112,798],{"stretchy":797},[1822,35114,35115,35117,35119],{},[91,35116,22],{},[91,35118,335],{},[91,35120,777],{},[91,35122,12941],{"mathvariant":818},[1822,35124,35125,35127,35129],{},[91,35126,12802],{},[91,35128,335],{},[91,35130,777],{},[461,35132,812],{"stretchy":797},[98,35134,35135],{"encoding":100},"\\begin{aligned}\n&\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_\\theta^{GAE}(s_n^t, a_n^t) \\nabla \\log P_\\theta(a_n^t | s_n^t) \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)} \\nabla \\log P_\\theta(a_n^t | s_n^t) \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)} \\frac{\\nabla P_\\theta(a_n^t | s_n^t)}{P_\\theta(a_n^t | s_n^t)} \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{\\nabla P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)}\n\\end{aligned}",[59,35137,35139],{"className":35138,"ariaHidden":106},[105],[59,35140,35142,35146],{"className":35141},[110],[59,35143],{"className":35144,"style":35145},[114],"height:13.6262em;vertical-align:-6.5631em;",[59,35147,35149],{"className":35148},[119],[59,35150,35152,35213],{"className":35151},[14362],[59,35153,35155],{"className":35154},[15153],[59,35156,35158,35204],{"className":35157},[131,132],[59,35159,35161,35201],{"className":35160},[136],[59,35162,35165,35174,35183,35192],{"className":35163,"style":35164},[140],"height:7.0631em;",[59,35166,35168,35171],{"style":35167},"top:-9.0631em;",[59,35169],{"className":35170,"style":15170},[148],[59,35172],{"className":35173},[119],[59,35175,35177,35180],{"style":35176},"top:-5.6565em;",[59,35178],{"className":35179,"style":15170},[148],[59,35181],{"className":35182},[119],[59,35184,35186,35189],{"style":35185},"top:-2.25em;",[59,35187],{"className":35188,"style":15170},[148],[59,35190],{"className":35191},[119],[59,35193,35195,35198],{"style":35194},"top:1.1565em;",[59,35196],{"className":35197,"style":15170},[148],[59,35199],{"className":35200},[119],[59,35202,227],{"className":35203},[226],[59,35205,35207],{"className":35206},[136],[59,35208,35211],{"className":35209,"style":35210},[140],"height:6.5631em;",[59,35212],{},[59,35214,35216],{"className":35215},[15377],[59,35217,35219,38925],{"className":35218},[131,132],[59,35220,35222,38922],{"className":35221},[136],[59,35223,35225,35817,36842,38061],{"className":35224,"style":35164},[140],[59,35226,35227,35230],{"style":35167},[59,35228],{"className":35229,"style":15170},[148],[59,35231,35233,35236,35298,35301,35368,35371,35475,35478,35538,35541,35592,35595,35598,35649,35652,35655,35658,35663,35666,35706,35709,35760,35763,35814],{"className":35232},[119],[59,35234],{"className":35235},[119],[59,35237,35239,35242,35295],{"className":35238},[119],[59,35240],{"className":35241},[123,124],[59,35243,35245],{"className":35244},[78],[59,35246,35248,35287],{"className":35247},[131,132],[59,35249,35251,35284],{"className":35250},[136],[59,35252,35254,35265,35273],{"className":35253,"style":16279},[140],[59,35255,35256,35259],{"style":15819},[59,35257],{"className":35258,"style":149},[148],[59,35260,35262],{"className":35261},[119],[59,35263,7322],{"className":35264,"style":7397},[119,192],[59,35266,35267,35270],{"style":274},[59,35268],{"className":35269,"style":149},[148],[59,35271],{"className":35272,"style":282},[281],[59,35274,35275,35278],{"style":15885},[59,35276],{"className":35277,"style":149},[148],[59,35279,35281],{"className":35280},[119],[59,35282,83],{"className":35283},[119],[59,35285,227],{"className":35286},[226],[59,35288,35290],{"className":35289},[136],[59,35291,35293],{"className":35292,"style":16319},[140],[59,35294],{},[59,35296],{"className":35297},[313,124],[59,35299],{"className":35300,"style":916},[499],[59,35302,35304],{"className":35303},[951,14238],[59,35305,35307,35360],{"className":35306},[131,132],[59,35308,35310,35357],{"className":35309},[136],[59,35311,35313,35333,35343],{"className":35312,"style":16340},[140],[59,35314,35315,35318],{"style":16343},[59,35316],{"className":35317,"style":15432},[148],[59,35319,35321],{"className":35320},[153,154,155,156],[59,35322,35324,35327,35330],{"className":35323},[119,156],[59,35325,335],{"className":35326},[119,192,156],[59,35328,815],{"className":35329},[504,156],[59,35331,83],{"className":35332},[119,156],[59,35334,35335,35338],{"style":15441},[59,35336],{"className":35337,"style":15432},[148],[59,35339,35340],{},[59,35341,1826],{"className":35342},[951,1910,15450],[59,35344,35345,35348],{"style":16374},[59,35346],{"className":35347,"style":15432},[148],[59,35349,35351],{"className":35350},[153,154,155,156],[59,35352,35354],{"className":35353},[119,156],[59,35355,7322],{"className":35356,"style":7397},[119,192,156],[59,35358,227],{"className":35359},[226],[59,35361,35363],{"className":35362},[136],[59,35364,35366],{"className":35365,"style":16393},[140],[59,35367],{},[59,35369],{"className":35370,"style":916},[499],[59,35372,35374],{"className":35373},[951,14238],[59,35375,35377,35467],{"className":35376},[131,132],[59,35378,35380,35464],{"className":35379},[136],[59,35381,35383,35403,35413],{"className":35382,"style":17149},[140],[59,35384,35385,35388],{"style":16343},[59,35386],{"className":35387,"style":15432},[148],[59,35389,35391],{"className":35390},[153,154,155,156],[59,35392,35394,35397,35400],{"className":35393},[119,156],[59,35395,777],{"className":35396},[119,192,156],[59,35398,815],{"className":35399},[504,156],[59,35401,83],{"className":35402},[119,156],[59,35404,35405,35408],{"style":15441},[59,35406],{"className":35407,"style":15432},[148],[59,35409,35410],{},[59,35411,1826],{"className":35412},[951,1910,15450],[59,35414,35415,35418],{"style":17182},[59,35416],{"className":35417,"style":15432},[148],[59,35419,35421],{"className":35420},[153,154,155,156],[59,35422,35424],{"className":35423},[119,156],[59,35425,35427,35430],{"className":35426},[119,156],[59,35428,13242],{"className":35429,"style":3185},[119,192,156],[59,35431,35433],{"className":35432},[196],[59,35434,35436,35456],{"className":35435},[131,132],[59,35437,35439,35453],{"className":35438},[136],[59,35440,35442],{"className":35441,"style":4302},[140],[59,35443,35444,35447],{"style":17212},[59,35445],{"className":35446,"style":213},[148],[59,35448,35450],{"className":35449},[153,217,218,156],[59,35451,335],{"className":35452},[119,192,156],[59,35454,227],{"className":35455},[226],[59,35457,35459],{"className":35458},[136],[59,35460,35462],{"className":35461,"style":4324},[140],[59,35463],{},[59,35465,227],{"className":35466},[226],[59,35468,35470],{"className":35469},[136],[59,35471,35473],{"className":35472,"style":16393},[140],[59,35474],{},[59,35476],{"className":35477,"style":916},[499],[59,35479,35481,35484],{"className":35480},[119],[59,35482,774],{"className":35483},[119,192],[59,35485,35487],{"className":35486},[196],[59,35488,35490,35530],{"className":35489},[131,132],[59,35491,35493,35527],{"className":35492},[136],[59,35494,35496,35507],{"className":35495,"style":24796},[140],[59,35497,35498,35501],{"style":17311},[59,35499],{"className":35500,"style":540},[148],[59,35502,35504],{"className":35503},[153,154,155,156],[59,35505,14051],{"className":35506,"style":3210},[119,192,156],[59,35508,35509,35512],{"style":16426},[59,35510],{"className":35511,"style":540},[148],[59,35513,35515],{"className":35514},[153,154,155,156],[59,35516,35518,35521,35524],{"className":35517},[119,156],[59,35519,27510],{"className":35520},[119,192,156],[59,35522,774],{"className":35523},[119,192,156],[59,35525,5607],{"className":35526,"style":6050},[119,192,156],[59,35528,227],{"className":35529},[226],[59,35531,35533],{"className":35532},[136],[59,35534,35536],{"className":35535,"style":17341},[140],[59,35537],{},[59,35539,798],{"className":35540},[123],[59,35542,35544,35547],{"className":35543},[119],[59,35545,12802],{"className":35546},[119,192],[59,35548,35550],{"className":35549},[196],[59,35551,35553,35584],{"className":35552},[131,132],[59,35554,35556,35581],{"className":35555},[136],[59,35557,35559,35570],{"className":35558,"style":17308},[140],[59,35560,35561,35564],{"style":17311},[59,35562],{"className":35563,"style":540},[148],[59,35565,35567],{"className":35566},[153,154,155,156],[59,35568,335],{"className":35569},[119,192,156],[59,35571,35572,35575],{"style":16426},[59,35573],{"className":35574,"style":540},[148],[59,35576,35578],{"className":35577},[153,154,155,156],[59,35579,777],{"className":35580},[119,192,156],[59,35582,227],{"className":35583},[226],[59,35585,35587],{"className":35586},[136],[59,35588,35590],{"className":35589,"style":17341},[140],[59,35591],{},[59,35593,803],{"className":35594},[912],[59,35596],{"className":35597,"style":916},[499],[59,35599,35601,35604],{"className":35600},[119],[59,35602,22],{"className":35603},[119,192],[59,35605,35607],{"className":35606},[196],[59,35608,35610,35641],{"className":35609},[131,132],[59,35611,35613,35638],{"className":35612},[136],[59,35614,35616,35627],{"className":35615,"style":17308},[140],[59,35617,35618,35621],{"style":17311},[59,35619],{"className":35620,"style":540},[148],[59,35622,35624],{"className":35623},[153,154,155,156],[59,35625,335],{"className":35626},[119,192,156],[59,35628,35629,35632],{"style":16426},[59,35630],{"className":35631,"style":540},[148],[59,35633,35635],{"className":35634},[153,154,155,156],[59,35636,777],{"className":35637},[119,192,156],[59,35639,227],{"className":35640},[226],[59,35642,35644],{"className":35643},[136],[59,35645,35647],{"className":35646,"style":17341},[140],[59,35648],{},[59,35650,812],{"className":35651},[313],[59,35653,14380],{"className":35654},[119],[59,35656],{"className":35657,"style":916},[499],[59,35659,16850,35661],{"className":35660},[951],[59,35662,16854],{"style":16853},[59,35664],{"className":35665,"style":916},[499],[59,35667,35669,35672],{"className":35668},[119],[59,35670,5604],{"className":35671,"style":3185},[119,192],[59,35673,35675],{"className":35674},[196],[59,35676,35678,35698],{"className":35677},[131,132],[59,35679,35681,35695],{"className":35680},[136],[59,35682,35684],{"className":35683,"style":1248},[140],[59,35685,35686,35689],{"style":7460},[59,35687],{"className":35688,"style":540},[148],[59,35690,35692],{"className":35691},[153,154,155,156],[59,35693,14051],{"className":35694,"style":3210},[119,192,156],[59,35696,227],{"className":35697},[226],[59,35699,35701],{"className":35700},[136],[59,35702,35704],{"className":35703,"style":1270},[140],[59,35705],{},[59,35707,798],{"className":35708},[123],[59,35710,35712,35715],{"className":35711},[119],[59,35713,22],{"className":35714},[119,192],[59,35716,35718],{"className":35717},[196],[59,35719,35721,35752],{"className":35720},[131,132],[59,35722,35724,35749],{"className":35723},[136],[59,35725,35727,35738],{"className":35726,"style":17308},[140],[59,35728,35729,35732],{"style":17311},[59,35730],{"className":35731,"style":540},[148],[59,35733,35735],{"className":35734},[153,154,155,156],[59,35736,335],{"className":35737},[119,192,156],[59,35739,35740,35743],{"style":16426},[59,35741],{"className":35742,"style":540},[148],[59,35744,35746],{"className":35745},[153,154,155,156],[59,35747,777],{"className":35748},[119,192,156],[59,35750,227],{"className":35751},[226],[59,35753,35755],{"className":35754},[136],[59,35756,35758],{"className":35757,"style":17341},[140],[59,35759],{},[59,35761,12941],{"className":35762},[119],[59,35764,35766,35769],{"className":35765},[119],[59,35767,12802],{"className":35768},[119,192],[59,35770,35772],{"className":35771},[196],[59,35773,35775,35806],{"className":35774},[131,132],[59,35776,35778,35803],{"className":35777},[136],[59,35779,35781,35792],{"className":35780,"style":17308},[140],[59,35782,35783,35786],{"style":17311},[59,35784],{"className":35785,"style":540},[148],[59,35787,35789],{"className":35788},[153,154,155,156],[59,35790,335],{"className":35791},[119,192,156],[59,35793,35794,35797],{"style":16426},[59,35795],{"className":35796,"style":540},[148],[59,35798,35800],{"className":35799},[153,154,155,156],[59,35801,777],{"className":35802},[119,192,156],[59,35804,227],{"className":35805},[226],[59,35807,35809],{"className":35808},[136],[59,35810,35812],{"className":35811,"style":17341},[140],[59,35813],{},[59,35815,812],{"className":35816},[313],[59,35818,35819,35822],{"style":35176},[59,35820],{"className":35821,"style":15170},[148],[59,35823,35825,35828,35831,35834,35837,35899,35902,35969,35972,36076,36079,36171,36174,36225,36228,36231,36282,36285,36677,36680,36683,36688,36691,36731,36734,36785,36788,36839],{"className":35824},[119],[59,35826],{"className":35827},[119],[59,35829],{"className":35830,"style":500},[499],[59,35832,815],{"className":35833},[504],[59,35835],{"className":35836,"style":500},[499],[59,35838,35840,35843,35896],{"className":35839},[119],[59,35841],{"className":35842},[123,124],[59,35844,35846],{"className":35845},[78],[59,35847,35849,35888],{"className":35848},[131,132],[59,35850,35852,35885],{"className":35851},[136],[59,35853,35855,35866,35874],{"className":35854,"style":16279},[140],[59,35856,35857,35860],{"style":15819},[59,35858],{"className":35859,"style":149},[148],[59,35861,35863],{"className":35862},[119],[59,35864,7322],{"className":35865,"style":7397},[119,192],[59,35867,35868,35871],{"style":274},[59,35869],{"className":35870,"style":149},[148],[59,35872],{"className":35873,"style":282},[281],[59,35875,35876,35879],{"style":15885},[59,35877],{"className":35878,"style":149},[148],[59,35880,35882],{"className":35881},[119],[59,35883,83],{"className":35884},[119],[59,35886,227],{"className":35887},[226],[59,35889,35891],{"className":35890},[136],[59,35892,35894],{"className":35893,"style":16319},[140],[59,35895],{},[59,35897],{"className":35898},[313,124],[59,35900],{"className":35901,"style":916},[499],[59,35903,35905],{"className":35904},[951,14238],[59,35906,35908,35961],{"className":35907},[131,132],[59,35909,35911,35958],{"className":35910},[136],[59,35912,35914,35934,35944],{"className":35913,"style":16340},[140],[59,35915,35916,35919],{"style":16343},[59,35917],{"className":35918,"style":15432},[148],[59,35920,35922],{"className":35921},[153,154,155,156],[59,35923,35925,35928,35931],{"className":35924},[119,156],[59,35926,335],{"className":35927},[119,192,156],[59,35929,815],{"className":35930},[504,156],[59,35932,83],{"className":35933},[119,156],[59,35935,35936,35939],{"style":15441},[59,35937],{"className":35938,"style":15432},[148],[59,35940,35941],{},[59,35942,1826],{"className":35943},[951,1910,15450],[59,35945,35946,35949],{"style":16374},[59,35947],{"className":35948,"style":15432},[148],[59,35950,35952],{"className":35951},[153,154,155,156],[59,35953,35955],{"className":35954},[119,156],[59,35956,7322],{"className":35957,"style":7397},[119,192,156],[59,35959,227],{"className":35960},[226],[59,35962,35964],{"className":35963},[136],[59,35965,35967],{"className":35966,"style":16393},[140],[59,35968],{},[59,35970],{"className":35971,"style":916},[499],[59,35973,35975],{"className":35974},[951,14238],[59,35976,35978,36068],{"className":35977},[131,132],[59,35979,35981,36065],{"className":35980},[136],[59,35982,35984,36004,36014],{"className":35983,"style":17149},[140],[59,35985,35986,35989],{"style":16343},[59,35987],{"className":35988,"style":15432},[148],[59,35990,35992],{"className":35991},[153,154,155,156],[59,35993,35995,35998,36001],{"className":35994},[119,156],[59,35996,777],{"className":35997},[119,192,156],[59,35999,815],{"className":36000},[504,156],[59,36002,83],{"className":36003},[119,156],[59,36005,36006,36009],{"style":15441},[59,36007],{"className":36008,"style":15432},[148],[59,36010,36011],{},[59,36012,1826],{"className":36013},[951,1910,15450],[59,36015,36016,36019],{"style":17182},[59,36017],{"className":36018,"style":15432},[148],[59,36020,36022],{"className":36021},[153,154,155,156],[59,36023,36025],{"className":36024},[119,156],[59,36026,36028,36031],{"className":36027},[119,156],[59,36029,13242],{"className":36030,"style":3185},[119,192,156],[59,36032,36034],{"className":36033},[196],[59,36035,36037,36057],{"className":36036},[131,132],[59,36038,36040,36054],{"className":36039},[136],[59,36041,36043],{"className":36042,"style":4302},[140],[59,36044,36045,36048],{"style":17212},[59,36046],{"className":36047,"style":213},[148],[59,36049,36051],{"className":36050},[153,217,218,156],[59,36052,335],{"className":36053},[119,192,156],[59,36055,227],{"className":36056},[226],[59,36058,36060],{"className":36059},[136],[59,36061,36063],{"className":36062,"style":4324},[140],[59,36064],{},[59,36066,227],{"className":36067},[226],[59,36069,36071],{"className":36070},[136],[59,36072,36074],{"className":36073,"style":16393},[140],[59,36075],{},[59,36077],{"className":36078,"style":916},[499],[59,36080,36082,36085],{"className":36081},[119],[59,36083,774],{"className":36084},[119,192],[59,36086,36088],{"className":36087},[196],[59,36089,36091,36163],{"className":36090},[131,132],[59,36092,36094,36160],{"className":36093},[136],[59,36095,36097,36140],{"className":36096,"style":24796},[140],[59,36098,36099,36102],{"style":17311},[59,36100],{"className":36101,"style":540},[148],[59,36103,36105],{"className":36104},[153,154,155,156],[59,36106,36108],{"className":36107},[119,156],[59,36109,36111,36114],{"className":36110},[119,156],[59,36112,14051],{"className":36113,"style":3210},[119,192,156],[59,36115,36117],{"className":36116},[196],[59,36118,36120],{"className":36119},[131],[59,36121,36123],{"className":36122},[136],[59,36124,36126],{"className":36125,"style":19144},[140],[59,36127,36128,36131],{"style":19147},[59,36129],{"className":36130,"style":213},[148],[59,36132,36134],{"className":36133},[153,217,218,156],[59,36135,36137],{"className":36136},[119,156],[59,36138,18984],{"className":36139},[119,156],[59,36141,36142,36145],{"style":16426},[59,36143],{"className":36144,"style":540},[148],[59,36146,36148],{"className":36147},[153,154,155,156],[59,36149,36151,36154,36157],{"className":36150},[119,156],[59,36152,27510],{"className":36153},[119,192,156],[59,36155,774],{"className":36156},[119,192,156],[59,36158,5607],{"className":36159,"style":6050},[119,192,156],[59,36161,227],{"className":36162},[226],[59,36164,36166],{"className":36165},[136],[59,36167,36169],{"className":36168,"style":17341},[140],[59,36170],{},[59,36172,798],{"className":36173},[123],[59,36175,36177,36180],{"className":36176},[119],[59,36178,12802],{"className":36179},[119,192],[59,36181,36183],{"className":36182},[196],[59,36184,36186,36217],{"className":36185},[131,132],[59,36187,36189,36214],{"className":36188},[136],[59,36190,36192,36203],{"className":36191,"style":17308},[140],[59,36193,36194,36197],{"style":17311},[59,36195],{"className":36196,"style":540},[148],[59,36198,36200],{"className":36199},[153,154,155,156],[59,36201,335],{"className":36202},[119,192,156],[59,36204,36205,36208],{"style":16426},[59,36206],{"className":36207,"style":540},[148],[59,36209,36211],{"className":36210},[153,154,155,156],[59,36212,777],{"className":36213},[119,192,156],[59,36215,227],{"className":36216},[226],[59,36218,36220],{"className":36219},[136],[59,36221,36223],{"className":36222,"style":17341},[140],[59,36224],{},[59,36226,803],{"className":36227},[912],[59,36229],{"className":36230,"style":916},[499],[59,36232,36234,36237],{"className":36233},[119],[59,36235,22],{"className":36236},[119,192],[59,36238,36240],{"className":36239},[196],[59,36241,36243,36274],{"className":36242},[131,132],[59,36244,36246,36271],{"className":36245},[136],[59,36247,36249,36260],{"className":36248,"style":17308},[140],[59,36250,36251,36254],{"style":17311},[59,36252],{"className":36253,"style":540},[148],[59,36255,36257],{"className":36256},[153,154,155,156],[59,36258,335],{"className":36259},[119,192,156],[59,36261,36262,36265],{"style":16426},[59,36263],{"className":36264,"style":540},[148],[59,36266,36268],{"className":36267},[153,154,155,156],[59,36269,777],{"className":36270},[119,192,156],[59,36272,227],{"className":36273},[226],[59,36275,36277],{"className":36276},[136],[59,36278,36280],{"className":36279,"style":17341},[140],[59,36281],{},[59,36283,812],{"className":36284},[313],[59,36286,36288,36291,36674],{"className":36287},[119],[59,36289],{"className":36290},[123,124],[59,36292,36294],{"className":36293},[78],[59,36295,36297,36666],{"className":36296},[131,132],[59,36298,36300,36663],{"className":36299},[136],[59,36301,36304,36496,36504],{"className":36302,"style":36303},[140],"height:1.4706em;",[59,36305,36306,36309],{"style":15819},[59,36307],{"className":36308,"style":149},[148],[59,36310,36312,36384,36387,36439,36442,36493],{"className":36311},[119],[59,36313,36315,36318],{"className":36314},[119],[59,36316,5604],{"className":36317,"style":3185},[119,192],[59,36319,36321],{"className":36320},[196],[59,36322,36324,36376],{"className":36323},[131,132],[59,36325,36327,36373],{"className":36326},[136],[59,36328,36330],{"className":36329,"style":1248},[140],[59,36331,36332,36335],{"style":7460},[59,36333],{"className":36334,"style":540},[148],[59,36336,36338],{"className":36337},[153,154,155,156],[59,36339,36341],{"className":36340},[119,156],[59,36342,36344,36347],{"className":36343},[119,156],[59,36345,14051],{"className":36346,"style":3210},[119,192,156],[59,36348,36350],{"className":36349},[196],[59,36351,36353],{"className":36352},[131],[59,36354,36356],{"className":36355},[136],[59,36357,36359],{"className":36358,"style":19144},[140],[59,36360,36361,36364],{"style":19147},[59,36362],{"className":36363,"style":213},[148],[59,36365,36367],{"className":36366},[153,217,218,156],[59,36368,36370],{"className":36369},[119,156],[59,36371,18984],{"className":36372},[119,156],[59,36374,227],{"className":36375},[226],[59,36377,36379],{"className":36378},[136],[59,36380,36382],{"className":36381,"style":1270},[140],[59,36383],{},[59,36385,798],{"className":36386},[123],[59,36388,36390,36393],{"className":36389},[119],[59,36391,22],{"className":36392},[119,192],[59,36394,36396],{"className":36395},[196],[59,36397,36399,36431],{"className":36398},[131,132],[59,36400,36402,36428],{"className":36401},[136],[59,36403,36406,36417],{"className":36404,"style":36405},[140],"height:0.7196em;",[59,36407,36408,36411],{"style":17311},[59,36409],{"className":36410,"style":540},[148],[59,36412,36414],{"className":36413},[153,154,155,156],[59,36415,335],{"className":36416},[119,192,156],[59,36418,36419,36422],{"style":16529},[59,36420],{"className":36421,"style":540},[148],[59,36423,36425],{"className":36424},[153,154,155,156],[59,36426,777],{"className":36427},[119,192,156],[59,36429,227],{"className":36430},[226],[59,36432,36434],{"className":36433},[136],[59,36435,36437],{"className":36436,"style":17341},[140],[59,36438],{},[59,36440,12941],{"className":36441},[119],[59,36443,36445,36448],{"className":36444},[119],[59,36446,12802],{"className":36447},[119,192],[59,36449,36451],{"className":36450},[196],[59,36452,36454,36485],{"className":36453},[131,132],[59,36455,36457,36482],{"className":36456},[136],[59,36458,36460,36471],{"className":36459,"style":36405},[140],[59,36461,36462,36465],{"style":17311},[59,36463],{"className":36464,"style":540},[148],[59,36466,36468],{"className":36467},[153,154,155,156],[59,36469,335],{"className":36470},[119,192,156],[59,36472,36473,36476],{"style":16529},[59,36474],{"className":36475,"style":540},[148],[59,36477,36479],{"className":36478},[153,154,155,156],[59,36480,777],{"className":36481},[119,192,156],[59,36483,227],{"className":36484},[226],[59,36486,36488],{"className":36487},[136],[59,36489,36491],{"className":36490,"style":17341},[140],[59,36492],{},[59,36494,812],{"className":36495},[313],[59,36497,36498,36501],{"style":274},[59,36499],{"className":36500,"style":149},[148],[59,36502],{"className":36503,"style":282},[281],[59,36505,36506,36509],{"style":15885},[59,36507],{"className":36508,"style":149},[148],[59,36510,36512,36552,36555,36606,36609,36660],{"className":36511},[119],[59,36513,36515,36518],{"className":36514},[119],[59,36516,5604],{"className":36517,"style":3185},[119,192],[59,36519,36521],{"className":36520},[196],[59,36522,36524,36544],{"className":36523},[131,132],[59,36525,36527,36541],{"className":36526},[136],[59,36528,36530],{"className":36529,"style":1248},[140],[59,36531,36532,36535],{"style":7460},[59,36533],{"className":36534,"style":540},[148],[59,36536,36538],{"className":36537},[153,154,155,156],[59,36539,14051],{"className":36540,"style":3210},[119,192,156],[59,36542,227],{"className":36543},[226],[59,36545,36547],{"className":36546},[136],[59,36548,36550],{"className":36549,"style":1270},[140],[59,36551],{},[59,36553,798],{"className":36554},[123],[59,36556,36558,36561],{"className":36557},[119],[59,36559,22],{"className":36560},[119,192],[59,36562,36564],{"className":36563},[196],[59,36565,36567,36598],{"className":36566},[131,132],[59,36568,36570,36595],{"className":36569},[136],[59,36571,36573,36584],{"className":36572,"style":18849},[140],[59,36574,36575,36578],{"style":17311},[59,36576],{"className":36577,"style":540},[148],[59,36579,36581],{"className":36580},[153,154,155,156],[59,36582,335],{"className":36583},[119,192,156],[59,36585,36586,36589],{"style":536},[59,36587],{"className":36588,"style":540},[148],[59,36590,36592],{"className":36591},[153,154,155,156],[59,36593,777],{"className":36594},[119,192,156],[59,36596,227],{"className":36597},[226],[59,36599,36601],{"className":36600},[136],[59,36602,36604],{"className":36603,"style":17341},[140],[59,36605],{},[59,36607,12941],{"className":36608},[119],[59,36610,36612,36615],{"className":36611},[119],[59,36613,12802],{"className":36614},[119,192],[59,36616,36618],{"className":36617},[196],[59,36619,36621,36652],{"className":36620},[131,132],[59,36622,36624,36649],{"className":36623},[136],[59,36625,36627,36638],{"className":36626,"style":18849},[140],[59,36628,36629,36632],{"style":17311},[59,36630],{"className":36631,"style":540},[148],[59,36633,36635],{"className":36634},[153,154,155,156],[59,36636,335],{"className":36637},[119,192,156],[59,36639,36640,36643],{"style":536},[59,36641],{"className":36642,"style":540},[148],[59,36644,36646],{"className":36645},[153,154,155,156],[59,36647,777],{"className":36648},[119,192,156],[59,36650,227],{"className":36651},[226],[59,36653,36655],{"className":36654},[136],[59,36656,36658],{"className":36657,"style":17341},[140],[59,36659],{},[59,36661,812],{"className":36662},[313],[59,36664,227],{"className":36665},[226],[59,36667,36669],{"className":36668},[136],[59,36670,36672],{"className":36671,"style":15950},[140],[59,36673],{},[59,36675],{"className":36676},[313,124],[59,36678,14380],{"className":36679},[119],[59,36681],{"className":36682,"style":916},[499],[59,36684,16850,36686],{"className":36685},[951],[59,36687,16854],{"style":16853},[59,36689],{"className":36690,"style":916},[499],[59,36692,36694,36697],{"className":36693},[119],[59,36695,5604],{"className":36696,"style":3185},[119,192],[59,36698,36700],{"className":36699},[196],[59,36701,36703,36723],{"className":36702},[131,132],[59,36704,36706,36720],{"className":36705},[136],[59,36707,36709],{"className":36708,"style":1248},[140],[59,36710,36711,36714],{"style":7460},[59,36712],{"className":36713,"style":540},[148],[59,36715,36717],{"className":36716},[153,154,155,156],[59,36718,14051],{"className":36719,"style":3210},[119,192,156],[59,36721,227],{"className":36722},[226],[59,36724,36726],{"className":36725},[136],[59,36727,36729],{"className":36728,"style":1270},[140],[59,36730],{},[59,36732,798],{"className":36733},[123],[59,36735,36737,36740],{"className":36736},[119],[59,36738,22],{"className":36739},[119,192],[59,36741,36743],{"className":36742},[196],[59,36744,36746,36777],{"className":36745},[131,132],[59,36747,36749,36774],{"className":36748},[136],[59,36750,36752,36763],{"className":36751,"style":17308},[140],[59,36753,36754,36757],{"style":17311},[59,36755],{"className":36756,"style":540},[148],[59,36758,36760],{"className":36759},[153,154,155,156],[59,36761,335],{"className":36762},[119,192,156],[59,36764,36765,36768],{"style":16426},[59,36766],{"className":36767,"style":540},[148],[59,36769,36771],{"className":36770},[153,154,155,156],[59,36772,777],{"className":36773},[119,192,156],[59,36775,227],{"className":36776},[226],[59,36778,36780],{"className":36779},[136],[59,36781,36783],{"className":36782,"style":17341},[140],[59,36784],{},[59,36786,12941],{"className":36787},[119],[59,36789,36791,36794],{"className":36790},[119],[59,36792,12802],{"className":36793},[119,192],[59,36795,36797],{"className":36796},[196],[59,36798,36800,36831],{"className":36799},[131,132],[59,36801,36803,36828],{"className":36802},[136],[59,36804,36806,36817],{"className":36805,"style":17308},[140],[59,36807,36808,36811],{"style":17311},[59,36809],{"className":36810,"style":540},[148],[59,36812,36814],{"className":36813},[153,154,155,156],[59,36815,335],{"className":36816},[119,192,156],[59,36818,36819,36822],{"style":16426},[59,36820],{"className":36821,"style":540},[148],[59,36823,36825],{"className":36824},[153,154,155,156],[59,36826,777],{"className":36827},[119,192,156],[59,36829,227],{"className":36830},[226],[59,36832,36834],{"className":36833},[136],[59,36835,36837],{"className":36836,"style":17341},[140],[59,36838],{},[59,36840,812],{"className":36841},[313],[59,36843,36844,36847],{"style":35185},[59,36845],{"className":36846,"style":15170},[148],[59,36848,36850,36853,36856,36859,36862,36924,36927,36994,36997,37101,37104,37196,37199,37250,37253,37256,37307,37310,37700],{"className":36849},[119],[59,36851],{"className":36852},[119],[59,36854],{"className":36855,"style":500},[499],[59,36857,815],{"className":36858},[504],[59,36860],{"className":36861,"style":500},[499],[59,36863,36865,36868,36921],{"className":36864},[119],[59,36866],{"className":36867},[123,124],[59,36869,36871],{"className":36870},[78],[59,36872,36874,36913],{"className":36873},[131,132],[59,36875,36877,36910],{"className":36876},[136],[59,36878,36880,36891,36899],{"className":36879,"style":16279},[140],[59,36881,36882,36885],{"style":15819},[59,36883],{"className":36884,"style":149},[148],[59,36886,36888],{"className":36887},[119],[59,36889,7322],{"className":36890,"style":7397},[119,192],[59,36892,36893,36896],{"style":274},[59,36894],{"className":36895,"style":149},[148],[59,36897],{"className":36898,"style":282},[281],[59,36900,36901,36904],{"style":15885},[59,36902],{"className":36903,"style":149},[148],[59,36905,36907],{"className":36906},[119],[59,36908,83],{"className":36909},[119],[59,36911,227],{"className":36912},[226],[59,36914,36916],{"className":36915},[136],[59,36917,36919],{"className":36918,"style":16319},[140],[59,36920],{},[59,36922],{"className":36923},[313,124],[59,36925],{"className":36926,"style":916},[499],[59,36928,36930],{"className":36929},[951,14238],[59,36931,36933,36986],{"className":36932},[131,132],[59,36934,36936,36983],{"className":36935},[136],[59,36937,36939,36959,36969],{"className":36938,"style":16340},[140],[59,36940,36941,36944],{"style":16343},[59,36942],{"className":36943,"style":15432},[148],[59,36945,36947],{"className":36946},[153,154,155,156],[59,36948,36950,36953,36956],{"className":36949},[119,156],[59,36951,335],{"className":36952},[119,192,156],[59,36954,815],{"className":36955},[504,156],[59,36957,83],{"className":36958},[119,156],[59,36960,36961,36964],{"style":15441},[59,36962],{"className":36963,"style":15432},[148],[59,36965,36966],{},[59,36967,1826],{"className":36968},[951,1910,15450],[59,36970,36971,36974],{"style":16374},[59,36972],{"className":36973,"style":15432},[148],[59,36975,36977],{"className":36976},[153,154,155,156],[59,36978,36980],{"className":36979},[119,156],[59,36981,7322],{"className":36982,"style":7397},[119,192,156],[59,36984,227],{"className":36985},[226],[59,36987,36989],{"className":36988},[136],[59,36990,36992],{"className":36991,"style":16393},[140],[59,36993],{},[59,36995],{"className":36996,"style":916},[499],[59,36998,37000],{"className":36999},[951,14238],[59,37001,37003,37093],{"className":37002},[131,132],[59,37004,37006,37090],{"className":37005},[136],[59,37007,37009,37029,37039],{"className":37008,"style":17149},[140],[59,37010,37011,37014],{"style":16343},[59,37012],{"className":37013,"style":15432},[148],[59,37015,37017],{"className":37016},[153,154,155,156],[59,37018,37020,37023,37026],{"className":37019},[119,156],[59,37021,777],{"className":37022},[119,192,156],[59,37024,815],{"className":37025},[504,156],[59,37027,83],{"className":37028},[119,156],[59,37030,37031,37034],{"style":15441},[59,37032],{"className":37033,"style":15432},[148],[59,37035,37036],{},[59,37037,1826],{"className":37038},[951,1910,15450],[59,37040,37041,37044],{"style":17182},[59,37042],{"className":37043,"style":15432},[148],[59,37045,37047],{"className":37046},[153,154,155,156],[59,37048,37050],{"className":37049},[119,156],[59,37051,37053,37056],{"className":37052},[119,156],[59,37054,13242],{"className":37055,"style":3185},[119,192,156],[59,37057,37059],{"className":37058},[196],[59,37060,37062,37082],{"className":37061},[131,132],[59,37063,37065,37079],{"className":37064},[136],[59,37066,37068],{"className":37067,"style":4302},[140],[59,37069,37070,37073],{"style":17212},[59,37071],{"className":37072,"style":213},[148],[59,37074,37076],{"className":37075},[153,217,218,156],[59,37077,335],{"className":37078},[119,192,156],[59,37080,227],{"className":37081},[226],[59,37083,37085],{"className":37084},[136],[59,37086,37088],{"className":37087,"style":4324},[140],[59,37089],{},[59,37091,227],{"className":37092},[226],[59,37094,37096],{"className":37095},[136],[59,37097,37099],{"className":37098,"style":16393},[140],[59,37100],{},[59,37102],{"className":37103,"style":916},[499],[59,37105,37107,37110],{"className":37106},[119],[59,37108,774],{"className":37109},[119,192],[59,37111,37113],{"className":37112},[196],[59,37114,37116,37188],{"className":37115},[131,132],[59,37117,37119,37185],{"className":37118},[136],[59,37120,37122,37165],{"className":37121,"style":24796},[140],[59,37123,37124,37127],{"style":17311},[59,37125],{"className":37126,"style":540},[148],[59,37128,37130],{"className":37129},[153,154,155,156],[59,37131,37133],{"className":37132},[119,156],[59,37134,37136,37139],{"className":37135},[119,156],[59,37137,14051],{"className":37138,"style":3210},[119,192,156],[59,37140,37142],{"className":37141},[196],[59,37143,37145],{"className":37144},[131],[59,37146,37148],{"className":37147},[136],[59,37149,37151],{"className":37150,"style":19144},[140],[59,37152,37153,37156],{"style":19147},[59,37154],{"className":37155,"style":213},[148],[59,37157,37159],{"className":37158},[153,217,218,156],[59,37160,37162],{"className":37161},[119,156],[59,37163,18984],{"className":37164},[119,156],[59,37166,37167,37170],{"style":16426},[59,37168],{"className":37169,"style":540},[148],[59,37171,37173],{"className":37172},[153,154,155,156],[59,37174,37176,37179,37182],{"className":37175},[119,156],[59,37177,27510],{"className":37178},[119,192,156],[59,37180,774],{"className":37181},[119,192,156],[59,37183,5607],{"className":37184,"style":6050},[119,192,156],[59,37186,227],{"className":37187},[226],[59,37189,37191],{"className":37190},[136],[59,37192,37194],{"className":37193,"style":17341},[140],[59,37195],{},[59,37197,798],{"className":37198},[123],[59,37200,37202,37205],{"className":37201},[119],[59,37203,12802],{"className":37204},[119,192],[59,37206,37208],{"className":37207},[196],[59,37209,37211,37242],{"className":37210},[131,132],[59,37212,37214,37239],{"className":37213},[136],[59,37215,37217,37228],{"className":37216,"style":17308},[140],[59,37218,37219,37222],{"style":17311},[59,37220],{"className":37221,"style":540},[148],[59,37223,37225],{"className":37224},[153,154,155,156],[59,37226,335],{"className":37227},[119,192,156],[59,37229,37230,37233],{"style":16426},[59,37231],{"className":37232,"style":540},[148],[59,37234,37236],{"className":37235},[153,154,155,156],[59,37237,777],{"className":37238},[119,192,156],[59,37240,227],{"className":37241},[226],[59,37243,37245],{"className":37244},[136],[59,37246,37248],{"className":37247,"style":17341},[140],[59,37249],{},[59,37251,803],{"className":37252},[912],[59,37254],{"className":37255,"style":916},[499],[59,37257,37259,37262],{"className":37258},[119],[59,37260,22],{"className":37261},[119,192],[59,37263,37265],{"className":37264},[196],[59,37266,37268,37299],{"className":37267},[131,132],[59,37269,37271,37296],{"className":37270},[136],[59,37272,37274,37285],{"className":37273,"style":17308},[140],[59,37275,37276,37279],{"style":17311},[59,37277],{"className":37278,"style":540},[148],[59,37280,37282],{"className":37281},[153,154,155,156],[59,37283,335],{"className":37284},[119,192,156],[59,37286,37287,37290],{"style":16426},[59,37288],{"className":37289,"style":540},[148],[59,37291,37293],{"className":37292},[153,154,155,156],[59,37294,777],{"className":37295},[119,192,156],[59,37297,227],{"className":37298},[226],[59,37300,37302],{"className":37301},[136],[59,37303,37305],{"className":37304,"style":17341},[140],[59,37306],{},[59,37308,812],{"className":37309},[313],[59,37311,37313,37316,37697],{"className":37312},[119],[59,37314],{"className":37315},[123,124],[59,37317,37319],{"className":37318},[78],[59,37320,37322,37689],{"className":37321},[131,132],[59,37323,37325,37686],{"className":37324},[136],[59,37326,37328,37519,37527],{"className":37327,"style":36303},[140],[59,37329,37330,37333],{"style":15819},[59,37331],{"className":37332,"style":149},[148],[59,37334,37336,37408,37411,37462,37465,37516],{"className":37335},[119],[59,37337,37339,37342],{"className":37338},[119],[59,37340,5604],{"className":37341,"style":3185},[119,192],[59,37343,37345],{"className":37344},[196],[59,37346,37348,37400],{"className":37347},[131,132],[59,37349,37351,37397],{"className":37350},[136],[59,37352,37354],{"className":37353,"style":1248},[140],[59,37355,37356,37359],{"style":7460},[59,37357],{"className":37358,"style":540},[148],[59,37360,37362],{"className":37361},[153,154,155,156],[59,37363,37365],{"className":37364},[119,156],[59,37366,37368,37371],{"className":37367},[119,156],[59,37369,14051],{"className":37370,"style":3210},[119,192,156],[59,37372,37374],{"className":37373},[196],[59,37375,37377],{"className":37376},[131],[59,37378,37380],{"className":37379},[136],[59,37381,37383],{"className":37382,"style":19144},[140],[59,37384,37385,37388],{"style":19147},[59,37386],{"className":37387,"style":213},[148],[59,37389,37391],{"className":37390},[153,217,218,156],[59,37392,37394],{"className":37393},[119,156],[59,37395,18984],{"className":37396},[119,156],[59,37398,227],{"className":37399},[226],[59,37401,37403],{"className":37402},[136],[59,37404,37406],{"className":37405,"style":1270},[140],[59,37407],{},[59,37409,798],{"className":37410},[123],[59,37412,37414,37417],{"className":37413},[119],[59,37415,22],{"className":37416},[119,192],[59,37418,37420],{"className":37419},[196],[59,37421,37423,37454],{"className":37422},[131,132],[59,37424,37426,37451],{"className":37425},[136],[59,37427,37429,37440],{"className":37428,"style":36405},[140],[59,37430,37431,37434],{"style":17311},[59,37432],{"className":37433,"style":540},[148],[59,37435,37437],{"className":37436},[153,154,155,156],[59,37438,335],{"className":37439},[119,192,156],[59,37441,37442,37445],{"style":16529},[59,37443],{"className":37444,"style":540},[148],[59,37446,37448],{"className":37447},[153,154,155,156],[59,37449,777],{"className":37450},[119,192,156],[59,37452,227],{"className":37453},[226],[59,37455,37457],{"className":37456},[136],[59,37458,37460],{"className":37459,"style":17341},[140],[59,37461],{},[59,37463,12941],{"className":37464},[119],[59,37466,37468,37471],{"className":37467},[119],[59,37469,12802],{"className":37470},[119,192],[59,37472,37474],{"className":37473},[196],[59,37475,37477,37508],{"className":37476},[131,132],[59,37478,37480,37505],{"className":37479},[136],[59,37481,37483,37494],{"className":37482,"style":36405},[140],[59,37484,37485,37488],{"style":17311},[59,37486],{"className":37487,"style":540},[148],[59,37489,37491],{"className":37490},[153,154,155,156],[59,37492,335],{"className":37493},[119,192,156],[59,37495,37496,37499],{"style":16529},[59,37497],{"className":37498,"style":540},[148],[59,37500,37502],{"className":37501},[153,154,155,156],[59,37503,777],{"className":37504},[119,192,156],[59,37506,227],{"className":37507},[226],[59,37509,37511],{"className":37510},[136],[59,37512,37514],{"className":37513,"style":17341},[140],[59,37515],{},[59,37517,812],{"className":37518},[313],[59,37520,37521,37524],{"style":274},[59,37522],{"className":37523,"style":149},[148],[59,37525],{"className":37526,"style":282},[281],[59,37528,37529,37532],{"style":15885},[59,37530],{"className":37531,"style":149},[148],[59,37533,37535,37575,37578,37629,37632,37683],{"className":37534},[119],[59,37536,37538,37541],{"className":37537},[119],[59,37539,5604],{"className":37540,"style":3185},[119,192],[59,37542,37544],{"className":37543},[196],[59,37545,37547,37567],{"className":37546},[131,132],[59,37548,37550,37564],{"className":37549},[136],[59,37551,37553],{"className":37552,"style":1248},[140],[59,37554,37555,37558],{"style":7460},[59,37556],{"className":37557,"style":540},[148],[59,37559,37561],{"className":37560},[153,154,155,156],[59,37562,14051],{"className":37563,"style":3210},[119,192,156],[59,37565,227],{"className":37566},[226],[59,37568,37570],{"className":37569},[136],[59,37571,37573],{"className":37572,"style":1270},[140],[59,37574],{},[59,37576,798],{"className":37577},[123],[59,37579,37581,37584],{"className":37580},[119],[59,37582,22],{"className":37583},[119,192],[59,37585,37587],{"className":37586},[196],[59,37588,37590,37621],{"className":37589},[131,132],[59,37591,37593,37618],{"className":37592},[136],[59,37594,37596,37607],{"className":37595,"style":18849},[140],[59,37597,37598,37601],{"style":17311},[59,37599],{"className":37600,"style":540},[148],[59,37602,37604],{"className":37603},[153,154,155,156],[59,37605,335],{"className":37606},[119,192,156],[59,37608,37609,37612],{"style":536},[59,37610],{"className":37611,"style":540},[148],[59,37613,37615],{"className":37614},[153,154,155,156],[59,37616,777],{"className":37617},[119,192,156],[59,37619,227],{"className":37620},[226],[59,37622,37624],{"className":37623},[136],[59,37625,37627],{"className":37626,"style":17341},[140],[59,37628],{},[59,37630,12941],{"className":37631},[119],[59,37633,37635,37638],{"className":37634},[119],[59,37636,12802],{"className":37637},[119,192],[59,37639,37641],{"className":37640},[196],[59,37642,37644,37675],{"className":37643},[131,132],[59,37645,37647,37672],{"className":37646},[136],[59,37648,37650,37661],{"className":37649,"style":18849},[140],[59,37651,37652,37655],{"style":17311},[59,37653],{"className":37654,"style":540},[148],[59,37656,37658],{"className":37657},[153,154,155,156],[59,37659,335],{"className":37660},[119,192,156],[59,37662,37663,37666],{"style":536},[59,37664],{"className":37665,"style":540},[148],[59,37667,37669],{"className":37668},[153,154,155,156],[59,37670,777],{"className":37671},[119,192,156],[59,37673,227],{"className":37674},[226],[59,37676,37678],{"className":37677},[136],[59,37679,37681],{"className":37680,"style":17341},[140],[59,37682],{},[59,37684,812],{"className":37685},[313],[59,37687,227],{"className":37688},[226],[59,37690,37692],{"className":37691},[136],[59,37693,37695],{"className":37694,"style":15950},[140],[59,37696],{},[59,37698],{"className":37699},[313,124],[59,37701,37703,37706,38058],{"className":37702},[119],[59,37704],{"className":37705},[123,124],[59,37707,37709],{"className":37708},[78],[59,37710,37712,38050],{"className":37711},[131,132],[59,37713,37715,38047],{"className":37714},[136],[59,37716,37718,37877,37885],{"className":37717,"style":36303},[140],[59,37719,37720,37723],{"style":15819},[59,37721],{"className":37722,"style":149},[148],[59,37724,37726,37766,37769,37820,37823,37874],{"className":37725},[119],[59,37727,37729,37732],{"className":37728},[119],[59,37730,5604],{"className":37731,"style":3185},[119,192],[59,37733,37735],{"className":37734},[196],[59,37736,37738,37758],{"className":37737},[131,132],[59,37739,37741,37755],{"className":37740},[136],[59,37742,37744],{"className":37743,"style":1248},[140],[59,37745,37746,37749],{"style":7460},[59,37747],{"className":37748,"style":540},[148],[59,37750,37752],{"className":37751},[153,154,155,156],[59,37753,14051],{"className":37754,"style":3210},[119,192,156],[59,37756,227],{"className":37757},[226],[59,37759,37761],{"className":37760},[136],[59,37762,37764],{"className":37763,"style":1270},[140],[59,37765],{},[59,37767,798],{"className":37768},[123],[59,37770,37772,37775],{"className":37771},[119],[59,37773,22],{"className":37774},[119,192],[59,37776,37778],{"className":37777},[196],[59,37779,37781,37812],{"className":37780},[131,132],[59,37782,37784,37809],{"className":37783},[136],[59,37785,37787,37798],{"className":37786,"style":36405},[140],[59,37788,37789,37792],{"style":17311},[59,37790],{"className":37791,"style":540},[148],[59,37793,37795],{"className":37794},[153,154,155,156],[59,37796,335],{"className":37797},[119,192,156],[59,37799,37800,37803],{"style":16529},[59,37801],{"className":37802,"style":540},[148],[59,37804,37806],{"className":37805},[153,154,155,156],[59,37807,777],{"className":37808},[119,192,156],[59,37810,227],{"className":37811},[226],[59,37813,37815],{"className":37814},[136],[59,37816,37818],{"className":37817,"style":17341},[140],[59,37819],{},[59,37821,12941],{"className":37822},[119],[59,37824,37826,37829],{"className":37825},[119],[59,37827,12802],{"className":37828},[119,192],[59,37830,37832],{"className":37831},[196],[59,37833,37835,37866],{"className":37834},[131,132],[59,37836,37838,37863],{"className":37837},[136],[59,37839,37841,37852],{"className":37840,"style":36405},[140],[59,37842,37843,37846],{"style":17311},[59,37844],{"className":37845,"style":540},[148],[59,37847,37849],{"className":37848},[153,154,155,156],[59,37850,335],{"className":37851},[119,192,156],[59,37853,37854,37857],{"style":16529},[59,37855],{"className":37856,"style":540},[148],[59,37858,37860],{"className":37859},[153,154,155,156],[59,37861,777],{"className":37862},[119,192,156],[59,37864,227],{"className":37865},[226],[59,37867,37869],{"className":37868},[136],[59,37870,37872],{"className":37871,"style":17341},[140],[59,37873],{},[59,37875,812],{"className":37876},[313],[59,37878,37879,37882],{"style":274},[59,37880],{"className":37881,"style":149},[148],[59,37883],{"className":37884,"style":282},[281],[59,37886,37887,37890],{"style":15885},[59,37888],{"className":37889,"style":149},[148],[59,37891,37893,37896,37936,37939,37990,37993,38044],{"className":37892},[119],[59,37894,14380],{"className":37895},[119],[59,37897,37899,37902],{"className":37898},[119],[59,37900,5604],{"className":37901,"style":3185},[119,192],[59,37903,37905],{"className":37904},[196],[59,37906,37908,37928],{"className":37907},[131,132],[59,37909,37911,37925],{"className":37910},[136],[59,37912,37914],{"className":37913,"style":1248},[140],[59,37915,37916,37919],{"style":7460},[59,37917],{"className":37918,"style":540},[148],[59,37920,37922],{"className":37921},[153,154,155,156],[59,37923,14051],{"className":37924,"style":3210},[119,192,156],[59,37926,227],{"className":37927},[226],[59,37929,37931],{"className":37930},[136],[59,37932,37934],{"className":37933,"style":1270},[140],[59,37935],{},[59,37937,798],{"className":37938},[123],[59,37940,37942,37945],{"className":37941},[119],[59,37943,22],{"className":37944},[119,192],[59,37946,37948],{"className":37947},[196],[59,37949,37951,37982],{"className":37950},[131,132],[59,37952,37954,37979],{"className":37953},[136],[59,37955,37957,37968],{"className":37956,"style":18849},[140],[59,37958,37959,37962],{"style":17311},[59,37960],{"className":37961,"style":540},[148],[59,37963,37965],{"className":37964},[153,154,155,156],[59,37966,335],{"className":37967},[119,192,156],[59,37969,37970,37973],{"style":536},[59,37971],{"className":37972,"style":540},[148],[59,37974,37976],{"className":37975},[153,154,155,156],[59,37977,777],{"className":37978},[119,192,156],[59,37980,227],{"className":37981},[226],[59,37983,37985],{"className":37984},[136],[59,37986,37988],{"className":37987,"style":17341},[140],[59,37989],{},[59,37991,12941],{"className":37992},[119],[59,37994,37996,37999],{"className":37995},[119],[59,37997,12802],{"className":37998},[119,192],[59,38000,38002],{"className":38001},[196],[59,38003,38005,38036],{"className":38004},[131,132],[59,38006,38008,38033],{"className":38007},[136],[59,38009,38011,38022],{"className":38010,"style":18849},[140],[59,38012,38013,38016],{"style":17311},[59,38014],{"className":38015,"style":540},[148],[59,38017,38019],{"className":38018},[153,154,155,156],[59,38020,335],{"className":38021},[119,192,156],[59,38023,38024,38027],{"style":536},[59,38025],{"className":38026,"style":540},[148],[59,38028,38030],{"className":38029},[153,154,155,156],[59,38031,777],{"className":38032},[119,192,156],[59,38034,227],{"className":38035},[226],[59,38037,38039],{"className":38038},[136],[59,38040,38042],{"className":38041,"style":17341},[140],[59,38043],{},[59,38045,812],{"className":38046},[313],[59,38048,227],{"className":38049},[226],[59,38051,38053],{"className":38052},[136],[59,38054,38056],{"className":38055,"style":15950},[140],[59,38057],{},[59,38059],{"className":38060},[313,124],[59,38062,38063,38066],{"style":35194},[59,38064],{"className":38065,"style":15170},[148],[59,38067,38069,38072,38075,38078,38081,38143,38146,38213,38216,38320,38323,38415,38418,38469,38472,38475,38526,38529],{"className":38068},[119],[59,38070],{"className":38071},[119],[59,38073],{"className":38074,"style":500},[499],[59,38076,815],{"className":38077},[504],[59,38079],{"className":38080,"style":500},[499],[59,38082,38084,38087,38140],{"className":38083},[119],[59,38085],{"className":38086},[123,124],[59,38088,38090],{"className":38089},[78],[59,38091,38093,38132],{"className":38092},[131,132],[59,38094,38096,38129],{"className":38095},[136],[59,38097,38099,38110,38118],{"className":38098,"style":16279},[140],[59,38100,38101,38104],{"style":15819},[59,38102],{"className":38103,"style":149},[148],[59,38105,38107],{"className":38106},[119],[59,38108,7322],{"className":38109,"style":7397},[119,192],[59,38111,38112,38115],{"style":274},[59,38113],{"className":38114,"style":149},[148],[59,38116],{"className":38117,"style":282},[281],[59,38119,38120,38123],{"style":15885},[59,38121],{"className":38122,"style":149},[148],[59,38124,38126],{"className":38125},[119],[59,38127,83],{"className":38128},[119],[59,38130,227],{"className":38131},[226],[59,38133,38135],{"className":38134},[136],[59,38136,38138],{"className":38137,"style":16319},[140],[59,38139],{},[59,38141],{"className":38142},[313,124],[59,38144],{"className":38145,"style":916},[499],[59,38147,38149],{"className":38148},[951,14238],[59,38150,38152,38205],{"className":38151},[131,132],[59,38153,38155,38202],{"className":38154},[136],[59,38156,38158,38178,38188],{"className":38157,"style":16340},[140],[59,38159,38160,38163],{"style":16343},[59,38161],{"className":38162,"style":15432},[148],[59,38164,38166],{"className":38165},[153,154,155,156],[59,38167,38169,38172,38175],{"className":38168},[119,156],[59,38170,335],{"className":38171},[119,192,156],[59,38173,815],{"className":38174},[504,156],[59,38176,83],{"className":38177},[119,156],[59,38179,38180,38183],{"style":15441},[59,38181],{"className":38182,"style":15432},[148],[59,38184,38185],{},[59,38186,1826],{"className":38187},[951,1910,15450],[59,38189,38190,38193],{"style":16374},[59,38191],{"className":38192,"style":15432},[148],[59,38194,38196],{"className":38195},[153,154,155,156],[59,38197,38199],{"className":38198},[119,156],[59,38200,7322],{"className":38201,"style":7397},[119,192,156],[59,38203,227],{"className":38204},[226],[59,38206,38208],{"className":38207},[136],[59,38209,38211],{"className":38210,"style":16393},[140],[59,38212],{},[59,38214],{"className":38215,"style":916},[499],[59,38217,38219],{"className":38218},[951,14238],[59,38220,38222,38312],{"className":38221},[131,132],[59,38223,38225,38309],{"className":38224},[136],[59,38226,38228,38248,38258],{"className":38227,"style":17149},[140],[59,38229,38230,38233],{"style":16343},[59,38231],{"className":38232,"style":15432},[148],[59,38234,38236],{"className":38235},[153,154,155,156],[59,38237,38239,38242,38245],{"className":38238},[119,156],[59,38240,777],{"className":38241},[119,192,156],[59,38243,815],{"className":38244},[504,156],[59,38246,83],{"className":38247},[119,156],[59,38249,38250,38253],{"style":15441},[59,38251],{"className":38252,"style":15432},[148],[59,38254,38255],{},[59,38256,1826],{"className":38257},[951,1910,15450],[59,38259,38260,38263],{"style":17182},[59,38261],{"className":38262,"style":15432},[148],[59,38264,38266],{"className":38265},[153,154,155,156],[59,38267,38269],{"className":38268},[119,156],[59,38270,38272,38275],{"className":38271},[119,156],[59,38273,13242],{"className":38274,"style":3185},[119,192,156],[59,38276,38278],{"className":38277},[196],[59,38279,38281,38301],{"className":38280},[131,132],[59,38282,38284,38298],{"className":38283},[136],[59,38285,38287],{"className":38286,"style":4302},[140],[59,38288,38289,38292],{"style":17212},[59,38290],{"className":38291,"style":213},[148],[59,38293,38295],{"className":38294},[153,217,218,156],[59,38296,335],{"className":38297},[119,192,156],[59,38299,227],{"className":38300},[226],[59,38302,38304],{"className":38303},[136],[59,38305,38307],{"className":38306,"style":4324},[140],[59,38308],{},[59,38310,227],{"className":38311},[226],[59,38313,38315],{"className":38314},[136],[59,38316,38318],{"className":38317,"style":16393},[140],[59,38319],{},[59,38321],{"className":38322,"style":916},[499],[59,38324,38326,38329],{"className":38325},[119],[59,38327,774],{"className":38328},[119,192],[59,38330,38332],{"className":38331},[196],[59,38333,38335,38407],{"className":38334},[131,132],[59,38336,38338,38404],{"className":38337},[136],[59,38339,38341,38384],{"className":38340,"style":24796},[140],[59,38342,38343,38346],{"style":17311},[59,38344],{"className":38345,"style":540},[148],[59,38347,38349],{"className":38348},[153,154,155,156],[59,38350,38352],{"className":38351},[119,156],[59,38353,38355,38358],{"className":38354},[119,156],[59,38356,14051],{"className":38357,"style":3210},[119,192,156],[59,38359,38361],{"className":38360},[196],[59,38362,38364],{"className":38363},[131],[59,38365,38367],{"className":38366},[136],[59,38368,38370],{"className":38369,"style":19144},[140],[59,38371,38372,38375],{"style":19147},[59,38373],{"className":38374,"style":213},[148],[59,38376,38378],{"className":38377},[153,217,218,156],[59,38379,38381],{"className":38380},[119,156],[59,38382,18984],{"className":38383},[119,156],[59,38385,38386,38389],{"style":16426},[59,38387],{"className":38388,"style":540},[148],[59,38390,38392],{"className":38391},[153,154,155,156],[59,38393,38395,38398,38401],{"className":38394},[119,156],[59,38396,27510],{"className":38397},[119,192,156],[59,38399,774],{"className":38400},[119,192,156],[59,38402,5607],{"className":38403,"style":6050},[119,192,156],[59,38405,227],{"className":38406},[226],[59,38408,38410],{"className":38409},[136],[59,38411,38413],{"className":38412,"style":17341},[140],[59,38414],{},[59,38416,798],{"className":38417},[123],[59,38419,38421,38424],{"className":38420},[119],[59,38422,12802],{"className":38423},[119,192],[59,38425,38427],{"className":38426},[196],[59,38428,38430,38461],{"className":38429},[131,132],[59,38431,38433,38458],{"className":38432},[136],[59,38434,38436,38447],{"className":38435,"style":17308},[140],[59,38437,38438,38441],{"style":17311},[59,38439],{"className":38440,"style":540},[148],[59,38442,38444],{"className":38443},[153,154,155,156],[59,38445,335],{"className":38446},[119,192,156],[59,38448,38449,38452],{"style":16426},[59,38450],{"className":38451,"style":540},[148],[59,38453,38455],{"className":38454},[153,154,155,156],[59,38456,777],{"className":38457},[119,192,156],[59,38459,227],{"className":38460},[226],[59,38462,38464],{"className":38463},[136],[59,38465,38467],{"className":38466,"style":17341},[140],[59,38468],{},[59,38470,803],{"className":38471},[912],[59,38473],{"className":38474,"style":916},[499],[59,38476,38478,38481],{"className":38477},[119],[59,38479,22],{"className":38480},[119,192],[59,38482,38484],{"className":38483},[196],[59,38485,38487,38518],{"className":38486},[131,132],[59,38488,38490,38515],{"className":38489},[136],[59,38491,38493,38504],{"className":38492,"style":17308},[140],[59,38494,38495,38498],{"style":17311},[59,38496],{"className":38497,"style":540},[148],[59,38499,38501],{"className":38500},[153,154,155,156],[59,38502,335],{"className":38503},[119,192,156],[59,38505,38506,38509],{"style":16426},[59,38507],{"className":38508,"style":540},[148],[59,38510,38512],{"className":38511},[153,154,155,156],[59,38513,777],{"className":38514},[119,192,156],[59,38516,227],{"className":38517},[226],[59,38519,38521],{"className":38520},[136],[59,38522,38524],{"className":38523,"style":17341},[140],[59,38525],{},[59,38527,812],{"className":38528},[313],[59,38530,38532,38535,38919],{"className":38531},[119],[59,38533],{"className":38534},[123,124],[59,38536,38538],{"className":38537},[78],[59,38539,38541,38911],{"className":38540},[131,132],[59,38542,38544,38908],{"className":38543},[136],[59,38545,38547,38738,38746],{"className":38546,"style":36303},[140],[59,38548,38549,38552],{"style":15819},[59,38550],{"className":38551,"style":149},[148],[59,38553,38555,38627,38630,38681,38684,38735],{"className":38554},[119],[59,38556,38558,38561],{"className":38557},[119],[59,38559,5604],{"className":38560,"style":3185},[119,192],[59,38562,38564],{"className":38563},[196],[59,38565,38567,38619],{"className":38566},[131,132],[59,38568,38570,38616],{"className":38569},[136],[59,38571,38573],{"className":38572,"style":1248},[140],[59,38574,38575,38578],{"style":7460},[59,38576],{"className":38577,"style":540},[148],[59,38579,38581],{"className":38580},[153,154,155,156],[59,38582,38584],{"className":38583},[119,156],[59,38585,38587,38590],{"className":38586},[119,156],[59,38588,14051],{"className":38589,"style":3210},[119,192,156],[59,38591,38593],{"className":38592},[196],[59,38594,38596],{"className":38595},[131],[59,38597,38599],{"className":38598},[136],[59,38600,38602],{"className":38601,"style":19144},[140],[59,38603,38604,38607],{"style":19147},[59,38605],{"className":38606,"style":213},[148],[59,38608,38610],{"className":38609},[153,217,218,156],[59,38611,38613],{"className":38612},[119,156],[59,38614,18984],{"className":38615},[119,156],[59,38617,227],{"className":38618},[226],[59,38620,38622],{"className":38621},[136],[59,38623,38625],{"className":38624,"style":1270},[140],[59,38626],{},[59,38628,798],{"className":38629},[123],[59,38631,38633,38636],{"className":38632},[119],[59,38634,22],{"className":38635},[119,192],[59,38637,38639],{"className":38638},[196],[59,38640,38642,38673],{"className":38641},[131,132],[59,38643,38645,38670],{"className":38644},[136],[59,38646,38648,38659],{"className":38647,"style":36405},[140],[59,38649,38650,38653],{"style":17311},[59,38651],{"className":38652,"style":540},[148],[59,38654,38656],{"className":38655},[153,154,155,156],[59,38657,335],{"className":38658},[119,192,156],[59,38660,38661,38664],{"style":16529},[59,38662],{"className":38663,"style":540},[148],[59,38665,38667],{"className":38666},[153,154,155,156],[59,38668,777],{"className":38669},[119,192,156],[59,38671,227],{"className":38672},[226],[59,38674,38676],{"className":38675},[136],[59,38677,38679],{"className":38678,"style":17341},[140],[59,38680],{},[59,38682,12941],{"className":38683},[119],[59,38685,38687,38690],{"className":38686},[119],[59,38688,12802],{"className":38689},[119,192],[59,38691,38693],{"className":38692},[196],[59,38694,38696,38727],{"className":38695},[131,132],[59,38697,38699,38724],{"className":38698},[136],[59,38700,38702,38713],{"className":38701,"style":36405},[140],[59,38703,38704,38707],{"style":17311},[59,38705],{"className":38706,"style":540},[148],[59,38708,38710],{"className":38709},[153,154,155,156],[59,38711,335],{"className":38712},[119,192,156],[59,38714,38715,38718],{"style":16529},[59,38716],{"className":38717,"style":540},[148],[59,38719,38721],{"className":38720},[153,154,155,156],[59,38722,777],{"className":38723},[119,192,156],[59,38725,227],{"className":38726},[226],[59,38728,38730],{"className":38729},[136],[59,38731,38733],{"className":38732,"style":17341},[140],[59,38734],{},[59,38736,812],{"className":38737},[313],[59,38739,38740,38743],{"style":274},[59,38741],{"className":38742,"style":149},[148],[59,38744],{"className":38745,"style":282},[281],[59,38747,38748,38751],{"style":15885},[59,38749],{"className":38750,"style":149},[148],[59,38752,38754,38757,38797,38800,38851,38854,38905],{"className":38753},[119],[59,38755,14380],{"className":38756},[119],[59,38758,38760,38763],{"className":38759},[119],[59,38761,5604],{"className":38762,"style":3185},[119,192],[59,38764,38766],{"className":38765},[196],[59,38767,38769,38789],{"className":38768},[131,132],[59,38770,38772,38786],{"className":38771},[136],[59,38773,38775],{"className":38774,"style":1248},[140],[59,38776,38777,38780],{"style":7460},[59,38778],{"className":38779,"style":540},[148],[59,38781,38783],{"className":38782},[153,154,155,156],[59,38784,14051],{"className":38785,"style":3210},[119,192,156],[59,38787,227],{"className":38788},[226],[59,38790,38792],{"className":38791},[136],[59,38793,38795],{"className":38794,"style":1270},[140],[59,38796],{},[59,38798,798],{"className":38799},[123],[59,38801,38803,38806],{"className":38802},[119],[59,38804,22],{"className":38805},[119,192],[59,38807,38809],{"className":38808},[196],[59,38810,38812,38843],{"className":38811},[131,132],[59,38813,38815,38840],{"className":38814},[136],[59,38816,38818,38829],{"className":38817,"style":18849},[140],[59,38819,38820,38823],{"style":17311},[59,38821],{"className":38822,"style":540},[148],[59,38824,38826],{"className":38825},[153,154,155,156],[59,38827,335],{"className":38828},[119,192,156],[59,38830,38831,38834],{"style":536},[59,38832],{"className":38833,"style":540},[148],[59,38835,38837],{"className":38836},[153,154,155,156],[59,38838,777],{"className":38839},[119,192,156],[59,38841,227],{"className":38842},[226],[59,38844,38846],{"className":38845},[136],[59,38847,38849],{"className":38848,"style":17341},[140],[59,38850],{},[59,38852,12941],{"className":38853},[119],[59,38855,38857,38860],{"className":38856},[119],[59,38858,12802],{"className":38859},[119,192],[59,38861,38863],{"className":38862},[196],[59,38864,38866,38897],{"className":38865},[131,132],[59,38867,38869,38894],{"className":38868},[136],[59,38870,38872,38883],{"className":38871,"style":18849},[140],[59,38873,38874,38877],{"style":17311},[59,38875],{"className":38876,"style":540},[148],[59,38878,38880],{"className":38879},[153,154,155,156],[59,38881,335],{"className":38882},[119,192,156],[59,38884,38885,38888],{"style":536},[59,38886],{"className":38887,"style":540},[148],[59,38889,38891],{"className":38890},[153,154,155,156],[59,38892,777],{"className":38893},[119,192,156],[59,38895,227],{"className":38896},[226],[59,38898,38900],{"className":38899},[136],[59,38901,38903],{"className":38902,"style":17341},[140],[59,38904],{},[59,38906,812],{"className":38907},[313],[59,38909,227],{"className":38910},[226],[59,38912,38914],{"className":38913},[136],[59,38915,38917],{"className":38916,"style":15950},[140],[59,38918],{},[59,38920],{"className":38921},[313,124],[59,38923,227],{"className":38924},[226],[59,38926,38928],{"className":38927},[136],[59,38929,38931],{"className":38930,"style":35210},[140],[59,38932],{},[15,38934,38935],{},"对应地，极大化目标时可用如下损失（最小化时取负号）：",[59,38937,38939],{"className":38938},[14350],[59,38940,38942,39110],{"className":38941},[62],[59,38943,38945],{"className":38944},[66],[68,38946,38947],{"xmlns":70,"display":9481},[72,38948,38949,39107],{},[75,38950,38951,38953,38955,38957,38959,38961,38963,38969,38983,39001,39019,39021,39029,39031,39039,39041],{},[91,38952,7816],{},[91,38954,792],{},[91,38956,12802],{},[91,38958,12802],{},[461,38960,815],{},[461,38962,18365],{},[78,38964,38965,38967],{},[81,38966,83],{},[91,38968,7322],{},[14683,38970,38971,38973,38981],{},[461,38972,1826],{},[75,38974,38975,38977,38979],{},[91,38976,335],{},[461,38978,815],{},[81,38980,83],{},[91,38982,7322],{},[14683,38984,38985,38987,38995],{},[461,38986,1826],{},[75,38988,38989,38991,38993],{},[91,38990,777],{},[461,38992,815],{},[81,38994,83],{},[88,38996,38997,38999],{},[91,38998,13242],{},[91,39000,335],{},[1822,39002,39003,39005,39011],{},[91,39004,774],{},[465,39006,39007,39009],{},[91,39008,14051],{},[461,39010,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,39012,39013,39015,39017],{},[91,39014,27510],{},[91,39016,774],{},[91,39018,5607],{},[461,39020,798],{"stretchy":797},[1822,39022,39023,39025,39027],{},[91,39024,12802],{},[91,39026,335],{},[91,39028,777],{},[461,39030,803],{"separator":106},[1822,39032,39033,39035,39037],{},[91,39034,22],{},[91,39036,335],{},[91,39038,777],{},[461,39040,812],{"stretchy":797},[78,39042,39043,39073],{},[75,39044,39045,39051,39053,39061,39063,39071],{},[88,39046,39047,39049],{},[91,39048,5604],{},[91,39050,14051],{},[461,39052,798],{"stretchy":797},[1822,39054,39055,39057,39059],{},[91,39056,22],{},[91,39058,335],{},[91,39060,777],{},[91,39062,12941],{"mathvariant":818},[1822,39064,39065,39067,39069],{},[91,39066,12802],{},[91,39068,335],{},[91,39070,777],{},[461,39072,812],{"stretchy":797},[75,39074,39075,39085,39087,39095,39097,39105],{},[88,39076,39077,39079],{},[91,39078,5604],{},[465,39080,39081,39083],{},[91,39082,14051],{},[461,39084,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,39086,798],{"stretchy":797},[1822,39088,39089,39091,39093],{},[91,39090,22],{},[91,39092,335],{},[91,39094,777],{},[91,39096,12941],{"mathvariant":818},[1822,39098,39099,39101,39103],{},[91,39100,12802],{},[91,39102,335],{},[91,39104,777],{},[461,39106,812],{"stretchy":797},[98,39108,39109],{"encoding":100},"Loss = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)}",[59,39111,39113,39134],{"className":39112,"ariaHidden":106},[105],[59,39114,39116,39119,39122,39125,39128,39131],{"className":39115},[110],[59,39117],{"className":39118,"style":14003},[114],[59,39120,7816],{"className":39121},[119,192],[59,39123,18466],{"className":39124},[119,192],[59,39126],{"className":39127,"style":500},[499],[59,39129,815],{"className":39130},[504],[59,39132],{"className":39133,"style":500},[499],[59,39135,39137,39140,39143,39205,39208,39275,39278,39382,39385,39477,39480,39531,39534,39537,39588,39591],{"className":39136},[110],[59,39138],{"className":39139,"style":31173},[114],[59,39141,18365],{"className":39142},[119],[59,39144,39146,39149,39202],{"className":39145},[119],[59,39147],{"className":39148},[123,124],[59,39150,39152],{"className":39151},[78],[59,39153,39155,39194],{"className":39154},[131,132],[59,39156,39158,39191],{"className":39157},[136],[59,39159,39161,39172,39180],{"className":39160,"style":16279},[140],[59,39162,39163,39166],{"style":15819},[59,39164],{"className":39165,"style":149},[148],[59,39167,39169],{"className":39168},[119],[59,39170,7322],{"className":39171,"style":7397},[119,192],[59,39173,39174,39177],{"style":274},[59,39175],{"className":39176,"style":149},[148],[59,39178],{"className":39179,"style":282},[281],[59,39181,39182,39185],{"style":15885},[59,39183],{"className":39184,"style":149},[148],[59,39186,39188],{"className":39187},[119],[59,39189,83],{"className":39190},[119],[59,39192,227],{"className":39193},[226],[59,39195,39197],{"className":39196},[136],[59,39198,39200],{"className":39199,"style":16319},[140],[59,39201],{},[59,39203],{"className":39204},[313,124],[59,39206],{"className":39207,"style":916},[499],[59,39209,39211],{"className":39210},[951,14238],[59,39212,39214,39267],{"className":39213},[131,132],[59,39215,39217,39264],{"className":39216},[136],[59,39218,39220,39240,39250],{"className":39219,"style":16340},[140],[59,39221,39222,39225],{"style":16343},[59,39223],{"className":39224,"style":15432},[148],[59,39226,39228],{"className":39227},[153,154,155,156],[59,39229,39231,39234,39237],{"className":39230},[119,156],[59,39232,335],{"className":39233},[119,192,156],[59,39235,815],{"className":39236},[504,156],[59,39238,83],{"className":39239},[119,156],[59,39241,39242,39245],{"style":15441},[59,39243],{"className":39244,"style":15432},[148],[59,39246,39247],{},[59,39248,1826],{"className":39249},[951,1910,15450],[59,39251,39252,39255],{"style":16374},[59,39253],{"className":39254,"style":15432},[148],[59,39256,39258],{"className":39257},[153,154,155,156],[59,39259,39261],{"className":39260},[119,156],[59,39262,7322],{"className":39263,"style":7397},[119,192,156],[59,39265,227],{"className":39266},[226],[59,39268,39270],{"className":39269},[136],[59,39271,39273],{"className":39272,"style":16393},[140],[59,39274],{},[59,39276],{"className":39277,"style":916},[499],[59,39279,39281],{"className":39280},[951,14238],[59,39282,39284,39374],{"className":39283},[131,132],[59,39285,39287,39371],{"className":39286},[136],[59,39288,39290,39310,39320],{"className":39289,"style":17149},[140],[59,39291,39292,39295],{"style":16343},[59,39293],{"className":39294,"style":15432},[148],[59,39296,39298],{"className":39297},[153,154,155,156],[59,39299,39301,39304,39307],{"className":39300},[119,156],[59,39302,777],{"className":39303},[119,192,156],[59,39305,815],{"className":39306},[504,156],[59,39308,83],{"className":39309},[119,156],[59,39311,39312,39315],{"style":15441},[59,39313],{"className":39314,"style":15432},[148],[59,39316,39317],{},[59,39318,1826],{"className":39319},[951,1910,15450],[59,39321,39322,39325],{"style":17182},[59,39323],{"className":39324,"style":15432},[148],[59,39326,39328],{"className":39327},[153,154,155,156],[59,39329,39331],{"className":39330},[119,156],[59,39332,39334,39337],{"className":39333},[119,156],[59,39335,13242],{"className":39336,"style":3185},[119,192,156],[59,39338,39340],{"className":39339},[196],[59,39341,39343,39363],{"className":39342},[131,132],[59,39344,39346,39360],{"className":39345},[136],[59,39347,39349],{"className":39348,"style":4302},[140],[59,39350,39351,39354],{"style":17212},[59,39352],{"className":39353,"style":213},[148],[59,39355,39357],{"className":39356},[153,217,218,156],[59,39358,335],{"className":39359},[119,192,156],[59,39361,227],{"className":39362},[226],[59,39364,39366],{"className":39365},[136],[59,39367,39369],{"className":39368,"style":4324},[140],[59,39370],{},[59,39372,227],{"className":39373},[226],[59,39375,39377],{"className":39376},[136],[59,39378,39380],{"className":39379,"style":16393},[140],[59,39381],{},[59,39383],{"className":39384,"style":916},[499],[59,39386,39388,39391],{"className":39387},[119],[59,39389,774],{"className":39390},[119,192],[59,39392,39394],{"className":39393},[196],[59,39395,39397,39469],{"className":39396},[131,132],[59,39398,39400,39466],{"className":39399},[136],[59,39401,39403,39446],{"className":39402,"style":24796},[140],[59,39404,39405,39408],{"style":17311},[59,39406],{"className":39407,"style":540},[148],[59,39409,39411],{"className":39410},[153,154,155,156],[59,39412,39414],{"className":39413},[119,156],[59,39415,39417,39420],{"className":39416},[119,156],[59,39418,14051],{"className":39419,"style":3210},[119,192,156],[59,39421,39423],{"className":39422},[196],[59,39424,39426],{"className":39425},[131],[59,39427,39429],{"className":39428},[136],[59,39430,39432],{"className":39431,"style":19144},[140],[59,39433,39434,39437],{"style":19147},[59,39435],{"className":39436,"style":213},[148],[59,39438,39440],{"className":39439},[153,217,218,156],[59,39441,39443],{"className":39442},[119,156],[59,39444,18984],{"className":39445},[119,156],[59,39447,39448,39451],{"style":16426},[59,39449],{"className":39450,"style":540},[148],[59,39452,39454],{"className":39453},[153,154,155,156],[59,39455,39457,39460,39463],{"className":39456},[119,156],[59,39458,27510],{"className":39459},[119,192,156],[59,39461,774],{"className":39462},[119,192,156],[59,39464,5607],{"className":39465,"style":6050},[119,192,156],[59,39467,227],{"className":39468},[226],[59,39470,39472],{"className":39471},[136],[59,39473,39475],{"className":39474,"style":17341},[140],[59,39476],{},[59,39478,798],{"className":39479},[123],[59,39481,39483,39486],{"className":39482},[119],[59,39484,12802],{"className":39485},[119,192],[59,39487,39489],{"className":39488},[196],[59,39490,39492,39523],{"className":39491},[131,132],[59,39493,39495,39520],{"className":39494},[136],[59,39496,39498,39509],{"className":39497,"style":17308},[140],[59,39499,39500,39503],{"style":17311},[59,39501],{"className":39502,"style":540},[148],[59,39504,39506],{"className":39505},[153,154,155,156],[59,39507,335],{"className":39508},[119,192,156],[59,39510,39511,39514],{"style":16426},[59,39512],{"className":39513,"style":540},[148],[59,39515,39517],{"className":39516},[153,154,155,156],[59,39518,777],{"className":39519},[119,192,156],[59,39521,227],{"className":39522},[226],[59,39524,39526],{"className":39525},[136],[59,39527,39529],{"className":39528,"style":17341},[140],[59,39530],{},[59,39532,803],{"className":39533},[912],[59,39535],{"className":39536,"style":916},[499],[59,39538,39540,39543],{"className":39539},[119],[59,39541,22],{"className":39542},[119,192],[59,39544,39546],{"className":39545},[196],[59,39547,39549,39580],{"className":39548},[131,132],[59,39550,39552,39577],{"className":39551},[136],[59,39553,39555,39566],{"className":39554,"style":17308},[140],[59,39556,39557,39560],{"style":17311},[59,39558],{"className":39559,"style":540},[148],[59,39561,39563],{"className":39562},[153,154,155,156],[59,39564,335],{"className":39565},[119,192,156],[59,39567,39568,39571],{"style":16426},[59,39569],{"className":39570,"style":540},[148],[59,39572,39574],{"className":39573},[153,154,155,156],[59,39575,777],{"className":39576},[119,192,156],[59,39578,227],{"className":39579},[226],[59,39581,39583],{"className":39582},[136],[59,39584,39586],{"className":39585,"style":17341},[140],[59,39587],{},[59,39589,812],{"className":39590},[313],[59,39592,39594,39597,39978],{"className":39593},[119],[59,39595],{"className":39596},[123,124],[59,39598,39600],{"className":39599},[78],[59,39601,39603,39970],{"className":39602},[131,132],[59,39604,39606,39967],{"className":39605},[136],[59,39607,39609,39800,39808],{"className":39608,"style":36303},[140],[59,39610,39611,39614],{"style":15819},[59,39612],{"className":39613,"style":149},[148],[59,39615,39617,39689,39692,39743,39746,39797],{"className":39616},[119],[59,39618,39620,39623],{"className":39619},[119],[59,39621,5604],{"className":39622,"style":3185},[119,192],[59,39624,39626],{"className":39625},[196],[59,39627,39629,39681],{"className":39628},[131,132],[59,39630,39632,39678],{"className":39631},[136],[59,39633,39635],{"className":39634,"style":1248},[140],[59,39636,39637,39640],{"style":7460},[59,39638],{"className":39639,"style":540},[148],[59,39641,39643],{"className":39642},[153,154,155,156],[59,39644,39646],{"className":39645},[119,156],[59,39647,39649,39652],{"className":39648},[119,156],[59,39650,14051],{"className":39651,"style":3210},[119,192,156],[59,39653,39655],{"className":39654},[196],[59,39656,39658],{"className":39657},[131],[59,39659,39661],{"className":39660},[136],[59,39662,39664],{"className":39663,"style":19144},[140],[59,39665,39666,39669],{"style":19147},[59,39667],{"className":39668,"style":213},[148],[59,39670,39672],{"className":39671},[153,217,218,156],[59,39673,39675],{"className":39674},[119,156],[59,39676,18984],{"className":39677},[119,156],[59,39679,227],{"className":39680},[226],[59,39682,39684],{"className":39683},[136],[59,39685,39687],{"className":39686,"style":1270},[140],[59,39688],{},[59,39690,798],{"className":39691},[123],[59,39693,39695,39698],{"className":39694},[119],[59,39696,22],{"className":39697},[119,192],[59,39699,39701],{"className":39700},[196],[59,39702,39704,39735],{"className":39703},[131,132],[59,39705,39707,39732],{"className":39706},[136],[59,39708,39710,39721],{"className":39709,"style":36405},[140],[59,39711,39712,39715],{"style":17311},[59,39713],{"className":39714,"style":540},[148],[59,39716,39718],{"className":39717},[153,154,155,156],[59,39719,335],{"className":39720},[119,192,156],[59,39722,39723,39726],{"style":16529},[59,39724],{"className":39725,"style":540},[148],[59,39727,39729],{"className":39728},[153,154,155,156],[59,39730,777],{"className":39731},[119,192,156],[59,39733,227],{"className":39734},[226],[59,39736,39738],{"className":39737},[136],[59,39739,39741],{"className":39740,"style":17341},[140],[59,39742],{},[59,39744,12941],{"className":39745},[119],[59,39747,39749,39752],{"className":39748},[119],[59,39750,12802],{"className":39751},[119,192],[59,39753,39755],{"className":39754},[196],[59,39756,39758,39789],{"className":39757},[131,132],[59,39759,39761,39786],{"className":39760},[136],[59,39762,39764,39775],{"className":39763,"style":36405},[140],[59,39765,39766,39769],{"style":17311},[59,39767],{"className":39768,"style":540},[148],[59,39770,39772],{"className":39771},[153,154,155,156],[59,39773,335],{"className":39774},[119,192,156],[59,39776,39777,39780],{"style":16529},[59,39778],{"className":39779,"style":540},[148],[59,39781,39783],{"className":39782},[153,154,155,156],[59,39784,777],{"className":39785},[119,192,156],[59,39787,227],{"className":39788},[226],[59,39790,39792],{"className":39791},[136],[59,39793,39795],{"className":39794,"style":17341},[140],[59,39796],{},[59,39798,812],{"className":39799},[313],[59,39801,39802,39805],{"style":274},[59,39803],{"className":39804,"style":149},[148],[59,39806],{"className":39807,"style":282},[281],[59,39809,39810,39813],{"style":15885},[59,39811],{"className":39812,"style":149},[148],[59,39814,39816,39856,39859,39910,39913,39964],{"className":39815},[119],[59,39817,39819,39822],{"className":39818},[119],[59,39820,5604],{"className":39821,"style":3185},[119,192],[59,39823,39825],{"className":39824},[196],[59,39826,39828,39848],{"className":39827},[131,132],[59,39829,39831,39845],{"className":39830},[136],[59,39832,39834],{"className":39833,"style":1248},[140],[59,39835,39836,39839],{"style":7460},[59,39837],{"className":39838,"style":540},[148],[59,39840,39842],{"className":39841},[153,154,155,156],[59,39843,14051],{"className":39844,"style":3210},[119,192,156],[59,39846,227],{"className":39847},[226],[59,39849,39851],{"className":39850},[136],[59,39852,39854],{"className":39853,"style":1270},[140],[59,39855],{},[59,39857,798],{"className":39858},[123],[59,39860,39862,39865],{"className":39861},[119],[59,39863,22],{"className":39864},[119,192],[59,39866,39868],{"className":39867},[196],[59,39869,39871,39902],{"className":39870},[131,132],[59,39872,39874,39899],{"className":39873},[136],[59,39875,39877,39888],{"className":39876,"style":18849},[140],[59,39878,39879,39882],{"style":17311},[59,39880],{"className":39881,"style":540},[148],[59,39883,39885],{"className":39884},[153,154,155,156],[59,39886,335],{"className":39887},[119,192,156],[59,39889,39890,39893],{"style":536},[59,39891],{"className":39892,"style":540},[148],[59,39894,39896],{"className":39895},[153,154,155,156],[59,39897,777],{"className":39898},[119,192,156],[59,39900,227],{"className":39901},[226],[59,39903,39905],{"className":39904},[136],[59,39906,39908],{"className":39907,"style":17341},[140],[59,39909],{},[59,39911,12941],{"className":39912},[119],[59,39914,39916,39919],{"className":39915},[119],[59,39917,12802],{"className":39918},[119,192],[59,39920,39922],{"className":39921},[196],[59,39923,39925,39956],{"className":39924},[131,132],[59,39926,39928,39953],{"className":39927},[136],[59,39929,39931,39942],{"className":39930,"style":18849},[140],[59,39932,39933,39936],{"style":17311},[59,39934],{"className":39935,"style":540},[148],[59,39937,39939],{"className":39938},[153,154,155,156],[59,39940,335],{"className":39941},[119,192,156],[59,39943,39944,39947],{"style":536},[59,39945],{"className":39946,"style":540},[148],[59,39948,39950],{"className":39949},[153,154,155,156],[59,39951,777],{"className":39952},[119,192,156],[59,39954,227],{"className":39955},[226],[59,39957,39959],{"className":39958},[136],[59,39960,39962],{"className":39961,"style":17341},[140],[59,39963],{},[59,39965,812],{"className":39966},[313],[59,39968,227],{"className":39969},[226],[59,39971,39973],{"className":39972},[136],[59,39974,39976],{"className":39975,"style":15950},[140],[59,39977],{},[59,39979],{"className":39980},[313,124],[15,39982,39983],{},"同时，你学习的目标模型不能和我们的差距过大，否则很难学到有用的经验和教训。",[39985,39986,39988],"h4",{"id":39987},"ppo-penalty在-surrogate-上加-kl-惩罚","PPO-Penalty：在 surrogate 上加 KL 惩罚",[15,39990,39991,39992,40024],{},"如何给我们训练的策略和参考的策略增加相差不能过大这个约束呢？那就是加上 KL 散度的约束。KL 散度就是衡量两个分布差异大小的指标。两个分布差异越小，KL散度越小，分布差异越大，KL散度越大。通过因子 ",[59,39993,39995,40010],{"className":39994},[62],[59,39996,39998],{"className":39997},[66],[68,39999,40000],{"xmlns":70},[72,40001,40002,40007],{},[75,40003,40004],{},[91,40005,40006],{},"β",[98,40008,40009],{"encoding":100},"\\beta",[59,40011,40013],{"className":40012,"ariaHidden":106},[105],[59,40014,40016,40020],{"className":40015},[110],[59,40017],{"className":40018,"style":40019},[114],"height:0.8889em;vertical-align:-0.1944em;",[59,40021,40006],{"className":40022,"style":40023},[119,192],"margin-right:0.05278em;"," 来调整 KL 散度影响的大小。",[59,40026,40028],{"className":40027},[14350],[59,40029,40031,40238],{"className":40030},[62],[59,40032,40034],{"className":40033},[66],[68,40035,40036],{"xmlns":70,"display":9481},[72,40037,40038,40235],{},[75,40039,40040,40042,40044,40046,40053,40055,40057,40063,40077,40095,40113,40115,40123,40125,40133,40135,40201,40203,40205,40207,40213,40215,40221,40223,40233],{},[91,40041,7816],{},[91,40043,792],{},[91,40045,12802],{},[88,40047,40048,40050],{},[91,40049,12802],{},[3277,40051,40052],{},"ppo",[461,40054,815],{},[461,40056,18365],{},[78,40058,40059,40061],{},[81,40060,83],{},[91,40062,7322],{},[14683,40064,40065,40067,40075],{},[461,40066,1826],{},[75,40068,40069,40071,40073],{},[91,40070,335],{},[461,40072,815],{},[81,40074,83],{},[91,40076,7322],{},[14683,40078,40079,40081,40089],{},[461,40080,1826],{},[75,40082,40083,40085,40087],{},[91,40084,777],{},[461,40086,815],{},[81,40088,83],{},[88,40090,40091,40093],{},[91,40092,13242],{},[91,40094,335],{},[1822,40096,40097,40099,40105],{},[91,40098,774],{},[465,40100,40101,40103],{},[91,40102,14051],{},[461,40104,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,40106,40107,40109,40111],{},[91,40108,27510],{},[91,40110,774],{},[91,40112,5607],{},[461,40114,798],{"stretchy":797},[1822,40116,40117,40119,40121],{},[91,40118,12802],{},[91,40120,335],{},[91,40122,777],{},[461,40124,803],{"separator":106},[1822,40126,40127,40129,40131],{},[91,40128,22],{},[91,40130,335],{},[91,40132,777],{},[461,40134,812],{"stretchy":797},[78,40136,40137,40167],{},[75,40138,40139,40145,40147,40155,40157,40165],{},[88,40140,40141,40143],{},[91,40142,5604],{},[91,40144,14051],{},[461,40146,798],{"stretchy":797},[1822,40148,40149,40151,40153],{},[91,40150,22],{},[91,40152,335],{},[91,40154,777],{},[91,40156,12941],{"mathvariant":818},[1822,40158,40159,40161,40163],{},[91,40160,12802],{},[91,40162,335],{},[91,40164,777],{},[461,40166,812],{"stretchy":797},[75,40168,40169,40179,40181,40189,40191,40199],{},[88,40170,40171,40173],{},[91,40172,5604],{},[465,40174,40175,40177],{},[91,40176,14051],{},[461,40178,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,40180,798],{"stretchy":797},[1822,40182,40183,40185,40187],{},[91,40184,22],{},[91,40186,335],{},[91,40188,777],{},[91,40190,12941],{"mathvariant":818},[1822,40192,40193,40195,40197],{},[91,40194,12802],{},[91,40196,335],{},[91,40198,777],{},[461,40200,812],{"stretchy":797},[461,40202,5777],{},[91,40204,40006],{},[3277,40206,27595],{},[75,40208,40209,40211],{},[91,40210,572],{"mathvariant":818},[91,40212,7816],{"mathvariant":818},[461,40214,798],{"stretchy":797},[88,40216,40217,40219],{},[91,40218,5604],{},[91,40220,14051],{},[461,40222,803],{"separator":106},[88,40224,40225,40227],{},[91,40226,5604],{},[465,40228,40229,40231],{},[91,40230,14051],{},[461,40232,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,40234,812],{"stretchy":797},[98,40236,40237],{"encoding":100},"Loss_{\\text{ppo}} = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)} + \\beta\\,\\mathrm{KL}(P_{\\theta}, P_{\\theta'})",[59,40239,40241,40311,41170],{"className":40240,"ariaHidden":106},[105],[59,40242,40244,40248,40251,40255,40302,40305,40308],{"className":40243},[110],[59,40245],{"className":40246,"style":40247},[114],"height:0.9694em;vertical-align:-0.2861em;",[59,40249,7816],{"className":40250},[119,192],[59,40252,40254],{"className":40253},[119,192],"os",[59,40256,40258,40261],{"className":40257},[119],[59,40259,12802],{"className":40260},[119,192],[59,40262,40264],{"className":40263},[196],[59,40265,40267,40293],{"className":40266},[131,132],[59,40268,40270,40290],{"className":40269},[136],[59,40271,40273],{"className":40272,"style":1322},[140],[59,40274,40275,40278],{"style":1251},[59,40276],{"className":40277,"style":540},[148],[59,40279,40281],{"className":40280},[153,154,155,156],[59,40282,40284],{"className":40283},[119,156],[59,40285,40287],{"className":40286},[119,11854,156],[59,40288,40052],{"className":40289},[119,156],[59,40291,227],{"className":40292},[226],[59,40294,40296],{"className":40295},[136],[59,40297,40300],{"className":40298,"style":40299},[140],"height:0.2861em;",[59,40301],{},[59,40303],{"className":40304,"style":500},[499],[59,40306,815],{"className":40307},[504],[59,40309],{"className":40310,"style":500},[499],[59,40312,40314,40317,40320,40382,40385,40452,40455,40559,40562,40654,40657,40708,40711,40714,40765,40768,41161,41164,41167],{"className":40313},[110],[59,40315],{"className":40316,"style":31173},[114],[59,40318,18365],{"className":40319},[119],[59,40321,40323,40326,40379],{"className":40322},[119],[59,40324],{"className":40325},[123,124],[59,40327,40329],{"className":40328},[78],[59,40330,40332,40371],{"className":40331},[131,132],[59,40333,40335,40368],{"className":40334},[136],[59,40336,40338,40349,40357],{"className":40337,"style":16279},[140],[59,40339,40340,40343],{"style":15819},[59,40341],{"className":40342,"style":149},[148],[59,40344,40346],{"className":40345},[119],[59,40347,7322],{"className":40348,"style":7397},[119,192],[59,40350,40351,40354],{"style":274},[59,40352],{"className":40353,"style":149},[148],[59,40355],{"className":40356,"style":282},[281],[59,40358,40359,40362],{"style":15885},[59,40360],{"className":40361,"style":149},[148],[59,40363,40365],{"className":40364},[119],[59,40366,83],{"className":40367},[119],[59,40369,227],{"className":40370},[226],[59,40372,40374],{"className":40373},[136],[59,40375,40377],{"className":40376,"style":16319},[140],[59,40378],{},[59,40380],{"className":40381},[313,124],[59,40383],{"className":40384,"style":916},[499],[59,40386,40388],{"className":40387},[951,14238],[59,40389,40391,40444],{"className":40390},[131,132],[59,40392,40394,40441],{"className":40393},[136],[59,40395,40397,40417,40427],{"className":40396,"style":16340},[140],[59,40398,40399,40402],{"style":16343},[59,40400],{"className":40401,"style":15432},[148],[59,40403,40405],{"className":40404},[153,154,155,156],[59,40406,40408,40411,40414],{"className":40407},[119,156],[59,40409,335],{"className":40410},[119,192,156],[59,40412,815],{"className":40413},[504,156],[59,40415,83],{"className":40416},[119,156],[59,40418,40419,40422],{"style":15441},[59,40420],{"className":40421,"style":15432},[148],[59,40423,40424],{},[59,40425,1826],{"className":40426},[951,1910,15450],[59,40428,40429,40432],{"style":16374},[59,40430],{"className":40431,"style":15432},[148],[59,40433,40435],{"className":40434},[153,154,155,156],[59,40436,40438],{"className":40437},[119,156],[59,40439,7322],{"className":40440,"style":7397},[119,192,156],[59,40442,227],{"className":40443},[226],[59,40445,40447],{"className":40446},[136],[59,40448,40450],{"className":40449,"style":16393},[140],[59,40451],{},[59,40453],{"className":40454,"style":916},[499],[59,40456,40458],{"className":40457},[951,14238],[59,40459,40461,40551],{"className":40460},[131,132],[59,40462,40464,40548],{"className":40463},[136],[59,40465,40467,40487,40497],{"className":40466,"style":17149},[140],[59,40468,40469,40472],{"style":16343},[59,40470],{"className":40471,"style":15432},[148],[59,40473,40475],{"className":40474},[153,154,155,156],[59,40476,40478,40481,40484],{"className":40477},[119,156],[59,40479,777],{"className":40480},[119,192,156],[59,40482,815],{"className":40483},[504,156],[59,40485,83],{"className":40486},[119,156],[59,40488,40489,40492],{"style":15441},[59,40490],{"className":40491,"style":15432},[148],[59,40493,40494],{},[59,40495,1826],{"className":40496},[951,1910,15450],[59,40498,40499,40502],{"style":17182},[59,40500],{"className":40501,"style":15432},[148],[59,40503,40505],{"className":40504},[153,154,155,156],[59,40506,40508],{"className":40507},[119,156],[59,40509,40511,40514],{"className":40510},[119,156],[59,40512,13242],{"className":40513,"style":3185},[119,192,156],[59,40515,40517],{"className":40516},[196],[59,40518,40520,40540],{"className":40519},[131,132],[59,40521,40523,40537],{"className":40522},[136],[59,40524,40526],{"className":40525,"style":4302},[140],[59,40527,40528,40531],{"style":17212},[59,40529],{"className":40530,"style":213},[148],[59,40532,40534],{"className":40533},[153,217,218,156],[59,40535,335],{"className":40536},[119,192,156],[59,40538,227],{"className":40539},[226],[59,40541,40543],{"className":40542},[136],[59,40544,40546],{"className":40545,"style":4324},[140],[59,40547],{},[59,40549,227],{"className":40550},[226],[59,40552,40554],{"className":40553},[136],[59,40555,40557],{"className":40556,"style":16393},[140],[59,40558],{},[59,40560],{"className":40561,"style":916},[499],[59,40563,40565,40568],{"className":40564},[119],[59,40566,774],{"className":40567},[119,192],[59,40569,40571],{"className":40570},[196],[59,40572,40574,40646],{"className":40573},[131,132],[59,40575,40577,40643],{"className":40576},[136],[59,40578,40580,40623],{"className":40579,"style":24796},[140],[59,40581,40582,40585],{"style":17311},[59,40583],{"className":40584,"style":540},[148],[59,40586,40588],{"className":40587},[153,154,155,156],[59,40589,40591],{"className":40590},[119,156],[59,40592,40594,40597],{"className":40593},[119,156],[59,40595,14051],{"className":40596,"style":3210},[119,192,156],[59,40598,40600],{"className":40599},[196],[59,40601,40603],{"className":40602},[131],[59,40604,40606],{"className":40605},[136],[59,40607,40609],{"className":40608,"style":19144},[140],[59,40610,40611,40614],{"style":19147},[59,40612],{"className":40613,"style":213},[148],[59,40615,40617],{"className":40616},[153,217,218,156],[59,40618,40620],{"className":40619},[119,156],[59,40621,18984],{"className":40622},[119,156],[59,40624,40625,40628],{"style":16426},[59,40626],{"className":40627,"style":540},[148],[59,40629,40631],{"className":40630},[153,154,155,156],[59,40632,40634,40637,40640],{"className":40633},[119,156],[59,40635,27510],{"className":40636},[119,192,156],[59,40638,774],{"className":40639},[119,192,156],[59,40641,5607],{"className":40642,"style":6050},[119,192,156],[59,40644,227],{"className":40645},[226],[59,40647,40649],{"className":40648},[136],[59,40650,40652],{"className":40651,"style":17341},[140],[59,40653],{},[59,40655,798],{"className":40656},[123],[59,40658,40660,40663],{"className":40659},[119],[59,40661,12802],{"className":40662},[119,192],[59,40664,40666],{"className":40665},[196],[59,40667,40669,40700],{"className":40668},[131,132],[59,40670,40672,40697],{"className":40671},[136],[59,40673,40675,40686],{"className":40674,"style":17308},[140],[59,40676,40677,40680],{"style":17311},[59,40678],{"className":40679,"style":540},[148],[59,40681,40683],{"className":40682},[153,154,155,156],[59,40684,335],{"className":40685},[119,192,156],[59,40687,40688,40691],{"style":16426},[59,40689],{"className":40690,"style":540},[148],[59,40692,40694],{"className":40693},[153,154,155,156],[59,40695,777],{"className":40696},[119,192,156],[59,40698,227],{"className":40699},[226],[59,40701,40703],{"className":40702},[136],[59,40704,40706],{"className":40705,"style":17341},[140],[59,40707],{},[59,40709,803],{"className":40710},[912],[59,40712],{"className":40713,"style":916},[499],[59,40715,40717,40720],{"className":40716},[119],[59,40718,22],{"className":40719},[119,192],[59,40721,40723],{"className":40722},[196],[59,40724,40726,40757],{"className":40725},[131,132],[59,40727,40729,40754],{"className":40728},[136],[59,40730,40732,40743],{"className":40731,"style":17308},[140],[59,40733,40734,40737],{"style":17311},[59,40735],{"className":40736,"style":540},[148],[59,40738,40740],{"className":40739},[153,154,155,156],[59,40741,335],{"className":40742},[119,192,156],[59,40744,40745,40748],{"style":16426},[59,40746],{"className":40747,"style":540},[148],[59,40749,40751],{"className":40750},[153,154,155,156],[59,40752,777],{"className":40753},[119,192,156],[59,40755,227],{"className":40756},[226],[59,40758,40760],{"className":40759},[136],[59,40761,40763],{"className":40762,"style":17341},[140],[59,40764],{},[59,40766,812],{"className":40767},[313],[59,40769,40771,40774,41158],{"className":40770},[119],[59,40772],{"className":40773},[123,124],[59,40775,40777],{"className":40776},[78],[59,40778,40780,41150],{"className":40779},[131,132],[59,40781,40783,41147],{"className":40782},[136],[59,40784,40786,40977,40985],{"className":40785,"style":36303},[140],[59,40787,40788,40791],{"style":15819},[59,40789],{"className":40790,"style":149},[148],[59,40792,40794,40866,40869,40920,40923,40974],{"className":40793},[119],[59,40795,40797,40800],{"className":40796},[119],[59,40798,5604],{"className":40799,"style":3185},[119,192],[59,40801,40803],{"className":40802},[196],[59,40804,40806,40858],{"className":40805},[131,132],[59,40807,40809,40855],{"className":40808},[136],[59,40810,40812],{"className":40811,"style":1248},[140],[59,40813,40814,40817],{"style":7460},[59,40815],{"className":40816,"style":540},[148],[59,40818,40820],{"className":40819},[153,154,155,156],[59,40821,40823],{"className":40822},[119,156],[59,40824,40826,40829],{"className":40825},[119,156],[59,40827,14051],{"className":40828,"style":3210},[119,192,156],[59,40830,40832],{"className":40831},[196],[59,40833,40835],{"className":40834},[131],[59,40836,40838],{"className":40837},[136],[59,40839,40841],{"className":40840,"style":19144},[140],[59,40842,40843,40846],{"style":19147},[59,40844],{"className":40845,"style":213},[148],[59,40847,40849],{"className":40848},[153,217,218,156],[59,40850,40852],{"className":40851},[119,156],[59,40853,18984],{"className":40854},[119,156],[59,40856,227],{"className":40857},[226],[59,40859,40861],{"className":40860},[136],[59,40862,40864],{"className":40863,"style":1270},[140],[59,40865],{},[59,40867,798],{"className":40868},[123],[59,40870,40872,40875],{"className":40871},[119],[59,40873,22],{"className":40874},[119,192],[59,40876,40878],{"className":40877},[196],[59,40879,40881,40912],{"className":40880},[131,132],[59,40882,40884,40909],{"className":40883},[136],[59,40885,40887,40898],{"className":40886,"style":36405},[140],[59,40888,40889,40892],{"style":17311},[59,40890],{"className":40891,"style":540},[148],[59,40893,40895],{"className":40894},[153,154,155,156],[59,40896,335],{"className":40897},[119,192,156],[59,40899,40900,40903],{"style":16529},[59,40901],{"className":40902,"style":540},[148],[59,40904,40906],{"className":40905},[153,154,155,156],[59,40907,777],{"className":40908},[119,192,156],[59,40910,227],{"className":40911},[226],[59,40913,40915],{"className":40914},[136],[59,40916,40918],{"className":40917,"style":17341},[140],[59,40919],{},[59,40921,12941],{"className":40922},[119],[59,40924,40926,40929],{"className":40925},[119],[59,40927,12802],{"className":40928},[119,192],[59,40930,40932],{"className":40931},[196],[59,40933,40935,40966],{"className":40934},[131,132],[59,40936,40938,40963],{"className":40937},[136],[59,40939,40941,40952],{"className":40940,"style":36405},[140],[59,40942,40943,40946],{"style":17311},[59,40944],{"className":40945,"style":540},[148],[59,40947,40949],{"className":40948},[153,154,155,156],[59,40950,335],{"className":40951},[119,192,156],[59,40953,40954,40957],{"style":16529},[59,40955],{"className":40956,"style":540},[148],[59,40958,40960],{"className":40959},[153,154,155,156],[59,40961,777],{"className":40962},[119,192,156],[59,40964,227],{"className":40965},[226],[59,40967,40969],{"className":40968},[136],[59,40970,40972],{"className":40971,"style":17341},[140],[59,40973],{},[59,40975,812],{"className":40976},[313],[59,40978,40979,40982],{"style":274},[59,40980],{"className":40981,"style":149},[148],[59,40983],{"className":40984,"style":282},[281],[59,40986,40987,40990],{"style":15885},[59,40988],{"className":40989,"style":149},[148],[59,40991,40993,41036,41039,41090,41093,41144],{"className":40992},[119],[59,40994,40996,40999],{"className":40995},[119],[59,40997,5604],{"className":40998,"style":3185},[119,192],[59,41000,41002],{"className":41001},[196],[59,41003,41005,41028],{"className":41004},[131,132],[59,41006,41008,41025],{"className":41007},[136],[59,41009,41011],{"className":41010,"style":1248},[140],[59,41012,41013,41016],{"style":7460},[59,41014],{"className":41015,"style":540},[148],[59,41017,41019],{"className":41018},[153,154,155,156],[59,41020,41022],{"className":41021},[119,156],[59,41023,14051],{"className":41024,"style":3210},[119,192,156],[59,41026,227],{"className":41027},[226],[59,41029,41031],{"className":41030},[136],[59,41032,41034],{"className":41033,"style":1270},[140],[59,41035],{},[59,41037,798],{"className":41038},[123],[59,41040,41042,41045],{"className":41041},[119],[59,41043,22],{"className":41044},[119,192],[59,41046,41048],{"className":41047},[196],[59,41049,41051,41082],{"className":41050},[131,132],[59,41052,41054,41079],{"className":41053},[136],[59,41055,41057,41068],{"className":41056,"style":18849},[140],[59,41058,41059,41062],{"style":17311},[59,41060],{"className":41061,"style":540},[148],[59,41063,41065],{"className":41064},[153,154,155,156],[59,41066,335],{"className":41067},[119,192,156],[59,41069,41070,41073],{"style":536},[59,41071],{"className":41072,"style":540},[148],[59,41074,41076],{"className":41075},[153,154,155,156],[59,41077,777],{"className":41078},[119,192,156],[59,41080,227],{"className":41081},[226],[59,41083,41085],{"className":41084},[136],[59,41086,41088],{"className":41087,"style":17341},[140],[59,41089],{},[59,41091,12941],{"className":41092},[119],[59,41094,41096,41099],{"className":41095},[119],[59,41097,12802],{"className":41098},[119,192],[59,41100,41102],{"className":41101},[196],[59,41103,41105,41136],{"className":41104},[131,132],[59,41106,41108,41133],{"className":41107},[136],[59,41109,41111,41122],{"className":41110,"style":18849},[140],[59,41112,41113,41116],{"style":17311},[59,41114],{"className":41115,"style":540},[148],[59,41117,41119],{"className":41118},[153,154,155,156],[59,41120,335],{"className":41121},[119,192,156],[59,41123,41124,41127],{"style":536},[59,41125],{"className":41126,"style":540},[148],[59,41128,41130],{"className":41129},[153,154,155,156],[59,41131,777],{"className":41132},[119,192,156],[59,41134,227],{"className":41135},[226],[59,41137,41139],{"className":41138},[136],[59,41140,41142],{"className":41141,"style":17341},[140],[59,41143],{},[59,41145,812],{"className":41146},[313],[59,41148,227],{"className":41149},[226],[59,41151,41153],{"className":41152},[136],[59,41154,41156],{"className":41155,"style":15950},[140],[59,41157],{},[59,41159],{"className":41160},[313,124],[59,41162],{"className":41163,"style":1872},[499],[59,41165,5777],{"className":41166},[553],[59,41168],{"className":41169,"style":1872},[499],[59,41171,41173,41176,41179,41182,41189,41192,41235,41238,41241,41313],{"className":41172},[110],[59,41174],{"className":41175,"style":877},[114],[59,41177,40006],{"className":41178,"style":40023},[119,192],[59,41180],{"className":41181,"style":916},[499],[59,41183,41185],{"className":41184},[119],[59,41186,41188],{"className":41187},[119,955],"KL",[59,41190,798],{"className":41191},[123],[59,41193,41195,41198],{"className":41194},[119],[59,41196,5604],{"className":41197,"style":3185},[119,192],[59,41199,41201],{"className":41200},[196],[59,41202,41204,41227],{"className":41203},[131,132],[59,41205,41207,41224],{"className":41206},[136],[59,41208,41210],{"className":41209,"style":1248},[140],[59,41211,41212,41215],{"style":7460},[59,41213],{"className":41214,"style":540},[148],[59,41216,41218],{"className":41217},[153,154,155,156],[59,41219,41221],{"className":41220},[119,156],[59,41222,14051],{"className":41223,"style":3210},[119,192,156],[59,41225,227],{"className":41226},[226],[59,41228,41230],{"className":41229},[136],[59,41231,41233],{"className":41232,"style":1270},[140],[59,41234],{},[59,41236,803],{"className":41237},[912],[59,41239],{"className":41240,"style":916},[499],[59,41242,41244,41247],{"className":41243},[119],[59,41245,5604],{"className":41246,"style":3185},[119,192],[59,41248,41250],{"className":41249},[196],[59,41251,41253,41305],{"className":41252},[131,132],[59,41254,41256,41302],{"className":41255},[136],[59,41257,41259],{"className":41258,"style":1248},[140],[59,41260,41261,41264],{"style":7460},[59,41262],{"className":41263,"style":540},[148],[59,41265,41267],{"className":41266},[153,154,155,156],[59,41268,41270],{"className":41269},[119,156],[59,41271,41273,41276],{"className":41272},[119,156],[59,41274,14051],{"className":41275,"style":3210},[119,192,156],[59,41277,41279],{"className":41278},[196],[59,41280,41282],{"className":41281},[131],[59,41283,41285],{"className":41284},[136],[59,41286,41288],{"className":41287,"style":19144},[140],[59,41289,41290,41293],{"style":19147},[59,41291],{"className":41292,"style":213},[148],[59,41294,41296],{"className":41295},[153,217,218,156],[59,41297,41299],{"className":41298},[119,156],[59,41300,18984],{"className":41301},[119,156],[59,41303,227],{"className":41304},[226],[59,41306,41308],{"className":41307},[136],[59,41309,41311],{"className":41310,"style":1270},[140],[59,41312],{},[59,41314,812],{"className":41315},[313],[39985,41317,41319],{"id":41318},"ppo-clip截断重要性采样比","PPO-Clip：截断重要性采样比",[15,41321,41322],{},"PPO 还有一种实现，是通过截断函数来替代 KL 散度，防止训练的策略和参考的策略偏差过大，可以看出它是由红色和蓝色两个部分构成，取两个部分的最小值。",[15,41324,41325,41326,41826,41827,6761,41880,41931],{},"其中红色部分是原始公式，蓝色部分是一个截断函数，截断函数内由三个部分组成。如果 ",[59,41327,41329,41407],{"className":41328},[62],[59,41330,41332],{"className":41331},[66],[68,41333,41334],{"xmlns":70},[72,41335,41336,41404],{},[75,41337,41338],{},[78,41339,41340,41370],{},[75,41341,41342,41348,41350,41358,41360,41368],{},[88,41343,41344,41346],{},[91,41345,5604],{},[91,41347,14051],{},[461,41349,798],{"stretchy":797},[1822,41351,41352,41354,41356],{},[91,41353,22],{},[91,41355,335],{},[91,41357,777],{},[91,41359,12941],{"mathvariant":818},[1822,41361,41362,41364,41366],{},[91,41363,12802],{},[91,41365,335],{},[91,41367,777],{},[461,41369,812],{"stretchy":797},[75,41371,41372,41382,41384,41392,41394,41402],{},[88,41373,41374,41376],{},[91,41375,5604],{},[465,41377,41378,41380],{},[91,41379,14051],{},[461,41381,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,41383,798],{"stretchy":797},[1822,41385,41386,41388,41390],{},[91,41387,22],{},[91,41389,335],{},[91,41391,777],{},[91,41393,12941],{"mathvariant":818},[1822,41395,41396,41398,41400],{},[91,41397,12802],{},[91,41399,335],{},[91,41401,777],{},[461,41403,812],{"stretchy":797},[98,41405,41406],{"encoding":100},"\\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)}",[59,41408,41410],{"className":41409,"ariaHidden":106},[105],[59,41411,41413,41417],{"className":41412},[110],[59,41414],{"className":41415,"style":41416},[114],"height:1.6646em;vertical-align:-0.5452em;",[59,41418,41420,41423,41823],{"className":41419},[119],[59,41421],{"className":41422},[123,124],[59,41424,41426],{"className":41425},[78],[59,41427,41429,41814],{"className":41428},[131,132],[59,41430,41432,41811],{"className":41431},[136],[59,41433,41436,41636,41644],{"className":41434,"style":41435},[140],"height:1.1194em;",[59,41437,41438,41441],{"style":18506},[59,41439],{"className":41440,"style":149},[148],[59,41442,41444],{"className":41443},[153,154,155,156],[59,41445,41447,41522,41525,41579,41582,41633],{"className":41446},[119,156],[59,41448,41450,41453],{"className":41449},[119,156],[59,41451,5604],{"className":41452,"style":3185},[119,192,156],[59,41454,41456],{"className":41455},[196],[59,41457,41459,41513],{"className":41458},[131,132],[59,41460,41462,41510],{"className":41461},[136],[59,41463,41465],{"className":41464,"style":206},[140],[59,41466,41468,41472],{"style":41467},"top:-2.3448em;margin-left:-0.1389em;margin-right:0.0714em;",[59,41469],{"className":41470,"style":41471},[148],"height:2.6068em;",[59,41473,41475],{"className":41474},[153,217,218,156],[59,41476,41478],{"className":41477},[119,156],[59,41479,41481,41484],{"className":41480},[119,156],[59,41482,14051],{"className":41483,"style":3210},[119,192,156],[59,41485,41487],{"className":41486},[196],[59,41488,41490],{"className":41489},[131],[59,41491,41493],{"className":41492},[136],[59,41494,41497],{"className":41495,"style":41496},[140],"height:0.8496em;",[59,41498,41500,41504],{"style":41499},"top:-2.8496em;margin-right:0.1em;",[59,41501],{"className":41502,"style":41503},[148],"height:2.5556em;",[59,41505,41507],{"className":41506},[119,156],[59,41508,18984],{"className":41509},[119,156],[59,41511,227],{"className":41512},[226],[59,41514,41516],{"className":41515},[136],[59,41517,41520],{"className":41518,"style":41519},[140],"height:0.262em;",[59,41521],{},[59,41523,798],{"className":41524},[123,156],[59,41526,41528,41531],{"className":41527},[119,156],[59,41529,22],{"className":41530},[119,192,156],[59,41532,41534],{"className":41533},[196],[59,41535,41537,41570],{"className":41536},[131,132],[59,41538,41540,41567],{"className":41539},[136],[59,41541,41544,41556],{"className":41542,"style":41543},[140],"height:0.7253em;",[59,41545,41547,41550],{"style":41546},"top:-2.214em;margin-left:0em;margin-right:0.0714em;",[59,41548],{"className":41549,"style":213},[148],[59,41551,41553],{"className":41552},[153,217,218,156],[59,41554,335],{"className":41555},[119,192,156],[59,41557,41558,41561],{"style":19147},[59,41559],{"className":41560,"style":213},[148],[59,41562,41564],{"className":41563},[153,217,218,156],[59,41565,777],{"className":41566},[119,192,156],[59,41568,227],{"className":41569},[226],[59,41571,41573],{"className":41572},[136],[59,41574,41577],{"className":41575,"style":41576},[140],"height:0.286em;",[59,41578],{},[59,41580,12941],{"className":41581},[119,156],[59,41583,41585,41588],{"className":41584},[119,156],[59,41586,12802],{"className":41587},[119,192,156],[59,41589,41591],{"className":41590},[196],[59,41592,41594,41625],{"className":41593},[131,132],[59,41595,41597,41622],{"className":41596},[136],[59,41598,41600,41611],{"className":41599,"style":41543},[140],[59,41601,41602,41605],{"style":41546},[59,41603],{"className":41604,"style":213},[148],[59,41606,41608],{"className":41607},[153,217,218,156],[59,41609,335],{"className":41610},[119,192,156],[59,41612,41613,41616],{"style":19147},[59,41614],{"className":41615,"style":213},[148],[59,41617,41619],{"className":41618},[153,217,218,156],[59,41620,777],{"className":41621},[119,192,156],[59,41623,227],{"className":41624},[226],[59,41626,41628],{"className":41627},[136],[59,41629,41631],{"className":41630,"style":41576},[140],[59,41632],{},[59,41634,812],{"className":41635},[313,156],[59,41637,41638,41641],{"style":274},[59,41639],{"className":41640,"style":149},[148],[59,41642],{"className":41643,"style":282},[281],[59,41645,41647,41650],{"style":41646},"top:-3.5102em;",[59,41648],{"className":41649,"style":149},[148],[59,41651,41653],{"className":41652},[153,154,155,156],[59,41654,41656,41699,41702,41754,41757,41808],{"className":41655},[119,156],[59,41657,41659,41662],{"className":41658},[119,156],[59,41660,5604],{"className":41661,"style":3185},[119,192,156],[59,41663,41665],{"className":41664},[196],[59,41666,41668,41691],{"className":41667},[131,132],[59,41669,41671,41688],{"className":41670},[136],[59,41672,41674],{"className":41673,"style":206},[140],[59,41675,41676,41679],{"style":14175},[59,41677],{"className":41678,"style":213},[148],[59,41680,41682],{"className":41681},[153,217,218,156],[59,41683,41685],{"className":41684},[119,156],[59,41686,14051],{"className":41687,"style":3210},[119,192,156],[59,41689,227],{"className":41690},[226],[59,41692,41694],{"className":41693},[136],[59,41695,41697],{"className":41696,"style":234},[140],[59,41698],{},[59,41700,798],{"className":41701},[123,156],[59,41703,41705,41708],{"className":41704},[119,156],[59,41706,22],{"className":41707},[119,192,156],[59,41709,41711],{"className":41710},[196],[59,41712,41714,41746],{"className":41713},[131,132],[59,41715,41717,41743],{"className":41716},[136],[59,41718,41721,41732],{"className":41719,"style":41720},[140],"height:0.8703em;",[59,41722,41723,41726],{"style":41546},[59,41724],{"className":41725,"style":213},[148],[59,41727,41729],{"className":41728},[153,217,218,156],[59,41730,335],{"className":41731},[119,192,156],[59,41733,41734,41737],{"style":1098},[59,41735],{"className":41736,"style":213},[148],[59,41738,41740],{"className":41739},[153,217,218,156],[59,41741,777],{"className":41742},[119,192,156],[59,41744,227],{"className":41745},[226],[59,41747,41749],{"className":41748},[136],[59,41750,41752],{"className":41751,"style":41576},[140],[59,41753],{},[59,41755,12941],{"className":41756},[119,156],[59,41758,41760,41763],{"className":41759},[119,156],[59,41761,12802],{"className":41762},[119,192,156],[59,41764,41766],{"className":41765},[196],[59,41767,41769,41800],{"className":41768},[131,132],[59,41770,41772,41797],{"className":41771},[136],[59,41773,41775,41786],{"className":41774,"style":41720},[140],[59,41776,41777,41780],{"style":41546},[59,41778],{"className":41779,"style":213},[148],[59,41781,41783],{"className":41782},[153,217,218,156],[59,41784,335],{"className":41785},[119,192,156],[59,41787,41788,41791],{"style":1098},[59,41789],{"className":41790,"style":213},[148],[59,41792,41794],{"className":41793},[153,217,218,156],[59,41795,777],{"className":41796},[119,192,156],[59,41798,227],{"className":41799},[226],[59,41801,41803],{"className":41802},[136],[59,41804,41806],{"className":41805,"style":41576},[140],[59,41807],{},[59,41809,812],{"className":41810},[313,156],[59,41812,227],{"className":41813},[226],[59,41815,41817],{"className":41816},[136],[59,41818,41821],{"className":41819,"style":41820},[140],"height:0.5452em;",[59,41822],{},[59,41824],{"className":41825},[313,124]," 的值介于 ",[59,41828,41830,41849],{"className":41829},[62],[59,41831,41833],{"className":41832},[66],[68,41834,41835],{"xmlns":70},[72,41836,41837,41846],{},[75,41838,41839,41841,41843],{},[81,41840,83],{},[461,41842,18365],{},[91,41844,41845],{},"ϵ",[98,41847,41848],{"encoding":100},"1-\\epsilon",[59,41850,41852,41871],{"className":41851,"ariaHidden":106},[105],[59,41853,41855,41859,41862,41865,41868],{"className":41854},[110],[59,41856],{"className":41857,"style":41858},[114],"height:0.7278em;vertical-align:-0.0833em;",[59,41860,83],{"className":41861},[119],[59,41863],{"className":41864,"style":1872},[499],[59,41866,18365],{"className":41867},[553],[59,41869],{"className":41870,"style":1872},[499],[59,41872,41874,41877],{"className":41873},[110],[59,41875],{"className":41876,"style":347},[114],[59,41878,41845],{"className":41879},[119,192],[59,41881,41883,41901],{"className":41882},[62],[59,41884,41886],{"className":41885},[66],[68,41887,41888],{"xmlns":70},[72,41889,41890,41898],{},[75,41891,41892,41894,41896],{},[81,41893,83],{},[461,41895,5777],{},[91,41897,41845],{},[98,41899,41900],{"encoding":100},"1+\\epsilon",[59,41902,41904,41922],{"className":41903,"ariaHidden":106},[105],[59,41905,41907,41910,41913,41916,41919],{"className":41906},[110],[59,41908],{"className":41909,"style":41858},[114],[59,41911,83],{"className":41912},[119],[59,41914],{"className":41915,"style":1872},[499],[59,41917,5777],{"className":41918},[553],[59,41920],{"className":41921,"style":1872},[499],[59,41923,41925,41928],{"className":41924},[110],[59,41926],{"className":41927,"style":347},[114],[59,41929,41845],{"className":41930},[119,192]," 之间，就返回原值，否则返回其中更接近的一个。这就限制的参考模型和训练模型差异的值不能太大。",[59,41933,41935],{"className":41934},[14350],[59,41936,41938,42274],{"className":41937},[62],[59,41939,41941],{"className":41940},[66],[68,41942,41943],{"xmlns":70,"display":9481},[72,41944,41945,42271],{},[75,41946,41947,41949,41951,41953,41960,41962,41964,41970,41984,42002,42005,42007],{},[91,41948,7816],{},[91,41950,792],{},[91,41952,12802],{},[88,41954,41955,41957],{},[91,41956,12802],{},[3277,41958,41959],{},"ppo2",[461,41961,815],{},[461,41963,18365],{},[78,41965,41966,41968],{},[81,41967,83],{},[91,41969,7322],{},[14683,41971,41972,41974,41982],{},[461,41973,1826],{},[75,41975,41976,41978,41980],{},[91,41977,335],{},[461,41979,815],{},[81,41981,83],{},[91,41983,7322],{},[14683,41985,41986,41988,41996],{},[461,41987,1826],{},[75,41989,41990,41992,41994],{},[91,41991,777],{},[461,41993,815],{},[81,41995,83],{},[88,41997,41998,42000],{},[91,41999,13242],{},[91,42001,335],{},[91,42003,42004],{},"min",[461,42006,822],{},[75,42008,42009,42011,42120,42122,42124,42269],{},[461,42010,798],{"fence":106},[14374,42012,42014,42032,42034,42042,42044,42052,42054],{"mathcolor":42013},"red",[1822,42015,42016,42018,42024],{},[91,42017,774],{},[465,42019,42020,42022],{},[91,42021,14051],{},[461,42023,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,42025,42026,42028,42030],{},[91,42027,27510],{},[91,42029,774],{},[91,42031,5607],{},[461,42033,798],{"stretchy":797},[1822,42035,42036,42038,42040],{},[91,42037,12802],{},[91,42039,335],{},[91,42041,777],{},[461,42043,803],{"separator":106},[1822,42045,42046,42048,42050],{},[91,42047,22],{},[91,42049,335],{},[91,42051,777],{},[461,42053,812],{"stretchy":797},[78,42055,42056,42086],{},[75,42057,42058,42064,42066,42074,42076,42084],{},[88,42059,42060,42062],{},[91,42061,5604],{},[91,42063,14051],{},[461,42065,798],{"stretchy":797},[1822,42067,42068,42070,42072],{},[91,42069,22],{},[91,42071,335],{},[91,42073,777],{},[91,42075,12941],{"mathvariant":818},[1822,42077,42078,42080,42082],{},[91,42079,12802],{},[91,42081,335],{},[91,42083,777],{},[461,42085,812],{"stretchy":797},[75,42087,42088,42098,42100,42108,42110,42118],{},[88,42089,42090,42092],{},[91,42091,5604],{},[465,42093,42094,42096],{},[91,42095,14051],{},[461,42097,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,42099,798],{"stretchy":797},[1822,42101,42102,42104,42106],{},[91,42103,22],{},[91,42105,335],{},[91,42107,777],{},[91,42109,12941],{"mathvariant":818},[1822,42111,42112,42114,42116],{},[91,42113,12802],{},[91,42115,335],{},[91,42117,777],{},[461,42119,812],{"stretchy":797},[461,42121,803],{"separator":106},[3277,42123,27595],{},[14374,42125,42127,42137,42229,42247,42249,42257,42259,42267],{"mathcolor":42126},"skyblue",[75,42128,42129,42131,42133,42135],{},[91,42130,2944],{"mathvariant":818},[91,42132,2903],{"mathvariant":818},[91,42134,789],{"mathvariant":818},[91,42136,15],{"mathvariant":818},[75,42138,42139,42141,42207,42209,42211,42213,42215,42217,42219,42221,42223,42225,42227],{},[461,42140,798],{"fence":106},[78,42142,42143,42173],{},[75,42144,42145,42151,42153,42161,42163,42171],{},[88,42146,42147,42149],{},[91,42148,5604],{},[91,42150,14051],{},[461,42152,798],{"stretchy":797},[1822,42154,42155,42157,42159],{},[91,42156,22],{},[91,42158,335],{},[91,42160,777],{},[91,42162,12941],{"mathvariant":818},[1822,42164,42165,42167,42169],{},[91,42166,12802],{},[91,42168,335],{},[91,42170,777],{},[461,42172,812],{"stretchy":797},[75,42174,42175,42185,42187,42195,42197,42205],{},[88,42176,42177,42179],{},[91,42178,5604],{},[465,42180,42181,42183],{},[91,42182,14051],{},[461,42184,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[461,42186,798],{"stretchy":797},[1822,42188,42189,42191,42193],{},[91,42190,22],{},[91,42192,335],{},[91,42194,777],{},[91,42196,12941],{"mathvariant":818},[1822,42198,42199,42201,42203],{},[91,42200,12802],{},[91,42202,335],{},[91,42204,777],{},[461,42206,812],{"stretchy":797},[461,42208,803],{"separator":106},[3277,42210,27595],{},[81,42212,83],{},[461,42214,18365],{},[91,42216,41845],{},[461,42218,803],{"separator":106},[3277,42220,27595],{},[81,42222,83],{},[461,42224,5777],{},[91,42226,41845],{},[461,42228,812],{"fence":106},[1822,42230,42231,42233,42239],{},[91,42232,774],{},[465,42234,42235,42237],{},[91,42236,14051],{},[461,42238,18984],{"mathvariant":818,"lspace":14366,"rspace":14366},[75,42240,42241,42243,42245],{},[91,42242,27510],{},[91,42244,774],{},[91,42246,5607],{},[461,42248,798],{"stretchy":797},[1822,42250,42251,42253,42255],{},[91,42252,12802],{},[91,42254,335],{},[91,42256,777],{},[461,42258,803],{"separator":106},[1822,42260,42261,42263,42265],{},[91,42262,22],{},[91,42264,335],{},[91,42266,777],{},[461,42268,812],{"stretchy":797},[461,42270,812],{"fence":106},[98,42272,42273],{"encoding":100},"Loss_{\\text{ppo2}} = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} \\min\\left(\\textcolor{red}{A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)}},\\,\\textcolor{skyblue}{\\mathrm{clip}\\left(\\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)},\\, 1-\\epsilon,\\, 1+\\epsilon\\right) A_{\\theta'}^{GAE}(s_n^t, a_n^t)}\\right)",[59,42275,42277,42344],{"className":42276,"ariaHidden":106},[105],[59,42278,42280,42283,42286,42289,42335,42338,42341],{"className":42279},[110],[59,42281],{"className":42282,"style":40247},[114],[59,42284,7816],{"className":42285},[119,192],[59,42287,40254],{"className":42288},[119,192],[59,42290,42292,42295],{"className":42291},[119],[59,42293,12802],{"className":42294},[119,192],[59,42296,42298],{"className":42297},[196],[59,42299,42301,42327],{"className":42300},[131,132],[59,42302,42304,42324],{"className":42303},[136],[59,42305,42307],{"className":42306,"style":3091},[140],[59,42308,42309,42312],{"style":1251},[59,42310],{"className":42311,"style":540},[148],[59,42313,42315],{"className":42314},[153,154,155,156],[59,42316,42318],{"className":42317},[119,156],[59,42319,42321],{"className":42320},[119,11854,156],[59,42322,41959],{"className":42323},[119,156],[59,42325,227],{"className":42326},[226],[59,42328,42330],{"className":42329},[136],[59,42331,42333],{"className":42332,"style":40299},[140],[59,42334],{},[59,42336],{"className":42337,"style":500},[499],[59,42339,815],{"className":42340},[504],[59,42342],{"className":42343,"style":500},[499],[59,42345,42347,42350,42353,42415,42418,42485,42488,42592,42595,42598,42601],{"className":42346},[110],[59,42348],{"className":42349,"style":31173},[114],[59,42351,18365],{"className":42352},[119],[59,42354,42356,42359,42412],{"className":42355},[119],[59,42357],{"className":42358},[123,124],[59,42360,42362],{"className":42361},[78],[59,42363,42365,42404],{"className":42364},[131,132],[59,42366,42368,42401],{"className":42367},[136],[59,42369,42371,42382,42390],{"className":42370,"style":16279},[140],[59,42372,42373,42376],{"style":15819},[59,42374],{"className":42375,"style":149},[148],[59,42377,42379],{"className":42378},[119],[59,42380,7322],{"className":42381,"style":7397},[119,192],[59,42383,42384,42387],{"style":274},[59,42385],{"className":42386,"style":149},[148],[59,42388],{"className":42389,"style":282},[281],[59,42391,42392,42395],{"style":15885},[59,42393],{"className":42394,"style":149},[148],[59,42396,42398],{"className":42397},[119],[59,42399,83],{"className":42400},[119],[59,42402,227],{"className":42403},[226],[59,42405,42407],{"className":42406},[136],[59,42408,42410],{"className":42409,"style":16319},[140],[59,42411],{},[59,42413],{"className":42414},[313,124],[59,42416],{"className":42417,"style":916},[499],[59,42419,42421],{"className":42420},[951,14238],[59,42422,42424,42477],{"className":42423},[131,132],[59,42425,42427,42474],{"className":42426},[136],[59,42428,42430,42450,42460],{"className":42429,"style":16340},[140],[59,42431,42432,42435],{"style":16343},[59,42433],{"className":42434,"style":15432},[148],[59,42436,42438],{"className":42437},[153,154,155,156],[59,42439,42441,42444,42447],{"className":42440},[119,156],[59,42442,335],{"className":42443},[119,192,156],[59,42445,815],{"className":42446},[504,156],[59,42448,83],{"className":42449},[119,156],[59,42451,42452,42455],{"style":15441},[59,42453],{"className":42454,"style":15432},[148],[59,42456,42457],{},[59,42458,1826],{"className":42459},[951,1910,15450],[59,42461,42462,42465],{"style":16374},[59,42463],{"className":42464,"style":15432},[148],[59,42466,42468],{"className":42467},[153,154,155,156],[59,42469,42471],{"className":42470},[119,156],[59,42472,7322],{"className":42473,"style":7397},[119,192,156],[59,42475,227],{"className":42476},[226],[59,42478,42480],{"className":42479},[136],[59,42481,42483],{"className":42482,"style":16393},[140],[59,42484],{},[59,42486],{"className":42487,"style":916},[499],[59,42489,42491],{"className":42490},[951,14238],[59,42492,42494,42584],{"className":42493},[131,132],[59,42495,42497,42581],{"className":42496},[136],[59,42498,42500,42520,42530],{"className":42499,"style":17149},[140],[59,42501,42502,42505],{"style":16343},[59,42503],{"className":42504,"style":15432},[148],[59,42506,42508],{"className":42507},[153,154,155,156],[59,42509,42511,42514,42517],{"className":42510},[119,156],[59,42512,777],{"className":42513},[119,192,156],[59,42515,815],{"className":42516},[504,156],[59,42518,83],{"className":42519},[119,156],[59,42521,42522,42525],{"style":15441},[59,42523],{"className":42524,"style":15432},[148],[59,42526,42527],{},[59,42528,1826],{"className":42529},[951,1910,15450],[59,42531,42532,42535],{"style":17182},[59,42533],{"className":42534,"style":15432},[148],[59,42536,42538],{"className":42537},[153,154,155,156],[59,42539,42541],{"className":42540},[119,156],[59,42542,42544,42547],{"className":42543},[119,156],[59,42545,13242],{"className":42546,"style":3185},[119,192,156],[59,42548,42550],{"className":42549},[196],[59,42551,42553,42573],{"className":42552},[131,132],[59,42554,42556,42570],{"className":42555},[136],[59,42557,42559],{"className":42558,"style":4302},[140],[59,42560,42561,42564],{"style":17212},[59,42562],{"className":42563,"style":213},[148],[59,42565,42567],{"className":42566},[153,217,218,156],[59,42568,335],{"className":42569},[119,192,156],[59,42571,227],{"className":42572},[226],[59,42574,42576],{"className":42575},[136],[59,42577,42579],{"className":42578,"style":4324},[140],[59,42580],{},[59,42582,227],{"className":42583},[226],[59,42585,42587],{"className":42586},[136],[59,42588,42590],{"className":42589,"style":16393},[140],[59,42591],{},[59,42593],{"className":42594,"style":916},[499],[59,42596,42004],{"className":42597},[951],[59,42599],{"className":42600,"style":916},[499],[59,42602,42604,42610,42705,42708,42759,42762,42765,42816,42819,43214,43217,43220,43223,43231,43234,43700,43703,43796,43799,43850,43853,43856,43907,43910],{"className":42603},[962],[59,42605,42607],{"className":42606,"style":967},[123,966],[59,42608,798],{"className":42609},[971,155],[59,42611,42614,42617],{"className":42612,"style":42613},[119],"color:red;",[59,42615,774],{"className":42616,"style":42613},[119,192],[59,42618,42620],{"className":42619},[196],[59,42621,42623,42697],{"className":42622},[131,132],[59,42624,42626,42694],{"className":42625},[136],[59,42627,42629,42673],{"className":42628,"style":24796},[140],[59,42630,42631,42634],{"style":17311},[59,42632],{"className":42633,"style":540},[148],[59,42635,42637],{"className":42636,"style":42613},[153,154,155,156],[59,42638,42640],{"className":42639,"style":42613},[119,156],[59,42641,42643,42647],{"className":42642,"style":42613},[119,156],[59,42644,14051],{"className":42645,"style":42646},[119,192,156],"margin-right:0.02778em;color:red;",[59,42648,42650],{"className":42649},[196],[59,42651,42653],{"className":42652},[131],[59,42654,42656],{"className":42655},[136],[59,42657,42659],{"className":42658,"style":19144},[140],[59,42660,42661,42664],{"style":19147},[59,42662],{"className":42663,"style":213},[148],[59,42665,42667],{"className":42666,"style":42613},[153,217,218,156],[59,42668,42670],{"className":42669,"style":42613},[119,156],[59,42671,18984],{"className":42672,"style":42613},[119,156],[59,42674,42675,42678],{"style":16426},[59,42676],{"className":42677,"style":540},[148],[59,42679,42681],{"className":42680,"style":42613},[153,154,155,156],[59,42682,42684,42687,42690],{"className":42683,"style":42613},[119,156],[59,42685,27510],{"className":42686,"style":42613},[119,192,156],[59,42688,774],{"className":42689,"style":42613},[119,192,156],[59,42691,5607],{"className":42692,"style":42693},[119,192,156],"margin-right:0.05764em;color:red;",[59,42695,227],{"className":42696},[226],[59,42698,42700],{"className":42699},[136],[59,42701,42703],{"className":42702,"style":17341},[140],[59,42704],{},[59,42706,798],{"className":42707,"style":42613},[123],[59,42709,42711,42714],{"className":42710,"style":42613},[119],[59,42712,12802],{"className":42713,"style":42613},[119,192],[59,42715,42717],{"className":42716},[196],[59,42718,42720,42751],{"className":42719},[131,132],[59,42721,42723,42748],{"className":42722},[136],[59,42724,42726,42737],{"className":42725,"style":17308},[140],[59,42727,42728,42731],{"style":17311},[59,42729],{"className":42730,"style":540},[148],[59,42732,42734],{"className":42733,"style":42613},[153,154,155,156],[59,42735,335],{"className":42736,"style":42613},[119,192,156],[59,42738,42739,42742],{"style":16426},[59,42740],{"className":42741,"style":540},[148],[59,42743,42745],{"className":42744,"style":42613},[153,154,155,156],[59,42746,777],{"className":42747,"style":42613},[119,192,156],[59,42749,227],{"className":42750},[226],[59,42752,42754],{"className":42753},[136],[59,42755,42757],{"className":42756,"style":17341},[140],[59,42758],{},[59,42760,803],{"className":42761,"style":42613},[912],[59,42763],{"className":42764,"style":916},[499],[59,42766,42768,42771],{"className":42767,"style":42613},[119],[59,42769,22],{"className":42770,"style":42613},[119,192],[59,42772,42774],{"className":42773},[196],[59,42775,42777,42808],{"className":42776},[131,132],[59,42778,42780,42805],{"className":42779},[136],[59,42781,42783,42794],{"className":42782,"style":17308},[140],[59,42784,42785,42788],{"style":17311},[59,42786],{"className":42787,"style":540},[148],[59,42789,42791],{"className":42790,"style":42613},[153,154,155,156],[59,42792,335],{"className":42793,"style":42613},[119,192,156],[59,42795,42796,42799],{"style":16426},[59,42797],{"className":42798,"style":540},[148],[59,42800,42802],{"className":42801,"style":42613},[153,154,155,156],[59,42803,777],{"className":42804,"style":42613},[119,192,156],[59,42806,227],{"className":42807},[226],[59,42809,42811],{"className":42810},[136],[59,42812,42814],{"className":42813,"style":17341},[140],[59,42815],{},[59,42817,812],{"className":42818,"style":42613},[313],[59,42820,42822,42825,43211],{"className":42821,"style":42613},[119],[59,42823],{"className":42824},[123,124],[59,42826,42828],{"className":42827},[78],[59,42829,42831,43203],{"className":42830},[131,132],[59,42832,42834,43200],{"className":42833},[136],[59,42835,42837,43029,43038],{"className":42836,"style":36303},[140],[59,42838,42839,42842],{"style":15819},[59,42840],{"className":42841,"style":149},[148],[59,42843,42845,42918,42921,42972,42975,43026],{"className":42844,"style":42613},[119],[59,42846,42848,42852],{"className":42847,"style":42613},[119],[59,42849,5604],{"className":42850,"style":42851},[119,192],"margin-right:0.13889em;color:red;",[59,42853,42855],{"className":42854},[196],[59,42856,42858,42910],{"className":42857},[131,132],[59,42859,42861,42907],{"className":42860},[136],[59,42862,42864],{"className":42863,"style":1248},[140],[59,42865,42866,42869],{"style":7460},[59,42867],{"className":42868,"style":540},[148],[59,42870,42872],{"className":42871,"style":42613},[153,154,155,156],[59,42873,42875],{"className":42874,"style":42613},[119,156],[59,42876,42878,42881],{"className":42877,"style":42613},[119,156],[59,42879,14051],{"className":42880,"style":42646},[119,192,156],[59,42882,42884],{"className":42883},[196],[59,42885,42887],{"className":42886},[131],[59,42888,42890],{"className":42889},[136],[59,42891,42893],{"className":42892,"style":19144},[140],[59,42894,42895,42898],{"style":19147},[59,42896],{"className":42897,"style":213},[148],[59,42899,42901],{"className":42900,"style":42613},[153,217,218,156],[59,42902,42904],{"className":42903,"style":42613},[119,156],[59,42905,18984],{"className":42906,"style":42613},[119,156],[59,42908,227],{"className":42909},[226],[59,42911,42913],{"className":42912},[136],[59,42914,42916],{"className":42915,"style":1270},[140],[59,42917],{},[59,42919,798],{"className":42920,"style":42613},[123],[59,42922,42924,42927],{"className":42923,"style":42613},[119],[59,42925,22],{"className":42926,"style":42613},[119,192],[59,42928,42930],{"className":42929},[196],[59,42931,42933,42964],{"className":42932},[131,132],[59,42934,42936,42961],{"className":42935},[136],[59,42937,42939,42950],{"className":42938,"style":36405},[140],[59,42940,42941,42944],{"style":17311},[59,42942],{"className":42943,"style":540},[148],[59,42945,42947],{"className":42946,"style":42613},[153,154,155,156],[59,42948,335],{"className":42949,"style":42613},[119,192,156],[59,42951,42952,42955],{"style":16529},[59,42953],{"className":42954,"style":540},[148],[59,42956,42958],{"className":42957,"style":42613},[153,154,155,156],[59,42959,777],{"className":42960,"style":42613},[119,192,156],[59,42962,227],{"className":42963},[226],[59,42965,42967],{"className":42966},[136],[59,42968,42970],{"className":42969,"style":17341},[140],[59,42971],{},[59,42973,12941],{"className":42974,"style":42613},[119],[59,42976,42978,42981],{"className":42977,"style":42613},[119],[59,42979,12802],{"className":42980,"style":42613},[119,192],[59,42982,42984],{"className":42983},[196],[59,42985,42987,43018],{"className":42986},[131,132],[59,42988,42990,43015],{"className":42989},[136],[59,42991,42993,43004],{"className":42992,"style":36405},[140],[59,42994,42995,42998],{"style":17311},[59,42996],{"className":42997,"style":540},[148],[59,42999,43001],{"className":43000,"style":42613},[153,154,155,156],[59,43002,335],{"className":43003,"style":42613},[119,192,156],[59,43005,43006,43009],{"style":16529},[59,43007],{"className":43008,"style":540},[148],[59,43010,43012],{"className":43011,"style":42613},[153,154,155,156],[59,43013,777],{"className":43014,"style":42613},[119,192,156],[59,43016,227],{"className":43017},[226],[59,43019,43021],{"className":43020},[136],[59,43022,43024],{"className":43023,"style":17341},[140],[59,43025],{},[59,43027,812],{"className":43028,"style":42613},[313],[59,43030,43031,43034],{"style":274},[59,43032],{"className":43033,"style":149},[148],[59,43035],{"className":43036,"style":43037},[281],"color:red;border-bottom-width:0.04em;",[59,43039,43040,43043],{"style":15885},[59,43041],{"className":43042,"style":149},[148],[59,43044,43046,43089,43092,43143,43146,43197],{"className":43045,"style":42613},[119],[59,43047,43049,43052],{"className":43048,"style":42613},[119],[59,43050,5604],{"className":43051,"style":42851},[119,192],[59,43053,43055],{"className":43054},[196],[59,43056,43058,43081],{"className":43057},[131,132],[59,43059,43061,43078],{"className":43060},[136],[59,43062,43064],{"className":43063,"style":1248},[140],[59,43065,43066,43069],{"style":7460},[59,43067],{"className":43068,"style":540},[148],[59,43070,43072],{"className":43071,"style":42613},[153,154,155,156],[59,43073,43075],{"className":43074,"style":42613},[119,156],[59,43076,14051],{"className":43077,"style":42646},[119,192,156],[59,43079,227],{"className":43080},[226],[59,43082,43084],{"className":43083},[136],[59,43085,43087],{"className":43086,"style":1270},[140],[59,43088],{},[59,43090,798],{"className":43091,"style":42613},[123],[59,43093,43095,43098],{"className":43094,"style":42613},[119],[59,43096,22],{"className":43097,"style":42613},[119,192],[59,43099,43101],{"className":43100},[196],[59,43102,43104,43135],{"className":43103},[131,132],[59,43105,43107,43132],{"className":43106},[136],[59,43108,43110,43121],{"className":43109,"style":18849},[140],[59,43111,43112,43115],{"style":17311},[59,43113],{"className":43114,"style":540},[148],[59,43116,43118],{"className":43117,"style":42613},[153,154,155,156],[59,43119,335],{"className":43120,"style":42613},[119,192,156],[59,43122,43123,43126],{"style":536},[59,43124],{"className":43125,"style":540},[148],[59,43127,43129],{"className":43128,"style":42613},[153,154,155,156],[59,43130,777],{"className":43131,"style":42613},[119,192,156],[59,43133,227],{"className":43134},[226],[59,43136,43138],{"className":43137},[136],[59,43139,43141],{"className":43140,"style":17341},[140],[59,43142],{},[59,43144,12941],{"className":43145,"style":42613},[119],[59,43147,43149,43152],{"className":43148,"style":42613},[119],[59,43150,12802],{"className":43151,"style":42613},[119,192],[59,43153,43155],{"className":43154},[196],[59,43156,43158,43189],{"className":43157},[131,132],[59,43159,43161,43186],{"className":43160},[136],[59,43162,43164,43175],{"className":43163,"style":18849},[140],[59,43165,43166,43169],{"style":17311},[59,43167],{"className":43168,"style":540},[148],[59,43170,43172],{"className":43171,"style":42613},[153,154,155,156],[59,43173,335],{"className":43174,"style":42613},[119,192,156],[59,43176,43177,43180],{"style":536},[59,43178],{"className":43179,"style":540},[148],[59,43181,43183],{"className":43182,"style":42613},[153,154,155,156],[59,43184,777],{"className":43185,"style":42613},[119,192,156],[59,43187,227],{"className":43188},[226],[59,43190,43192],{"className":43191},[136],[59,43193,43195],{"className":43194,"style":17341},[140],[59,43196],{},[59,43198,812],{"className":43199,"style":42613},[313],[59,43201,227],{"className":43202},[226],[59,43204,43206],{"className":43205},[136],[59,43207,43209],{"className":43208,"style":15950},[140],[59,43210],{},[59,43212],{"className":43213},[313,124],[59,43215,803],{"className":43216},[912],[59,43218],{"className":43219,"style":916},[499],[59,43221],{"className":43222,"style":916},[499],[59,43224,43227],{"className":43225,"style":43226},[119],"color:skyblue;",[59,43228,43230],{"className":43229,"style":43226},[119,955],"clip",[59,43232],{"className":43233,"style":916},[499],[59,43235,43237,43246,43642,43645,43649,43652,43655,43659,43662,43665,43668,43671,43674,43677,43680,43683,43686,43689,43692],{"className":43236,"style":43226},[962],[59,43238,43241],{"className":43239,"style":43240},[123,966],"color:skyblue;top:0em;",[59,43242,43244],{"className":43243,"style":43226},[971,155],[59,43245,798],{"style":43226},[59,43247,43249,43252,43639],{"className":43248,"style":43226},[119],[59,43250],{"className":43251},[123,124],[59,43253,43255],{"className":43254},[78],[59,43256,43258,43631],{"className":43257},[131,132],[59,43259,43261,43628],{"className":43260},[136],[59,43262,43264,43457,43466],{"className":43263,"style":36303},[140],[59,43265,43266,43269],{"style":15819},[59,43267],{"className":43268,"style":149},[148],[59,43270,43272,43346,43349,43400,43403,43454],{"className":43271,"style":43226},[119],[59,43273,43275,43279],{"className":43274,"style":43226},[119],[59,43276,5604],{"className":43277,"style":43278},[119,192],"margin-right:0.13889em;color:skyblue;",[59,43280,43282],{"className":43281},[196],[59,43283,43285,43338],{"className":43284},[131,132],[59,43286,43288,43335],{"className":43287},[136],[59,43289,43291],{"className":43290,"style":1248},[140],[59,43292,43293,43296],{"style":7460},[59,43294],{"className":43295,"style":540},[148],[59,43297,43299],{"className":43298,"style":43226},[153,154,155,156],[59,43300,43302],{"className":43301,"style":43226},[119,156],[59,43303,43305,43309],{"className":43304,"style":43226},[119,156],[59,43306,14051],{"className":43307,"style":43308},[119,192,156],"margin-right:0.02778em;color:skyblue;",[59,43310,43312],{"className":43311},[196],[59,43313,43315],{"className":43314},[131],[59,43316,43318],{"className":43317},[136],[59,43319,43321],{"className":43320,"style":19144},[140],[59,43322,43323,43326],{"style":19147},[59,43324],{"className":43325,"style":213},[148],[59,43327,43329],{"className":43328,"style":43226},[153,217,218,156],[59,43330,43332],{"className":43331,"style":43226},[119,156],[59,43333,18984],{"className":43334,"style":43226},[119,156],[59,43336,227],{"className":43337},[226],[59,43339,43341],{"className":43340},[136],[59,43342,43344],{"className":43343,"style":1270},[140],[59,43345],{},[59,43347,798],{"className":43348,"style":43226},[123],[59,43350,43352,43355],{"className":43351,"style":43226},[119],[59,43353,22],{"className":43354,"style":43226},[119,192],[59,43356,43358],{"className":43357},[196],[59,43359,43361,43392],{"className":43360},[131,132],[59,43362,43364,43389],{"className":43363},[136],[59,43365,43367,43378],{"className":43366,"style":36405},[140],[59,43368,43369,43372],{"style":17311},[59,43370],{"className":43371,"style":540},[148],[59,43373,43375],{"className":43374,"style":43226},[153,154,155,156],[59,43376,335],{"className":43377,"style":43226},[119,192,156],[59,43379,43380,43383],{"style":16529},[59,43381],{"className":43382,"style":540},[148],[59,43384,43386],{"className":43385,"style":43226},[153,154,155,156],[59,43387,777],{"className":43388,"style":43226},[119,192,156],[59,43390,227],{"className":43391},[226],[59,43393,43395],{"className":43394},[136],[59,43396,43398],{"className":43397,"style":17341},[140],[59,43399],{},[59,43401,12941],{"className":43402,"style":43226},[119],[59,43404,43406,43409],{"className":43405,"style":43226},[119],[59,43407,12802],{"className":43408,"style":43226},[119,192],[59,43410,43412],{"className":43411},[196],[59,43413,43415,43446],{"className":43414},[131,132],[59,43416,43418,43443],{"className":43417},[136],[59,43419,43421,43432],{"className":43420,"style":36405},[140],[59,43422,43423,43426],{"style":17311},[59,43424],{"className":43425,"style":540},[148],[59,43427,43429],{"className":43428,"style":43226},[153,154,155,156],[59,43430,335],{"className":43431,"style":43226},[119,192,156],[59,43433,43434,43437],{"style":16529},[59,43435],{"className":43436,"style":540},[148],[59,43438,43440],{"className":43439,"style":43226},[153,154,155,156],[59,43441,777],{"className":43442,"style":43226},[119,192,156],[59,43444,227],{"className":43445},[226],[59,43447,43449],{"className":43448},[136],[59,43450,43452],{"className":43451,"style":17341},[140],[59,43453],{},[59,43455,812],{"className":43456,"style":43226},[313],[59,43458,43459,43462],{"style":274},[59,43460],{"className":43461,"style":149},[148],[59,43463],{"className":43464,"style":43465},[281],"color:skyblue;border-bottom-width:0.04em;",[59,43467,43468,43471],{"style":15885},[59,43469],{"className":43470,"style":149},[148],[59,43472,43474,43517,43520,43571,43574,43625],{"className":43473,"style":43226},[119],[59,43475,43477,43480],{"className":43476,"style":43226},[119],[59,43478,5604],{"className":43479,"style":43278},[119,192],[59,43481,43483],{"className":43482},[196],[59,43484,43486,43509],{"className":43485},[131,132],[59,43487,43489,43506],{"className":43488},[136],[59,43490,43492],{"className":43491,"style":1248},[140],[59,43493,43494,43497],{"style":7460},[59,43495],{"className":43496,"style":540},[148],[59,43498,43500],{"className":43499,"style":43226},[153,154,155,156],[59,43501,43503],{"className":43502,"style":43226},[119,156],[59,43504,14051],{"className":43505,"style":43308},[119,192,156],[59,43507,227],{"className":43508},[226],[59,43510,43512],{"className":43511},[136],[59,43513,43515],{"className":43514,"style":1270},[140],[59,43516],{},[59,43518,798],{"className":43519,"style":43226},[123],[59,43521,43523,43526],{"className":43522,"style":43226},[119],[59,43524,22],{"className":43525,"style":43226},[119,192],[59,43527,43529],{"className":43528},[196],[59,43530,43532,43563],{"className":43531},[131,132],[59,43533,43535,43560],{"className":43534},[136],[59,43536,43538,43549],{"className":43537,"style":18849},[140],[59,43539,43540,43543],{"style":17311},[59,43541],{"className":43542,"style":540},[148],[59,43544,43546],{"className":43545,"style":43226},[153,154,155,156],[59,43547,335],{"className":43548,"style":43226},[119,192,156],[59,43550,43551,43554],{"style":536},[59,43552],{"className":43553,"style":540},[148],[59,43555,43557],{"className":43556,"style":43226},[153,154,155,156],[59,43558,777],{"className":43559,"style":43226},[119,192,156],[59,43561,227],{"className":43562},[226],[59,43564,43566],{"className":43565},[136],[59,43567,43569],{"className":43568,"style":17341},[140],[59,43570],{},[59,43572,12941],{"className":43573,"style":43226},[119],[59,43575,43577,43580],{"className":43576,"style":43226},[119],[59,43578,12802],{"className":43579,"style":43226},[119,192],[59,43581,43583],{"className":43582},[196],[59,43584,43586,43617],{"className":43585},[131,132],[59,43587,43589,43614],{"className":43588},[136],[59,43590,43592,43603],{"className":43591,"style":18849},[140],[59,43593,43594,43597],{"style":17311},[59,43595],{"className":43596,"style":540},[148],[59,43598,43600],{"className":43599,"style":43226},[153,154,155,156],[59,43601,335],{"className":43602,"style":43226},[119,192,156],[59,43604,43605,43608],{"style":536},[59,43606],{"className":43607,"style":540},[148],[59,43609,43611],{"className":43610,"style":43226},[153,154,155,156],[59,43612,777],{"className":43613,"style":43226},[119,192,156],[59,43615,227],{"className":43616},[226],[59,43618,43620],{"className":43619},[136],[59,43621,43623],{"className":43622,"style":17341},[140],[59,43624],{},[59,43626,812],{"className":43627,"style":43226},[313],[59,43629,227],{"className":43630},[226],[59,43632,43634],{"className":43633},[136],[59,43635,43637],{"className":43636,"style":15950},[140],[59,43638],{},[59,43640],{"className":43641},[313,124],[59,43643,803],{"className":43644,"style":43226},[912],[59,43646],{"className":43647,"style":43648},[499],"color:skyblue;margin-right:0.1667em;",[59,43650],{"className":43651,"style":43648},[499],[59,43653,83],{"className":43654,"style":43226},[119],[59,43656],{"className":43657,"style":43658},[499],"color:skyblue;margin-right:0.2222em;",[59,43660,18365],{"className":43661,"style":43226},[553],[59,43663],{"className":43664,"style":43658},[499],[59,43666,41845],{"className":43667,"style":43226},[119,192],[59,43669,803],{"className":43670,"style":43226},[912],[59,43672],{"className":43673,"style":43648},[499],[59,43675],{"className":43676,"style":43648},[499],[59,43678,83],{"className":43679,"style":43226},[119],[59,43681],{"className":43682,"style":43658},[499],[59,43684,5777],{"className":43685,"style":43226},[553],[59,43687],{"className":43688,"style":43658},[499],[59,43690,41845],{"className":43691,"style":43226},[119,192],[59,43693,43695],{"className":43694,"style":43240},[313,966],[59,43696,43698],{"className":43697,"style":43226},[971,155],[59,43699,812],{"style":43226},[59,43701],{"className":43702,"style":916},[499],[59,43704,43706,43709],{"className":43705,"style":43226},[119],[59,43707,774],{"className":43708,"style":43226},[119,192],[59,43710,43712],{"className":43711},[196],[59,43713,43715,43788],{"className":43714},[131,132],[59,43716,43718,43785],{"className":43717},[136],[59,43719,43721,43764],{"className":43720,"style":24796},[140],[59,43722,43723,43726],{"style":17311},[59,43724],{"className":43725,"style":540},[148],[59,43727,43729],{"className":43728,"style":43226},[153,154,155,156],[59,43730,43732],{"className":43731,"style":43226},[119,156],[59,43733,43735,43738],{"className":43734,"style":43226},[119,156],[59,43736,14051],{"className":43737,"style":43308},[119,192,156],[59,43739,43741],{"className":43740},[196],[59,43742,43744],{"className":43743},[131],[59,43745,43747],{"className":43746},[136],[59,43748,43750],{"className":43749,"style":19144},[140],[59,43751,43752,43755],{"style":19147},[59,43753],{"className":43754,"style":213},[148],[59,43756,43758],{"className":43757,"style":43226},[153,217,218,156],[59,43759,43761],{"className":43760,"style":43226},[119,156],[59,43762,18984],{"className":43763,"style":43226},[119,156],[59,43765,43766,43769],{"style":16426},[59,43767],{"className":43768,"style":540},[148],[59,43770,43772],{"className":43771,"style":43226},[153,154,155,156],[59,43773,43775,43778,43781],{"className":43774,"style":43226},[119,156],[59,43776,27510],{"className":43777,"style":43226},[119,192,156],[59,43779,774],{"className":43780,"style":43226},[119,192,156],[59,43782,5607],{"className":43783,"style":43784},[119,192,156],"margin-right:0.05764em;color:skyblue;",[59,43786,227],{"className":43787},[226],[59,43789,43791],{"className":43790},[136],[59,43792,43794],{"className":43793,"style":17341},[140],[59,43795],{},[59,43797,798],{"className":43798,"style":43226},[123],[59,43800,43802,43805],{"className":43801,"style":43226},[119],[59,43803,12802],{"className":43804,"style":43226},[119,192],[59,43806,43808],{"className":43807},[196],[59,43809,43811,43842],{"className":43810},[131,132],[59,43812,43814,43839],{"className":43813},[136],[59,43815,43817,43828],{"className":43816,"style":17308},[140],[59,43818,43819,43822],{"style":17311},[59,43820],{"className":43821,"style":540},[148],[59,43823,43825],{"className":43824,"style":43226},[153,154,155,156],[59,43826,335],{"className":43827,"style":43226},[119,192,156],[59,43829,43830,43833],{"style":16426},[59,43831],{"className":43832,"style":540},[148],[59,43834,43836],{"className":43835,"style":43226},[153,154,155,156],[59,43837,777],{"className":43838,"style":43226},[119,192,156],[59,43840,227],{"className":43841},[226],[59,43843,43845],{"className":43844},[136],[59,43846,43848],{"className":43847,"style":17341},[140],[59,43849],{},[59,43851,803],{"className":43852,"style":43226},[912],[59,43854],{"className":43855,"style":916},[499],[59,43857,43859,43862],{"className":43858,"style":43226},[119],[59,43860,22],{"className":43861,"style":43226},[119,192],[59,43863,43865],{"className":43864},[196],[59,43866,43868,43899],{"className":43867},[131,132],[59,43869,43871,43896],{"className":43870},[136],[59,43872,43874,43885],{"className":43873,"style":17308},[140],[59,43875,43876,43879],{"style":17311},[59,43877],{"className":43878,"style":540},[148],[59,43880,43882],{"className":43881,"style":43226},[153,154,155,156],[59,43883,335],{"className":43884,"style":43226},[119,192,156],[59,43886,43887,43890],{"style":16426},[59,43888],{"className":43889,"style":540},[148],[59,43891,43893],{"className":43892,"style":43226},[153,154,155,156],[59,43894,777],{"className":43895,"style":43226},[119,192,156],[59,43897,227],{"className":43898},[226],[59,43900,43902],{"className":43901},[136],[59,43903,43905],{"className":43904,"style":17341},[140],[59,43906],{},[59,43908,812],{"className":43909,"style":43226},[313],[59,43911,43913],{"className":43912,"style":967},[313,966],[59,43914,812],{"className":43915},[971,155],[11,43917,43918],{"id":43918},"参考资料",[15,43920,43921],{},[22,43922,43925],{"href":43923,"rel":43924},"https://www.bilibili.com/video/BV1iz421h7gb/",[26],"零基础学习强化学习算法：ppo",{"title":2395,"searchDepth":2429,"depth":2429,"links":43927},[43928,43929,43930,43931,43934,43938],{"id":12673,"depth":2429,"text":12673},{"id":12679,"depth":2429,"text":12679},{"id":13945,"depth":2429,"text":13945},{"id":19461,"depth":2429,"text":19462,"children":43932},[43933],{"id":27438,"depth":2436,"text":27439},{"id":31760,"depth":2429,"text":31761,"children":43935},[43936,43937],{"id":31767,"depth":2436,"text":31767},{"id":33572,"depth":2436,"text":33573},{"id":43918,"depth":2429,"text":43918},"记一下学习 PPO中学习到的东西",{"date":43941,"image":43942,"alt":43943,"tags":43944,"published":43946},"8 Mar 2026","/blogs-img/blog2.jpg","PPO学习笔记",[43945,12661],"Reinforcement learning",false,"/blogs/ppo",{"title":12668,"description":43939},"blogs/2. PPO学习笔记","ngXl85tXoyEZC4uVOAWn1IScbiXzrl-vgx3OlmObKkM",1774096958854]