[{"data":1,"prerenderedAt":48221},["ShallowReactive",2],{"blog-post-/blogs/cosmos-policy-code-walk":3,"all-blogs-for-navigation":968},{"id":4,"title":5,"body":6,"description":5,"extension":955,"meta":956,"navigation":963,"ogImage":958,"path":964,"seo":965,"stem":966,"__hash__":967},"content/blogs/7. cosmos policy code walk.md","cosmos policy code walk",{"type":7,"value":8,"toc":939},"minimark",[9,13,25,34,37,40,43,46,178,185,203,209,223,237,240,247,251,254,427,434,437,444,453,523,530,533,536,695,698,701,704,707,710,721,748,758,772,860,864,871,878,881,884,891,894,897,907,911,914,917,920,923,926,929,932,935],[10,11,12],"h2",{"id":12},"简介",[14,15,16,17,24],"p",{},"对于不知道什么是 World Action Model（WAM） 的读者，建议先阅读一下 ",[18,19,23],"a",{"href":20,"rel":21},"https://zhuanlan.zhihu.com/p/2019875354538428076",[22],"nofollow","这篇博客","，这篇博客非常清晰地介绍了什么是 WAM，和 VLA 的区别等。",[14,26,27,28,33],{},"这篇文章分析一下 cosmos policy 的代码实现，参考 ",[18,29,32],{"href":30,"rel":31},"https://github.com/nvlabs/cosmos-policy",[22],"官方代码仓库","，当前 commit 版本是 18a2accadf4e7a3531e56754102af5a24d2316da。",[10,35,36],{"id":36},"部署",[14,38,39],{},"仓库提供了 Dockerfile，可以直接部署，我在 HKUST 的 SuperPod 上使用，需要改写为 Apptainer,下面放出我改写后的文件。",[14,41,42],{},"构建好镜像后正常运行，可以按照指导说的那样进入交互式模式简单体验一下，这里不再赘述。",[10,44,45],{"id":45},"代码结构",[47,48,53],"pre",{"className":49,"code":50,"language":51,"meta":52,"style":52},"language-txt shiki shiki-themes github-light dracula","cosmos-policy/\n├── README.md / SETUP.md / LIBERO.md / ROBOCASA.md / ALOHA.md  # 文档入口\n├── docker/                                                     # 可忽略\n├── bin/                                                        # 可忽略\n└── cosmos_policy/                    ★ 主战场\n    ├── models/                       ★★★ 核心模型\n    ├── modules/                      ★★  SDE + Sampler\n    ├── datasets/                     ★★  数据管线\n    ├── experiments/robot/            ★★  推理 & 评测\n    ├── config/                       ★   实验配置\n    ├── scripts/train.py              ★   训练入口\n    ├── trainer.py                    ★   训练循环扩展\n    ├── conditioner.py                ★   条件对象\n    ├── constants.py                  ★   平台常量\n    ├── tokenizers/wan2pt1.py         ☆   policy tokenizer 封装\n    ├── utils/                        ☆   工具\n    └── _src/                         ✗  三个轮子，整体跳过\n        ├── imaginaire/   (~200 files)  训练框架\n        ├── predict2/     (~200 files)  视频生成模型\n        └── reason1/      (~50 files)   VLM 嵌入（Policy 主要用 T5）\n","txt","",[54,55,56,64,70,76,82,88,94,100,106,112,118,124,130,136,142,148,154,160,166,172],"code",{"__ignoreMap":52},[57,58,61],"span",{"class":59,"line":60},"line",1,[57,62,63],{},"cosmos-policy/\n",[57,65,67],{"class":59,"line":66},2,[57,68,69],{},"├── README.md / SETUP.md / LIBERO.md / ROBOCASA.md / ALOHA.md  # 文档入口\n",[57,71,73],{"class":59,"line":72},3,[57,74,75],{},"├── docker/                                                     # 可忽略\n",[57,77,79],{"class":59,"line":78},4,[57,80,81],{},"├── bin/                                                        # 可忽略\n",[57,83,85],{"class":59,"line":84},5,[57,86,87],{},"└── cosmos_policy/                    ★ 主战场\n",[57,89,91],{"class":59,"line":90},6,[57,92,93],{},"    ├── models/                       ★★★ 核心模型\n",[57,95,97],{"class":59,"line":96},7,[57,98,99],{},"    ├── modules/                      ★★  SDE + Sampler\n",[57,101,103],{"class":59,"line":102},8,[57,104,105],{},"    ├── datasets/                     ★★  数据管线\n",[57,107,109],{"class":59,"line":108},9,[57,110,111],{},"    ├── experiments/robot/            ★★  推理 & 评测\n",[57,113,115],{"class":59,"line":114},10,[57,116,117],{},"    ├── config/                       ★   实验配置\n",[57,119,121],{"class":59,"line":120},11,[57,122,123],{},"    ├── scripts/train.py              ★   训练入口\n",[57,125,127],{"class":59,"line":126},12,[57,128,129],{},"    ├── trainer.py                    ★   训练循环扩展\n",[57,131,133],{"class":59,"line":132},13,[57,134,135],{},"    ├── conditioner.py                ★   条件对象\n",[57,137,139],{"class":59,"line":138},14,[57,140,141],{},"    ├── constants.py                  ★   平台常量\n",[57,143,145],{"class":59,"line":144},15,[57,146,147],{},"    ├── tokenizers/wan2pt1.py         ☆   policy tokenizer 封装\n",[57,149,151],{"class":59,"line":150},16,[57,152,153],{},"    ├── utils/                        ☆   工具\n",[57,155,157],{"class":59,"line":156},17,[57,158,159],{},"    └── _src/                         ✗  三个轮子，整体跳过\n",[57,161,163],{"class":59,"line":162},18,[57,164,165],{},"        ├── imaginaire/   (~200 files)  训练框架\n",[57,167,169],{"class":59,"line":168},19,[57,170,171],{},"        ├── predict2/     (~200 files)  视频生成模型\n",[57,173,175],{"class":59,"line":174},20,[57,176,177],{},"        └── reason1/      (~50 files)   VLM 嵌入（Policy 主要用 T5）\n",[14,179,180,181,184],{},"所有的代码都放在 ",[54,182,183],{},"cosmos-policy"," 目录下，我们后面讲的时候默认 Workspace 在这里。",[14,186,187,190,191,194,195,198,199,202],{},[54,188,189],{},"_src"," 目录下存放了这里用到的三个轮子，分别是 nvidia 自己造的 ",[54,192,193],{},"imaginaire","、",[54,196,197],{},"predict2"," 和 ",[54,200,201],{},"reason1","，这三个库的用法这里不讲。",[14,204,205,208],{},[54,206,207],{},"models"," 目录下两个文件非常重要，定义了模型的结构，稍后我们重点讲。",[14,210,211,214,215,218,219,222],{},[54,212,213],{},"modules"," 下两个文件分别定义了噪声采样器 ",[54,216,217],{},"HybridEDMSDE"," 和去噪器 ",[54,220,221],{},"CosmosPolicySampler","。",[14,224,225,228,229,232,233,236],{},[54,226,227],{},"experiments/robot"," 下存放实验相关代码，",[54,230,231],{},"cosmos_utils.py"," 文件中实现了 ",[54,234,235],{},"get_acion()"," 函数，非常重要。",[10,238,239],{"id":239},"数据部分",[14,241,242,243,246],{},"重点看一下 ",[54,244,245],{},"datasets/libero_datasets.py"," 这个文件。",[248,249,250],"h3",{"id":250},"加载数据",[14,252,253],{},"从 171行到234行这一段是在加载演示数据，将每个 episode 存入内存，格式如下：",[47,255,259],{"className":256,"code":257,"language":258,"meta":52,"style":52},"language-python shiki shiki-themes github-light dracula","self.data[42] = {\n    \"images\":        np.ndarray,  # (150/T, 128/H, 128/W, 3) uint8\n    \"wrist_images\":  np.ndarray,  # (150/T, 128/H, 128/W, 3) uint8\n    \"proprio\":       np.ndarray,  # (150/T, 9) float32，z-score 归一化\n    \"actions\":       np.ndarray,  # (150/T, 7) float32，z-score 归一化\n    \"command\":       \"put both the alphabet soup and the tomato sauce in the basket\",\n    \"num_steps\":     150,\n    \"suite\":         \"libero_10_no_noops_rerendered\",\n    \"returns\":       np.ndarray,  # (150/T,) float32，demo 末尾 reward=1.0 折扣得到\n}\n","python",[54,260,261,285,305,319,334,348,370,387,408,422],{"__ignoreMap":52},[57,262,263,267,271,275,278,282],{"class":59,"line":60},[57,264,266],{"class":265},"sJti5","self",[57,268,270],{"class":269},"scbbO",".data[",[57,272,274],{"class":273},"soDru","42",[57,276,277],{"class":269},"] ",[57,279,281],{"class":280},"szJfE","=",[57,283,284],{"class":269}," {\n",[57,286,287,291,295,298,301],{"class":59,"line":66},[57,288,290],{"class":289},"sMWOi","    \"",[57,292,294],{"class":293},"sEzAm","images",[57,296,297],{"class":289},"\"",[57,299,300],{"class":269},":        np.ndarray,  ",[57,302,304],{"class":303},"sfgPZ","# (150/T, 128/H, 128/W, 3) uint8\n",[57,306,307,309,312,314,317],{"class":59,"line":72},[57,308,290],{"class":289},[57,310,311],{"class":293},"wrist_images",[57,313,297],{"class":289},[57,315,316],{"class":269},":  np.ndarray,  ",[57,318,304],{"class":303},[57,320,321,323,326,328,331],{"class":59,"line":78},[57,322,290],{"class":289},[57,324,325],{"class":293},"proprio",[57,327,297],{"class":289},[57,329,330],{"class":269},":       np.ndarray,  ",[57,332,333],{"class":303},"# (150/T, 9) float32，z-score 归一化\n",[57,335,336,338,341,343,345],{"class":59,"line":84},[57,337,290],{"class":289},[57,339,340],{"class":293},"actions",[57,342,297],{"class":289},[57,344,330],{"class":269},[57,346,347],{"class":303},"# (150/T, 7) float32，z-score 归一化\n",[57,349,350,352,355,357,360,362,365,367],{"class":59,"line":90},[57,351,290],{"class":289},[57,353,354],{"class":293},"command",[57,356,297],{"class":289},[57,358,359],{"class":269},":       ",[57,361,297],{"class":289},[57,363,364],{"class":293},"put both the alphabet soup and the tomato sauce in the basket",[57,366,297],{"class":289},[57,368,369],{"class":269},",\n",[57,371,372,374,377,379,382,385],{"class":59,"line":96},[57,373,290],{"class":289},[57,375,376],{"class":293},"num_steps",[57,378,297],{"class":289},[57,380,381],{"class":269},":     ",[57,383,384],{"class":273},"150",[57,386,369],{"class":269},[57,388,389,391,394,396,399,401,404,406],{"class":59,"line":102},[57,390,290],{"class":289},[57,392,393],{"class":293},"suite",[57,395,297],{"class":289},[57,397,398],{"class":269},":         ",[57,400,297],{"class":289},[57,402,403],{"class":293},"libero_10_no_noops_rerendered",[57,405,297],{"class":289},[57,407,369],{"class":269},[57,409,410,412,415,417,419],{"class":59,"line":108},[57,411,290],{"class":289},[57,413,414],{"class":293},"returns",[57,416,297],{"class":289},[57,418,330],{"class":269},[57,420,421],{"class":303},"# (150/T,) float32，demo 末尾 reward=1.0 折扣得到\n",[57,423,424],{"class":59,"line":114},[57,425,426],{"class":269},"}\n",[14,428,429,430,433],{},"指令从文件名中解析，之后计算价值，termial reward = 1.0，用传入的常数 gamma 做 Monte Carlo 折扣，之后缩放到 ",[57,431,432],{},"-1,1"," 之间。",[14,435,436],{},"之后跳到 401-414 行，这里构建了全局 step 索引，按 task suite 进行分组，进行均衡采样。",[248,438,440,443],{"id":439},"__getitem__-实现",[54,441,442],{},"__getitem__"," 实现",[14,445,446,448,449,452],{},[54,447,442],{}," 根据全局 step 索引 ",[54,450,451],{},"idx"," 区分三种数据来源：",[454,455,456,472],"table",{},[457,458,459],"thead",{},[460,461,462,466,469],"tr",{},[463,464,465],"th",{},"类型",[463,467,468],{},"定位方式",[463,470,471],{},"数据加载",[473,474,475,496,510],"tbody",{},[460,476,477,481,490],{},[478,479,480],"td",{},"demo",[478,482,483,486,487],{},[54,484,485],{},"idx % num_steps"," → ",[54,488,489],{},"_step_to_episode_map",[478,491,492,495],{},[54,493,494],{},"self.data","（已在内存）",[460,497,498,501,507],{},[478,499,500],{},"success rollout",[478,502,503,506],{},[54,504,505],{},"idx - adjusted_demo_count"," → success map",[478,508,509],{},"懒加载 HDF5",[460,511,512,515,521],{},[478,513,514],{},"failure rollout",[478,516,517,520],{},[54,518,519],{},"idx - demo - success"," → failure map",[478,522,509],{},[14,524,525,526,529],{},"另外 ",[54,527,528],{},"return_value_function_returns=True"," 且采样方式非 demo 时，50% 概率标为 world_model_sample，50% 标为 value_function_sample。",[14,531,532],{},"接下来是最重要的拼 image_list。",[14,534,535],{},"LIBERO 默认配置下的 latent 序列布局（num_duplicates_per_image=4，即每类占 4 个 RGB 帧 = 1 个 latent 帧）：",[454,537,538,554],{},[457,539,540],{},[460,541,542,545,548,551],{},[463,543,544],{},"current_sequence_idx",[463,546,547],{},"内容",[463,549,550],{},"实际图像",[463,552,553],{},"latent_idx 变量",[473,555,556,570,586,602,618,633,648,664,680],{},[460,557,558,561,564,567],{},[478,559,560],{},"0",[478,562,563],{},"blank (1帧)",[478,565,566],{},"全零",[478,568,569],{},"(VAE 时序压缩占位)",[460,571,572,575,578,581],{},[478,573,574],{},"1",[478,576,577],{},"current proprio",[478,579,580],{},"全零×4",[478,582,583],{},[54,584,585],{},"current_proprio_latent_idx",[460,587,588,591,594,597],{},[478,589,590],{},"2",[478,592,593],{},"current wrist",[478,595,596],{},"真实腕部×4",[478,598,599],{},[54,600,601],{},"current_wrist_image_latent_idx",[460,603,604,607,610,613],{},[478,605,606],{},"3",[478,608,609],{},"current primary",[478,611,612],{},"真实第三人称×4",[478,614,615],{},[54,616,617],{},"current_image_latent_idx",[460,619,620,623,626,628],{},[478,621,622],{},"4",[478,624,625],{},"action chunk",[478,627,580],{},[478,629,630],{},[54,631,632],{},"action_latent_idx",[460,634,635,638,641,643],{},[478,636,637],{},"5",[478,639,640],{},"future proprio",[478,642,580],{},[478,644,645],{},[54,646,647],{},"future_proprio_latent_idx",[460,649,650,653,656,659],{},[478,651,652],{},"6",[478,654,655],{},"future wrist",[478,657,658],{},"未来腕部×4",[478,660,661],{},[54,662,663],{},"future_wrist_image_latent_idx",[460,665,666,669,672,675],{},[478,667,668],{},"7",[478,670,671],{},"future primary",[478,673,674],{},"未来第三人称×4",[478,676,677],{},[54,678,679],{},"future_image_latent_idx",[460,681,682,685,688,690],{},[478,683,684],{},"8",[478,686,687],{},"value",[478,689,580],{},[478,691,692],{},[54,693,694],{},"value_latent_idx",[14,696,697],{},"这里还对应了 config 里的 state_t=9。",[14,699,700],{},"之后填充 Action Chunk，一个action_chunk 表示未来 chunk_size 步的动作，next_action_chunk 表示从 t+chunk_size 起的下一个动作，末尾不够则一直重复最后一个动作。",[14,702,703],{},"最后返回填充完整的字典。",[14,705,706],{},"上面有很多全零的部分，这里会在下一部分“核心模型”处填充对应内容。",[10,708,709],{"id":709},"核心模型",[14,711,712,713,716,717,720],{},"先讲 ",[54,714,715],{},"text2world_model.py","，因为 ",[54,718,719],{},"video2world_model.py"," 是继承自这个文件的。",[14,722,723,724,727,728,731,732,735,736,739,740,743,744,747],{},"后文张量维度统一记为 ",[54,725,726],{},"B × C × T × H × W","：",[54,729,730],{},"B"," 为 batch size，",[54,733,734],{},"C"," 为 latent 通道数，",[54,737,738],{},"T"," 为时间帧数，",[54,741,742],{},"H"," 与 ",[54,745,746],{},"W"," 分别为空间高宽。",[14,749,750,751,198,754,757],{},"首先有两个函数，分别是 ",[54,752,753],{},"replace_latent_with_action_chunk()",[54,755,756],{},"replace_latent_with_proprio","。这两个函数的作用是填充上面还是全零的内容。",[14,759,760,761,764,765,768,769,771],{},"另外是 ",[54,762,763],{},"CosmosPolicyModelConfig"," 这个类，这个类的定义和 ",[54,766,767],{},"modules/hybrid_edm_sde.py"," 中的 ",[54,770,217],{}," 是息息相关的，大概是父类采样是纯 log-normal 的方式，这里采用了 70% 纯 log-normal，30% uniform的方式，注意这里的 70% 和 30% 是期望。另外就是新增的五个字段，作用如下：",[454,773,774,787],{},[457,775,776],{},[460,777,778,781,784],{},[463,779,780],{},"字段",[463,782,783],{},"默认",[463,785,786],{},"作用",[473,788,789,804,818,832,846],{},[460,790,791,796,801],{},[478,792,793],{},[54,794,795],{},"mask_loss_for_action_future_state_prediction",[478,797,798],{},[54,799,800],{},"False",[478,802,803],{},"按样本类型分流 loss：demo→action，rollout→future state 或 value",[460,805,806,811,815],{},[478,807,808],{},[54,809,810],{},"mask_value_prediction_loss_for_policy_prediction",[478,812,813],{},[54,814,800],{},[478,816,817],{},"demo 同时优化 action+future state，不算 value loss",[460,819,820,825,829],{},[478,821,822],{},[54,823,824],{},"mask_current_state_action_for_value_prediction",[478,826,827],{},[54,828,800],{},[478,830,831],{},"value 样本只优化 value 帧（屏蔽 current state/action 帧的 loss）",[460,833,834,839,843],{},[478,835,836],{},[54,837,838],{},"mask_future_state_for_qvalue_prediction",[478,840,841],{},[54,842,800],{},[478,844,845],{},"Q(s,a) 预测时屏蔽 future state 帧",[460,847,848,853,857],{},[478,849,850],{},[54,851,852],{},"action_loss_multiplier",[478,854,855],{},[54,856,574],{},[478,858,859],{},"action 帧 loss 权重倍数（整数）",[248,861,863],{"id":862},"cosmospolicydiffusionmodel-类","CosmosPolicyDiffusionModel 类",[14,865,866,867,870],{},"这个是最重要的一段代码，除了 ",[54,868,869],{},"__init__"," 之外一共三个函数，但有些比较重要的函数在父类中实现，这里也讲一下作用。",[14,872,873,874,877],{},"训练流程在 ",[54,875,876],{},"training_step"," 中，首先更新一下训练统计的信息，之后选择性地编码文本。",[14,879,880],{},"之后先获取干净的潜变量和条件，然后采样出噪声水平和噪声，再进行一次广播操作将数据进行切分方便并行计算。",[14,882,883],{},"然后进行损失聚合，分为两种 mean 是对所有元素求平均，sum 是沿着通道维度求和并求平均，之后乘以缩放因子。",[14,885,886,887,890],{},"上面有个关键函数 ",[54,888,889],{},"compute_loss_with_epsilon_and_sigma","，下面讲这个。",[14,892,893],{},"这里传入了大量的数据，这些数据看名字就能看出作用，不难理解。",[14,895,896],{},"输入的原始 x_0 还是纯净的，没有注入 action/proprio/value 这些，首先进行备份然后注入，方法就是上面两个函数的展平重复填满，之后加噪得到 xt,之后去噪得到预测的 model_pred，然后对所有的元素都计算 loss，之后根据上面预设的 mask 方式控制每个 batch 样本的每个 latent 帧是否参与反传。",[14,898,899,902,903,906],{},[54,900,901],{},"generate_samples_from_batch"," 为推理的主入口，方式大概为将 denoise 函数包装到 ",[54,904,905],{},"x0_fn"," 中，然后传递给 sampler 调用，之后获取去噪的 latent 结果。",[248,908,910],{"id":909},"cosmospolicyvideo2worldmodel-类","CosmosPolicyVideo2WorldModel 类",[14,912,913],{},"这里重写了 get_data_and_condition，get_x0_fn_from_batch，denoise 和 draw_training_sigma_and_epsilon 三个函数，另外 Text2WorldModelConfig 类也增加了 Video2WorldConfig 类的一些字段。",[14,915,916],{},"LIBERO 实验覆盖：min_num_conditional_frames=4，sigma_conditional=0.0，state_t=9。",[14,918,919],{},"get_data_and_condition",[14,921,922],{},"这里将前 4 帧设置成了条件帧，同时按照 rollout 任务改 mask， world model模式 action 也设置成条件帧，value模式除了value全部是条件帧数，Q-value 模式下 mask 掉 future state。",[14,924,925],{},"此外还要往 gt_frames 中也注入非图像模式。",[14,927,928],{},"denoise 中设置了条件帧不去噪，只去噪后面需要去噪的几帧。",[10,930,931],{"id":931},"配置与训练入口",[10,933,934],{"id":934},"推理与评测",[936,937,938],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sJti5, html code.shiki .sJti5{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html pre.shiki code .scbbO, html code.shiki .scbbO{--shiki-default:#24292E;--shiki-dark:#F8F8F2}html pre.shiki code .soDru, html code.shiki .soDru{--shiki-default:#005CC5;--shiki-dark:#BD93F9}html pre.shiki code .szJfE, html code.shiki .szJfE{--shiki-default:#D73A49;--shiki-dark:#FF79C6}html pre.shiki code .sMWOi, html code.shiki .sMWOi{--shiki-default:#032F62;--shiki-dark:#E9F284}html pre.shiki code .sEzAm, html code.shiki .sEzAm{--shiki-default:#032F62;--shiki-dark:#F1FA8C}html pre.shiki code .sfgPZ, html code.shiki .sfgPZ{--shiki-default:#6A737D;--shiki-dark:#6272A4}",{"title":52,"searchDepth":66,"depth":66,"links":940},[941,942,943,944,949,953,954],{"id":12,"depth":66,"text":12},{"id":36,"depth":66,"text":36},{"id":45,"depth":66,"text":45},{"id":239,"depth":66,"text":239,"children":945},[946,947],{"id":250,"depth":72,"text":250},{"id":439,"depth":72,"text":948},"__getitem__ 实现",{"id":709,"depth":66,"text":709,"children":950},[951,952],{"id":862,"depth":72,"text":863},{"id":909,"depth":72,"text":910},{"id":931,"depth":66,"text":931},{"id":934,"depth":66,"text":934},"md",{"date":957,"image":958,"alt":959,"tags":960,"published":963},"9 May 2026","/blogs-img/blog8.jpg","cosmos policy 代码阅读",[961,962],"学习笔记","World Action Model",true,"/blogs/cosmos-policy-code-walk",{"title":5,"description":5},"blogs/7. cosmos policy code walk","OkTZpVjURRKjM5MJuEqxf2iiNAs1jTyeIkNOcDUFr6k",[969,13392,46091,47053,47243,47310,47560],{"id":970,"title":971,"body":972,"description":13382,"extension":955,"meta":13383,"navigation":963,"ogImage":13385,"path":13388,"seo":13389,"stem":13390,"__hash__":13391},"content/blogs/1. transformer复习笔记.md","transformer 复习笔记",{"type":7,"value":973,"toc":13366},[974,977,980,994,997,1004,1007,1010,1013,1271,1395,1711,2126,2132,2259,3134,3141,3345,3348,3582,3593,3596,3599,3604,3933,5278,5296,5520,5975,5978,6059,6062,6657,6999,7336,7689,7961,7964,7967,8305,8308,8465,8468,8471,8720,8859,8862,9375,9774,9778,9781,10390,10394,10397,11397,11400,11403,11693,11696,11853,11869,12462,12465,13169,13172,13350,13353,13358,13363],[10,975,976],{"id":976},"背景",[14,978,979],{},"距离首次学习 Transformer 已过去一年，最近在尝试回忆它的细节时，发现绝大部分内容已经模糊，于是决定系统地复盘一遍，并把过程记录下来，方便日后查阅。",[14,981,982,983,988,989,222],{},"本文重点记录对 Transformer 的理解与思考。代码实现主要参考 ",[18,984,987],{"href":985,"rel":986},"https://zh.d2l.ai/chapter_attention-mechanisms/transformer.html",[22],"动手学深度学习","，部分理论解释参考 ",[18,990,993],{"href":991,"rel":992},"https://nlp.seas.harvard.edu/annotated-transformer/",[22],"The Annotated Transformer",[10,995,996],{"id":996},"整体框架",[14,998,999],{},[1000,1001],"img",{"alt":1002,"src":1003},"transformer架构图","transformer/transformer.png",[10,1005,1006],{"id":1006},"注意力机制",[248,1008,1009],{"id":1009},"注意力函数",[14,1011,1012],{},"常见的注意力函数有两种：加性注意力（additive attention）与缩放点积注意力（scaled dot-product attention）。",[14,1014,1015,1016,1270],{},"缩放点积注意力相比朴素的点积注意力，多了一个缩放因子 ",[57,1017,1020,1058],{"className":1018},[1019],"katex",[57,1021,1024],{"className":1022},[1023],"katex-mathml",[1025,1026,1028],"math",{"xmlns":1027},"http://www.w3.org/1998/Math/MathML",[1029,1030,1031,1053],"semantics",{},[1032,1033,1034],"mrow",{},[1035,1036,1037,1040],"mfrac",{},[1038,1039,574],"mn",{},[1041,1042,1043],"msqrt",{},[1044,1045,1046,1050],"msub",{},[1047,1048,1049],"mi",{},"d",[1047,1051,1052],{},"k",[1054,1055,1057],"annotation",{"encoding":1056},"application/x-tex","\\frac{1}{\\sqrt{d_k}}",[57,1059,1063],{"className":1060,"ariaHidden":1062},[1061],"katex-html","true",[57,1064,1067,1072],{"className":1065},[1066],"base",[57,1068],{"className":1069,"style":1071},[1070],"strut","height:1.3831em;vertical-align:-0.538em;",[57,1073,1076,1081,1266],{"className":1074},[1075],"mord",[57,1077],{"className":1078},[1079,1080],"mopen","nulldelimiter",[57,1082,1084],{"className":1083},[1035],[57,1085,1089,1257],{"className":1086},[1087,1088],"vlist-t","vlist-t2",[57,1090,1093,1254],{"className":1091},[1092],"vlist-r",[57,1094,1098,1228,1239],{"className":1095,"style":1097},[1096],"vlist","height:0.8451em;",[57,1099,1101,1106],{"style":1100},"top:-2.5864em;",[57,1102],{"className":1103,"style":1105},[1104],"pstrut","height:3em;",[57,1107,1113],{"className":1108},[1109,1110,1111,1112],"sizing","reset-size6","size3","mtight",[57,1114,1116],{"className":1115},[1075,1112],[57,1117,1120],{"className":1118},[1075,1119,1112],"sqrt",[57,1121,1123,1219],{"className":1122},[1087,1088],[57,1124,1126,1216],{"className":1125},[1092],[57,1127,1130,1193],{"className":1128,"style":1129},[1096],"height:0.8622em;",[57,1131,1135,1138],{"className":1132,"style":1134},[1133],"svg-align","top:-3em;",[57,1136],{"className":1137,"style":1105},[1104],[57,1139,1142],{"className":1140,"style":1141},[1075,1112],"padding-left:0.833em;",[57,1143,1145,1149],{"className":1144},[1075,1112],[57,1146,1049],{"className":1147},[1075,1148,1112],"mathnormal",[57,1150,1153],{"className":1151},[1152],"msupsub",[57,1154,1156,1184],{"className":1155},[1087,1088],[57,1157,1159,1179],{"className":1158},[1092],[57,1160,1163],{"className":1161,"style":1162},[1096],"height:0.3448em;",[57,1164,1166,1170],{"style":1165},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[57,1167],{"className":1168,"style":1169},[1104],"height:2.5em;",[57,1171,1175],{"className":1172},[1109,1173,1174,1112],"reset-size3","size1",[57,1176,1052],{"className":1177,"style":1178},[1075,1148,1112],"margin-right:0.03148em;",[57,1180,1183],{"className":1181},[1182],"vlist-s","​",[57,1185,1187],{"className":1186},[1092],[57,1188,1191],{"className":1189,"style":1190},[1096],"height:0.1512em;",[57,1192],{},[57,1194,1196,1199],{"style":1195},"top:-2.8222em;",[57,1197],{"className":1198,"style":1105},[1104],[57,1200,1204],{"className":1201,"style":1203},[1202,1112],"hide-tail","min-width:0.853em;height:1.08em;",[1205,1206,1212],"svg",{"xmlns":1207,"width":1208,"height":1209,"viewBox":1210,"preserveAspectRatio":1211},"http://www.w3.org/2000/svg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[1213,1214],"path",{"d":1215},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[57,1217,1183],{"className":1218},[1182],[57,1220,1222],{"className":1221},[1092],[57,1223,1226],{"className":1224,"style":1225},[1096],"height:0.1778em;",[57,1227],{},[57,1229,1231,1234],{"style":1230},"top:-3.23em;",[57,1232],{"className":1233,"style":1105},[1104],[57,1235],{"className":1236,"style":1238},[1237],"frac-line","border-bottom-width:0.04em;",[57,1240,1242,1245],{"style":1241},"top:-3.394em;",[57,1243],{"className":1244,"style":1105},[1104],[57,1246,1248],{"className":1247},[1109,1110,1111,1112],[57,1249,1251],{"className":1250},[1075,1112],[57,1252,574],{"className":1253},[1075,1112],[57,1255,1183],{"className":1256},[1182],[57,1258,1260],{"className":1259},[1092],[57,1261,1264],{"className":1262,"style":1263},[1096],"height:0.538em;",[57,1265],{},[57,1267],{"className":1268},[1269,1080],"mclose","。加性注意力则通过一个带单隐藏层的前馈网络来计算兼容性分数。两者在理论复杂度上接近，但点积注意力可以直接复用高度优化的矩阵乘法实现，因此在实际中更快、也更省显存。",[14,1272,1273,1274,1304,1305,1334,1335,1364,1365,222],{},"为了贴近实际工程的批处理方式，下面以 minibatch 的形式描述：给定 ",[57,1275,1277,1291],{"className":1276},[1019],[57,1278,1280],{"className":1279},[1023],[1025,1281,1282],{"xmlns":1027},[1029,1283,1284,1289],{},[1032,1285,1286],{},[1047,1287,1288],{},"n",[1054,1290,1288],{"encoding":1056},[57,1292,1294],{"className":1293,"ariaHidden":1062},[1061],[57,1295,1297,1301],{"className":1296},[1066],[57,1298],{"className":1299,"style":1300},[1070],"height:0.4306em;",[57,1302,1288],{"className":1303},[1075,1148]," 个查询和 ",[57,1306,1308,1322],{"className":1307},[1019],[57,1309,1311],{"className":1310},[1023],[1025,1312,1313],{"xmlns":1027},[1029,1314,1315,1320],{},[1032,1316,1317],{},[1047,1318,1319],{},"m",[1054,1321,1319],{"encoding":1056},[57,1323,1325],{"className":1324,"ariaHidden":1062},[1061],[57,1326,1328,1331],{"className":1327},[1066],[57,1329],{"className":1330,"style":1300},[1070],[57,1332,1319],{"className":1333},[1075,1148]," 个键-值对，查询与键的维度均为 ",[57,1336,1338,1351],{"className":1337},[1019],[57,1339,1341],{"className":1340},[1023],[1025,1342,1343],{"xmlns":1027},[1029,1344,1345,1349],{},[1032,1346,1347],{},[1047,1348,1049],{},[1054,1350,1049],{"encoding":1056},[57,1352,1354],{"className":1353,"ariaHidden":1062},[1061],[57,1355,1357,1361],{"className":1356},[1066],[57,1358],{"className":1359,"style":1360},[1070],"height:0.6944em;",[57,1362,1049],{"className":1363},[1075,1148],"，值的维度为 ",[57,1366,1368,1382],{"className":1367},[1019],[57,1369,1371],{"className":1370},[1023],[1025,1372,1373],{"xmlns":1027},[1029,1374,1375,1380],{},[1032,1376,1377],{},[1047,1378,1379],{},"v",[1054,1381,1379],{"encoding":1056},[57,1383,1385],{"className":1384,"ariaHidden":1062},[1061],[57,1386,1388,1391],{"className":1387},[1066],[57,1389],{"className":1390,"style":1300},[1070],[57,1392,1379],{"className":1393,"style":1394},[1075,1148],"margin-right:0.03588em;",[14,1396,1397,1398,1510,1511,1610,1611,1710],{},"查询 ",[57,1399,1401,1436],{"className":1400},[1019],[57,1402,1404],{"className":1403},[1023],[1025,1405,1406],{"xmlns":1027},[1029,1407,1408,1433],{},[1032,1409,1410,1413,1417],{},[1047,1411,1412],{},"Q",[1414,1415,1416],"mo",{},"∈",[1418,1419,1420,1424],"msup",{},[1047,1421,1423],{"mathvariant":1422},"double-struck","R",[1032,1425,1426,1428,1431],{},[1047,1427,1288],{},[1414,1429,1430],{},"×",[1047,1432,1049],{},[1054,1434,1435],{"encoding":1056},"Q \\in \\mathbb{R}^{n \\times d}",[57,1437,1439,1461],{"className":1438,"ariaHidden":1062},[1061],[57,1440,1442,1446,1449,1454,1458],{"className":1441},[1066],[57,1443],{"className":1444,"style":1445},[1070],"height:0.8778em;vertical-align:-0.1944em;",[57,1447,1412],{"className":1448},[1075,1148],[57,1450],{"className":1451,"style":1453},[1452],"mspace","margin-right:0.2778em;",[57,1455,1416],{"className":1456},[1457],"mrel",[57,1459],{"className":1460,"style":1453},[1452],[57,1462,1464,1468],{"className":1463},[1066],[57,1465],{"className":1466,"style":1467},[1070],"height:0.8491em;",[57,1469,1471,1475],{"className":1470},[1075],[57,1472,1423],{"className":1473},[1075,1474],"mathbb",[57,1476,1478],{"className":1477},[1152],[57,1479,1481],{"className":1480},[1087],[57,1482,1484],{"className":1483},[1092],[57,1485,1487],{"className":1486,"style":1467},[1096],[57,1488,1490,1494],{"style":1489},"top:-3.063em;margin-right:0.05em;",[57,1491],{"className":1492,"style":1493},[1104],"height:2.7em;",[57,1495,1497],{"className":1496},[1109,1110,1111,1112],[57,1498,1500,1503,1507],{"className":1499},[1075,1112],[57,1501,1288],{"className":1502},[1075,1148,1112],[57,1504,1430],{"className":1505},[1506,1112],"mbin",[57,1508,1049],{"className":1509},[1075,1148,1112],"、键 ",[57,1512,1514,1543],{"className":1513},[1019],[57,1515,1517],{"className":1516},[1023],[1025,1518,1519],{"xmlns":1027},[1029,1520,1521,1540],{},[1032,1522,1523,1526,1528],{},[1047,1524,1525],{},"K",[1414,1527,1416],{},[1418,1529,1530,1532],{},[1047,1531,1423],{"mathvariant":1422},[1032,1533,1534,1536,1538],{},[1047,1535,1319],{},[1414,1537,1430],{},[1047,1539,1049],{},[1054,1541,1542],{"encoding":1056},"K \\in \\mathbb{R}^{m \\times d}",[57,1544,1546,1566],{"className":1545,"ariaHidden":1062},[1061],[57,1547,1549,1553,1557,1560,1563],{"className":1548},[1066],[57,1550],{"className":1551,"style":1552},[1070],"height:0.7224em;vertical-align:-0.0391em;",[57,1554,1525],{"className":1555,"style":1556},[1075,1148],"margin-right:0.07153em;",[57,1558],{"className":1559,"style":1453},[1452],[57,1561,1416],{"className":1562},[1457],[57,1564],{"className":1565,"style":1453},[1452],[57,1567,1569,1572],{"className":1568},[1066],[57,1570],{"className":1571,"style":1467},[1070],[57,1573,1575,1578],{"className":1574},[1075],[57,1576,1423],{"className":1577},[1075,1474],[57,1579,1581],{"className":1580},[1152],[57,1582,1584],{"className":1583},[1087],[57,1585,1587],{"className":1586},[1092],[57,1588,1590],{"className":1589,"style":1467},[1096],[57,1591,1592,1595],{"style":1489},[57,1593],{"className":1594,"style":1493},[1104],[57,1596,1598],{"className":1597},[1109,1110,1111,1112],[57,1599,1601,1604,1607],{"className":1600},[1075,1112],[57,1602,1319],{"className":1603},[1075,1148,1112],[57,1605,1430],{"className":1606},[1506,1112],[57,1608,1049],{"className":1609},[1075,1148,1112]," 与值 ",[57,1612,1614,1643],{"className":1613},[1019],[57,1615,1617],{"className":1616},[1023],[1025,1618,1619],{"xmlns":1027},[1029,1620,1621,1640],{},[1032,1622,1623,1626,1628],{},[1047,1624,1625],{},"V",[1414,1627,1416],{},[1418,1629,1630,1632],{},[1047,1631,1423],{"mathvariant":1422},[1032,1633,1634,1636,1638],{},[1047,1635,1319],{},[1414,1637,1430],{},[1047,1639,1379],{},[1054,1641,1642],{"encoding":1056},"V \\in \\mathbb{R}^{m \\times v}",[57,1644,1646,1665],{"className":1645,"ariaHidden":1062},[1061],[57,1647,1649,1652,1656,1659,1662],{"className":1648},[1066],[57,1650],{"className":1651,"style":1552},[1070],[57,1653,1625],{"className":1654,"style":1655},[1075,1148],"margin-right:0.22222em;",[57,1657],{"className":1658,"style":1453},[1452],[57,1660,1416],{"className":1661},[1457],[57,1663],{"className":1664,"style":1453},[1452],[57,1666,1668,1672],{"className":1667},[1066],[57,1669],{"className":1670,"style":1671},[1070],"height:0.7713em;",[57,1673,1675,1678],{"className":1674},[1075],[57,1676,1423],{"className":1677},[1075,1474],[57,1679,1681],{"className":1680},[1152],[57,1682,1684],{"className":1683},[1087],[57,1685,1687],{"className":1686},[1092],[57,1688,1690],{"className":1689,"style":1671},[1096],[57,1691,1692,1695],{"style":1489},[57,1693],{"className":1694,"style":1493},[1104],[57,1696,1698],{"className":1697},[1109,1110,1111,1112],[57,1699,1701,1704,1707],{"className":1700},[1075,1112],[57,1702,1319],{"className":1703},[1075,1148,1112],[57,1705,1430],{"className":1706},[1506,1112],[57,1708,1379],{"className":1709,"style":1394},[1075,1148,1112]," 的缩放点积注意力定义为：",[14,1712,1713],{},[57,1714,1716,1822],{"className":1715},[1019],[57,1717,1719],{"className":1718},[1023],[1025,1720,1721],{"xmlns":1027},[1029,1722,1723,1819],{},[1032,1724,1725,1751,1755,1757,1760,1762,1764,1766,1769,1771,1774,1777,1800,1802,1804,1816],{},[1032,1726,1727,1731,1734,1736,1739,1741,1743,1746,1749],{},[1047,1728,1730],{"mathvariant":1729},"normal","A",[1047,1732,1733],{"mathvariant":1729},"t",[1047,1735,1733],{"mathvariant":1729},[1047,1737,1738],{"mathvariant":1729},"e",[1047,1740,1288],{"mathvariant":1729},[1047,1742,1733],{"mathvariant":1729},[1047,1744,1745],{"mathvariant":1729},"i",[1047,1747,1748],{"mathvariant":1729},"o",[1047,1750,1288],{"mathvariant":1729},[1414,1752,1754],{"stretchy":1753},"false","(",[1047,1756,1412],{},[1414,1758,1759],{"separator":1062},",",[1047,1761,1525],{},[1414,1763,1759],{"separator":1062},[1047,1765,1625],{},[1414,1767,1768],{"stretchy":1753},")",[1414,1770,281],{},[1047,1772,1773],{"mathvariant":1729},"softmax",[1414,1775,1776],{},"⁡",[1032,1778,1779,1781,1798],{},[1414,1780,1754],{"fence":1062},[1035,1782,1783,1794],{},[1032,1784,1785,1787],{},[1047,1786,1412],{},[1418,1788,1789,1791],{},[1047,1790,1525],{},[1047,1792,1793],{"mathvariant":1729},"⊤",[1041,1795,1796],{},[1047,1797,1049],{},[1414,1799,1768],{"fence":1062},[1047,1801,1625],{},[1414,1803,1416],{},[1418,1805,1806,1808],{},[1047,1807,1423],{"mathvariant":1422},[1032,1809,1810,1812,1814],{},[1047,1811,1288],{},[1414,1813,1430],{},[1047,1815,1379],{},[1047,1817,1818],{"mathvariant":1729},".",[1054,1820,1821],{"encoding":1056}," \\mathrm{Attention}(Q,K,V) = \\operatorname{softmax}\\left(\\frac{QK^\\top}{\\sqrt{d}}\\right)V \\in \\mathbb{R}^{n \\times v}.",[57,1823,1825,1878,2079],{"className":1824,"ariaHidden":1062},[1061],[57,1826,1828,1832,1840,1843,1846,1850,1854,1857,1860,1863,1866,1869,1872,1875],{"className":1827},[1066],[57,1829],{"className":1830,"style":1831},[1070],"height:1em;vertical-align:-0.25em;",[57,1833,1835],{"className":1834},[1075],[57,1836,1839],{"className":1837},[1075,1838],"mathrm","Attention",[57,1841,1754],{"className":1842},[1079],[57,1844,1412],{"className":1845},[1075,1148],[57,1847,1759],{"className":1848},[1849],"mpunct",[57,1851],{"className":1852,"style":1853},[1452],"margin-right:0.1667em;",[57,1855,1525],{"className":1856,"style":1556},[1075,1148],[57,1858,1759],{"className":1859},[1849],[57,1861],{"className":1862,"style":1853},[1452],[57,1864,1625],{"className":1865,"style":1655},[1075,1148],[57,1867,1768],{"className":1868},[1269],[57,1870],{"className":1871,"style":1453},[1452],[57,1873,281],{"className":1874},[1457],[57,1876],{"className":1877,"style":1453},[1452],[57,1879,1881,1885,1892,1895,2064,2067,2070,2073,2076],{"className":1880},[1066],[57,1882],{"className":1883,"style":1884},[1070],"height:1.8em;vertical-align:-0.65em;",[57,1886,1889],{"className":1887},[1888],"mop",[57,1890,1773],{"className":1891},[1075,1838],[57,1893],{"className":1894,"style":1853},[1452],[57,1896,1899,1909,2058],{"className":1897},[1898],"minner",[57,1900,1904],{"className":1901,"style":1903},[1079,1902],"delimcenter","top:0em;",[57,1905,1754],{"className":1906},[1907,1908],"delimsizing","size2",[57,1910,1912,1915,2055],{"className":1911},[1075],[57,1913],{"className":1914},[1079,1080],[57,1916,1918],{"className":1917},[1035],[57,1919,1921,2047],{"className":1920},[1087,1088],[57,1922,1924,2044],{"className":1923},[1092],[57,1925,1928,1990,1998],{"className":1926,"style":1927},[1096],"height:1.095em;",[57,1929,1931,1934],{"style":1930},"top:-2.5335em;",[57,1932],{"className":1933,"style":1105},[1104],[57,1935,1937],{"className":1936},[1109,1110,1111,1112],[57,1938,1940],{"className":1939},[1075,1112],[57,1941,1943],{"className":1942},[1075,1119,1112],[57,1944,1946,1981],{"className":1945},[1087,1088],[57,1947,1949,1978],{"className":1948},[1092],[57,1950,1953,1965],{"className":1951,"style":1952},[1096],"height:0.9378em;",[57,1954,1956,1959],{"className":1955,"style":1134},[1133],[57,1957],{"className":1958,"style":1105},[1104],[57,1960,1962],{"className":1961,"style":1141},[1075,1112],[57,1963,1049],{"className":1964},[1075,1148,1112],[57,1966,1968,1971],{"style":1967},"top:-2.8978em;",[57,1969],{"className":1970,"style":1105},[1104],[57,1972,1974],{"className":1973,"style":1203},[1202,1112],[1205,1975,1976],{"xmlns":1207,"width":1208,"height":1209,"viewBox":1210,"preserveAspectRatio":1211},[1213,1977],{"d":1215},[57,1979,1183],{"className":1980},[1182],[57,1982,1984],{"className":1983},[1092],[57,1985,1988],{"className":1986,"style":1987},[1096],"height:0.1022em;",[57,1989],{},[57,1991,1992,1995],{"style":1230},[57,1993],{"className":1994,"style":1105},[1104],[57,1996],{"className":1997,"style":1238},[1237],[57,1999,2001,2004],{"style":2000},"top:-3.4461em;",[57,2002],{"className":2003,"style":1105},[1104],[57,2005,2007],{"className":2006},[1109,1110,1111,1112],[57,2008,2010,2013],{"className":2009},[1075,1112],[57,2011,1412],{"className":2012},[1075,1148,1112],[57,2014,2016,2019],{"className":2015},[1075,1112],[57,2017,1525],{"className":2018,"style":1556},[1075,1148,1112],[57,2020,2022],{"className":2021},[1152],[57,2023,2025],{"className":2024},[1087],[57,2026,2028],{"className":2027},[1092],[57,2029,2032],{"className":2030,"style":2031},[1096],"height:0.927em;",[57,2033,2035,2038],{"style":2034},"top:-2.931em;margin-right:0.0714em;",[57,2036],{"className":2037,"style":1169},[1104],[57,2039,2041],{"className":2040},[1109,1173,1174,1112],[57,2042,1793],{"className":2043},[1075,1112],[57,2045,1183],{"className":2046},[1182],[57,2048,2050],{"className":2049},[1092],[57,2051,2053],{"className":2052,"style":1263},[1096],[57,2054],{},[57,2056],{"className":2057},[1269,1080],[57,2059,2061],{"className":2060,"style":1903},[1269,1902],[57,2062,1768],{"className":2063},[1907,1908],[57,2065],{"className":2066,"style":1853},[1452],[57,2068,1625],{"className":2069,"style":1655},[1075,1148],[57,2071],{"className":2072,"style":1453},[1452],[57,2074,1416],{"className":2075},[1457],[57,2077],{"className":2078,"style":1453},[1452],[57,2080,2082,2085,2123],{"className":2081},[1066],[57,2083],{"className":2084,"style":1671},[1070],[57,2086,2088,2091],{"className":2087},[1075],[57,2089,1423],{"className":2090},[1075,1474],[57,2092,2094],{"className":2093},[1152],[57,2095,2097],{"className":2096},[1087],[57,2098,2100],{"className":2099},[1092],[57,2101,2103],{"className":2102,"style":1671},[1096],[57,2104,2105,2108],{"style":1489},[57,2106],{"className":2107,"style":1493},[1104],[57,2109,2111],{"className":2110},[1109,1110,1111,1112],[57,2112,2114,2117,2120],{"className":2113},[1075,1112],[57,2115,1288],{"className":2116},[1075,1148,1112],[57,2118,1430],{"className":2119},[1506,1112],[57,2121,1379],{"className":2122,"style":1394},[1075,1148,1112],[57,2124,1818],{"className":2125},[1075],[14,2127,2128],{},[1000,2129],{"alt":2130,"src":2131},"注意力","transformer/attention.png",[14,2133,2134,2135,2258],{},"具体过程是：先用查询与所有键做点积，再除以 ",[57,2136,2138,2158],{"className":2137},[1019],[57,2139,2141],{"className":2140},[1023],[1025,2142,2143],{"xmlns":1027},[1029,2144,2145,2155],{},[1032,2146,2147],{},[1041,2148,2149],{},[1044,2150,2151,2153],{},[1047,2152,1049],{},[1047,2154,1052],{},[1054,2156,2157],{"encoding":1056},"\\sqrt{d_k}",[57,2159,2161],{"className":2160,"ariaHidden":1062},[1061],[57,2162,2164,2168],{"className":2163},[1066],[57,2165],{"className":2166,"style":2167},[1070],"height:1.04em;vertical-align:-0.1828em;",[57,2169,2171],{"className":2170},[1075,1119],[57,2172,2174,2249],{"className":2173},[1087,1088],[57,2175,2177,2246],{"className":2176},[1092],[57,2178,2181,2233],{"className":2179,"style":2180},[1096],"height:0.8572em;",[57,2182,2184,2187],{"className":2183,"style":1134},[1133],[57,2185],{"className":2186,"style":1105},[1104],[57,2188,2190],{"className":2189,"style":1141},[1075],[57,2191,2193,2196],{"className":2192},[1075],[57,2194,1049],{"className":2195},[1075,1148],[57,2197,2199],{"className":2198},[1152],[57,2200,2202,2224],{"className":2201},[1087,1088],[57,2203,2205,2221],{"className":2204},[1092],[57,2206,2209],{"className":2207,"style":2208},[1096],"height:0.3361em;",[57,2210,2212,2215],{"style":2211},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[57,2213],{"className":2214,"style":1493},[1104],[57,2216,2218],{"className":2217},[1109,1110,1111,1112],[57,2219,1052],{"className":2220,"style":1178},[1075,1148,1112],[57,2222,1183],{"className":2223},[1182],[57,2225,2227],{"className":2226},[1092],[57,2228,2231],{"className":2229,"style":2230},[1096],"height:0.15em;",[57,2232],{},[57,2234,2236,2239],{"style":2235},"top:-2.8172em;",[57,2237],{"className":2238,"style":1105},[1104],[57,2240,2242],{"className":2241,"style":1203},[1202],[1205,2243,2244],{"xmlns":1207,"width":1208,"height":1209,"viewBox":1210,"preserveAspectRatio":1211},[1213,2245],{"d":1215},[57,2247,1183],{"className":2248},[1182],[57,2250,2252],{"className":2251},[1092],[57,2253,2256],{"className":2254,"style":2255},[1096],"height:0.1828em;",[57,2257],{}," 进行缩放，然后通过 softmax 得到值上的权重分布，最后对值做加权求和。",[14,2260,2261,2262,2333,2334,2403,2404,2434,2435,2463,2464,2493,2494,2522,2523,2820,2821,2849,2850,2919,2920,2948,2949,3133],{},"那么，为什么需要这个缩放因子呢？对较小的 ",[57,2263,2265,2283],{"className":2264},[1019],[57,2266,2268],{"className":2267},[1023],[1025,2269,2270],{"xmlns":1027},[1029,2271,2272,2280],{},[1032,2273,2274],{},[1044,2275,2276,2278],{},[1047,2277,1049],{},[1047,2279,1052],{},[1054,2281,2282],{"encoding":1056},"d_k",[57,2284,2286],{"className":2285,"ariaHidden":1062},[1061],[57,2287,2289,2293],{"className":2288},[1066],[57,2290],{"className":2291,"style":2292},[1070],"height:0.8444em;vertical-align:-0.15em;",[57,2294,2296,2299],{"className":2295},[1075],[57,2297,1049],{"className":2298},[1075,1148],[57,2300,2302],{"className":2301},[1152],[57,2303,2305,2325],{"className":2304},[1087,1088],[57,2306,2308,2322],{"className":2307},[1092],[57,2309,2311],{"className":2310,"style":2208},[1096],[57,2312,2313,2316],{"style":2211},[57,2314],{"className":2315,"style":1493},[1104],[57,2317,2319],{"className":2318},[1109,1110,1111,1112],[57,2320,1052],{"className":2321,"style":1178},[1075,1148,1112],[57,2323,1183],{"className":2324},[1182],[57,2326,2328],{"className":2327},[1092],[57,2329,2331],{"className":2330,"style":2230},[1096],[57,2332],{},"，加性注意力与点积注意力表现接近；但当 ",[57,2335,2337,2354],{"className":2336},[1019],[57,2338,2340],{"className":2339},[1023],[1025,2341,2342],{"xmlns":1027},[1029,2343,2344,2352],{},[1032,2345,2346],{},[1044,2347,2348,2350],{},[1047,2349,1049],{},[1047,2351,1052],{},[1054,2353,2282],{"encoding":1056},[57,2355,2357],{"className":2356,"ariaHidden":1062},[1061],[57,2358,2360,2363],{"className":2359},[1066],[57,2361],{"className":2362,"style":2292},[1070],[57,2364,2366,2369],{"className":2365},[1075],[57,2367,1049],{"className":2368},[1075,1148],[57,2370,2372],{"className":2371},[1152],[57,2373,2375,2395],{"className":2374},[1087,1088],[57,2376,2378,2392],{"className":2377},[1092],[57,2379,2381],{"className":2380,"style":2208},[1096],[57,2382,2383,2386],{"style":2211},[57,2384],{"className":2385,"style":1493},[1104],[57,2387,2389],{"className":2388},[1109,1110,1111,1112],[57,2390,1052],{"className":2391,"style":1178},[1075,1148,1112],[57,2393,1183],{"className":2394},[1182],[57,2396,2398],{"className":2397},[1092],[57,2399,2401],{"className":2400,"style":2230},[1096],[57,2402],{}," 较大时，加性注意力反而更优。原因在于：当维度很高时，点积的数值会被放大，导致 softmax 的输出几乎集中在一个位置上，梯度也随之消失。可以做一个简化的方差分析：假设查询 ",[57,2405,2407,2421],{"className":2406},[1019],[57,2408,2410],{"className":2409},[1023],[1025,2411,2412],{"xmlns":1027},[1029,2413,2414,2419],{},[1032,2415,2416],{},[1047,2417,2418],{},"q",[1054,2420,2418],{"encoding":1056},[57,2422,2424],{"className":2423,"ariaHidden":1062},[1061],[57,2425,2427,2431],{"className":2426},[1066],[57,2428],{"className":2429,"style":2430},[1070],"height:0.625em;vertical-align:-0.1944em;",[57,2432,2418],{"className":2433,"style":1394},[1075,1148]," 与键 ",[57,2436,2438,2451],{"className":2437},[1019],[57,2439,2441],{"className":2440},[1023],[1025,2442,2443],{"xmlns":1027},[1029,2444,2445,2449],{},[1032,2446,2447],{},[1047,2448,1052],{},[1054,2450,1052],{"encoding":1056},[57,2452,2454],{"className":2453,"ariaHidden":1062},[1061],[57,2455,2457,2460],{"className":2456},[1066],[57,2458],{"className":2459,"style":1360},[1070],[57,2461,1052],{"className":2462,"style":1178},[1075,1148]," 的每个分量都是均值 ",[57,2465,2467,2480],{"className":2466},[1019],[57,2468,2470],{"className":2469},[1023],[1025,2471,2472],{"xmlns":1027},[1029,2473,2474,2478],{},[1032,2475,2476],{},[1038,2477,560],{},[1054,2479,560],{"encoding":1056},[57,2481,2483],{"className":2482,"ariaHidden":1062},[1061],[57,2484,2486,2490],{"className":2485},[1066],[57,2487],{"className":2488,"style":2489},[1070],"height:0.6444em;",[57,2491,560],{"className":2492},[1075],"、方差 ",[57,2495,2497,2510],{"className":2496},[1019],[57,2498,2500],{"className":2499},[1023],[1025,2501,2502],{"xmlns":1027},[1029,2503,2504,2508],{},[1032,2505,2506],{},[1038,2507,574],{},[1054,2509,574],{"encoding":1056},[57,2511,2513],{"className":2512,"ariaHidden":1062},[1061],[57,2514,2516,2519],{"className":2515},[1066],[57,2517],{"className":2518,"style":2489},[1070],[57,2520,574],{"className":2521},[1075]," 的独立随机变量，则它们的点积 ",[57,2524,2526,2579],{"className":2525},[1019],[57,2527,2529],{"className":2528},[1023],[1025,2530,2531],{"xmlns":1027},[1029,2532,2533,2576],{},[1032,2534,2535,2537,2540,2542,2544,2564,2570],{},[1047,2536,2418],{},[1414,2538,2539],{},"⋅",[1047,2541,1052],{},[1414,2543,281],{},[2545,2546,2547,2550,2558],"msubsup",{},[1414,2548,2549],{},"∑",[1032,2551,2552,2554,2556],{},[1047,2553,1745],{},[1414,2555,281],{},[1038,2557,574],{},[1044,2559,2560,2562],{},[1047,2561,1049],{},[1047,2563,1052],{},[1044,2565,2566,2568],{},[1047,2567,2418],{},[1047,2569,1745],{},[1044,2571,2572,2574],{},[1047,2573,1052],{},[1047,2575,1745],{},[1054,2577,2578],{"encoding":1056},"q \\cdot k = \\sum_{i=1}^{d_k} q_i k_i",[57,2580,2582,2602,2620],{"className":2581,"ariaHidden":1062},[1061],[57,2583,2585,2589,2592,2596,2599],{"className":2584},[1066],[57,2586],{"className":2587,"style":2588},[1070],"height:0.6389em;vertical-align:-0.1944em;",[57,2590,2418],{"className":2591,"style":1394},[1075,1148],[57,2593],{"className":2594,"style":2595},[1452],"margin-right:0.2222em;",[57,2597,2539],{"className":2598},[1506],[57,2600],{"className":2601,"style":2595},[1452],[57,2603,2605,2608,2611,2614,2617],{"className":2604},[1066],[57,2606],{"className":2607,"style":1360},[1070],[57,2609,1052],{"className":2610,"style":1178},[1075,1148],[57,2612],{"className":2613,"style":1453},[1452],[57,2615,281],{"className":2616},[1457],[57,2618],{"className":2619,"style":1453},[1452],[57,2621,2623,2627,2734,2737,2779],{"className":2622},[1066],[57,2624],{"className":2625,"style":2626},[1070],"height:1.2887em;vertical-align:-0.2997em;",[57,2628,2630,2636],{"className":2629},[1888],[57,2631,2549],{"className":2632,"style":2635},[1888,2633,2634],"op-symbol","small-op","position:relative;top:0em;",[57,2637,2639],{"className":2638},[1152],[57,2640,2642,2725],{"className":2641},[1087,1088],[57,2643,2645,2722],{"className":2644},[1092],[57,2646,2649,2670],{"className":2647,"style":2648},[1096],"height:0.989em;",[57,2650,2652,2655],{"style":2651},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[57,2653],{"className":2654,"style":1493},[1104],[57,2656,2658],{"className":2657},[1109,1110,1111,1112],[57,2659,2661,2664,2667],{"className":2660},[1075,1112],[57,2662,1745],{"className":2663},[1075,1148,1112],[57,2665,281],{"className":2666},[1457,1112],[57,2668,574],{"className":2669},[1075,1112],[57,2671,2673,2676],{"style":2672},"top:-3.2029em;margin-right:0.05em;",[57,2674],{"className":2675,"style":1493},[1104],[57,2677,2679],{"className":2678},[1109,1110,1111,1112],[57,2680,2682],{"className":2681},[1075,1112],[57,2683,2685,2688],{"className":2684},[1075,1112],[57,2686,1049],{"className":2687},[1075,1148,1112],[57,2689,2691],{"className":2690},[1152],[57,2692,2694,2714],{"className":2693},[1087,1088],[57,2695,2697,2711],{"className":2696},[1092],[57,2698,2700],{"className":2699,"style":1162},[1096],[57,2701,2702,2705],{"style":1165},[57,2703],{"className":2704,"style":1169},[1104],[57,2706,2708],{"className":2707},[1109,1173,1174,1112],[57,2709,1052],{"className":2710,"style":1178},[1075,1148,1112],[57,2712,1183],{"className":2713},[1182],[57,2715,2717],{"className":2716},[1092],[57,2718,2720],{"className":2719,"style":1190},[1096],[57,2721],{},[57,2723,1183],{"className":2724},[1182],[57,2726,2728],{"className":2727},[1092],[57,2729,2732],{"className":2730,"style":2731},[1096],"height:0.2997em;",[57,2733],{},[57,2735],{"className":2736,"style":1853},[1452],[57,2738,2740,2743],{"className":2739},[1075],[57,2741,2418],{"className":2742,"style":1394},[1075,1148],[57,2744,2746],{"className":2745},[1152],[57,2747,2749,2771],{"className":2748},[1087,1088],[57,2750,2752,2768],{"className":2751},[1092],[57,2753,2756],{"className":2754,"style":2755},[1096],"height:0.3117em;",[57,2757,2759,2762],{"style":2758},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[57,2760],{"className":2761,"style":1493},[1104],[57,2763,2765],{"className":2764},[1109,1110,1111,1112],[57,2766,1745],{"className":2767},[1075,1148,1112],[57,2769,1183],{"className":2770},[1182],[57,2772,2774],{"className":2773},[1092],[57,2775,2777],{"className":2776,"style":2230},[1096],[57,2778],{},[57,2780,2782,2785],{"className":2781},[1075],[57,2783,1052],{"className":2784,"style":1178},[1075,1148],[57,2786,2788],{"className":2787},[1152],[57,2789,2791,2812],{"className":2790},[1087,1088],[57,2792,2794,2809],{"className":2793},[1092],[57,2795,2797],{"className":2796,"style":2755},[1096],[57,2798,2800,2803],{"style":2799},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[57,2801],{"className":2802,"style":1493},[1104],[57,2804,2806],{"className":2805},[1109,1110,1111,1112],[57,2807,1745],{"className":2808},[1075,1148,1112],[57,2810,1183],{"className":2811},[1182],[57,2813,2815],{"className":2814},[1092],[57,2816,2818],{"className":2817,"style":2230},[1096],[57,2819],{}," 的均值为 ",[57,2822,2824,2837],{"className":2823},[1019],[57,2825,2827],{"className":2826},[1023],[1025,2828,2829],{"xmlns":1027},[1029,2830,2831,2835],{},[1032,2832,2833],{},[1038,2834,560],{},[1054,2836,560],{"encoding":1056},[57,2838,2840],{"className":2839,"ariaHidden":1062},[1061],[57,2841,2843,2846],{"className":2842},[1066],[57,2844],{"className":2845,"style":2489},[1070],[57,2847,560],{"className":2848},[1075],"、方差为 ",[57,2851,2853,2870],{"className":2852},[1019],[57,2854,2856],{"className":2855},[1023],[1025,2857,2858],{"xmlns":1027},[1029,2859,2860,2868],{},[1032,2861,2862],{},[1044,2863,2864,2866],{},[1047,2865,1049],{},[1047,2867,1052],{},[1054,2869,2282],{"encoding":1056},[57,2871,2873],{"className":2872,"ariaHidden":1062},[1061],[57,2874,2876,2879],{"className":2875},[1066],[57,2877],{"className":2878,"style":2292},[1070],[57,2880,2882,2885],{"className":2881},[1075],[57,2883,1049],{"className":2884},[1075,1148],[57,2886,2888],{"className":2887},[1152],[57,2889,2891,2911],{"className":2890},[1087,1088],[57,2892,2894,2908],{"className":2893},[1092],[57,2895,2897],{"className":2896,"style":2208},[1096],[57,2898,2899,2902],{"style":2211},[57,2900],{"className":2901,"style":1493},[1104],[57,2903,2905],{"className":2904},[1109,1110,1111,1112],[57,2906,1052],{"className":2907,"style":1178},[1075,1148,1112],[57,2909,1183],{"className":2910},[1182],[57,2912,2914],{"className":2913},[1092],[57,2915,2917],{"className":2916,"style":2230},[1096],[57,2918],{},"。为了把方差稳定回 ",[57,2921,2923,2936],{"className":2922},[1019],[57,2924,2926],{"className":2925},[1023],[1025,2927,2928],{"xmlns":1027},[1029,2929,2930,2934],{},[1032,2931,2932],{},[1038,2933,574],{},[1054,2935,574],{"encoding":1056},[57,2937,2939],{"className":2938,"ariaHidden":1062},[1061],[57,2940,2942,2945],{"className":2941},[1066],[57,2943],{"className":2944,"style":2489},[1070],[57,2946,574],{"className":2947},[1075],"，就需要乘上 ",[57,2950,2952,2975],{"className":2951},[1019],[57,2953,2955],{"className":2954},[1023],[1025,2956,2957],{"xmlns":1027},[1029,2958,2959,2973],{},[1032,2960,2961],{},[1035,2962,2963,2965],{},[1038,2964,574],{},[1041,2966,2967],{},[1044,2968,2969,2971],{},[1047,2970,1049],{},[1047,2972,1052],{},[1054,2974,1057],{"encoding":1056},[57,2976,2978],{"className":2977,"ariaHidden":1062},[1061],[57,2979,2981,2984],{"className":2980},[1066],[57,2982],{"className":2983,"style":1071},[1070],[57,2985,2987,2990,3130],{"className":2986},[1075],[57,2988],{"className":2989},[1079,1080],[57,2991,2993],{"className":2992},[1035],[57,2994,2996,3122],{"className":2995},[1087,1088],[57,2997,2999,3119],{"className":2998},[1092],[57,3000,3002,3097,3105],{"className":3001,"style":1097},[1096],[57,3003,3004,3007],{"style":1100},[57,3005],{"className":3006,"style":1105},[1104],[57,3008,3010],{"className":3009},[1109,1110,1111,1112],[57,3011,3013],{"className":3012},[1075,1112],[57,3014,3016],{"className":3015},[1075,1119,1112],[57,3017,3019,3089],{"className":3018},[1087,1088],[57,3020,3022,3086],{"className":3021},[1092],[57,3023,3025,3074],{"className":3024,"style":1129},[1096],[57,3026,3028,3031],{"className":3027,"style":1134},[1133],[57,3029],{"className":3030,"style":1105},[1104],[57,3032,3034],{"className":3033,"style":1141},[1075,1112],[57,3035,3037,3040],{"className":3036},[1075,1112],[57,3038,1049],{"className":3039},[1075,1148,1112],[57,3041,3043],{"className":3042},[1152],[57,3044,3046,3066],{"className":3045},[1087,1088],[57,3047,3049,3063],{"className":3048},[1092],[57,3050,3052],{"className":3051,"style":1162},[1096],[57,3053,3054,3057],{"style":1165},[57,3055],{"className":3056,"style":1169},[1104],[57,3058,3060],{"className":3059},[1109,1173,1174,1112],[57,3061,1052],{"className":3062,"style":1178},[1075,1148,1112],[57,3064,1183],{"className":3065},[1182],[57,3067,3069],{"className":3068},[1092],[57,3070,3072],{"className":3071,"style":1190},[1096],[57,3073],{},[57,3075,3076,3079],{"style":1195},[57,3077],{"className":3078,"style":1105},[1104],[57,3080,3082],{"className":3081,"style":1203},[1202,1112],[1205,3083,3084],{"xmlns":1207,"width":1208,"height":1209,"viewBox":1210,"preserveAspectRatio":1211},[1213,3085],{"d":1215},[57,3087,1183],{"className":3088},[1182],[57,3090,3092],{"className":3091},[1092],[57,3093,3095],{"className":3094,"style":1225},[1096],[57,3096],{},[57,3098,3099,3102],{"style":1230},[57,3100],{"className":3101,"style":1105},[1104],[57,3103],{"className":3104,"style":1238},[1237],[57,3106,3107,3110],{"style":1241},[57,3108],{"className":3109,"style":1105},[1104],[57,3111,3113],{"className":3112},[1109,1110,1111,1112],[57,3114,3116],{"className":3115},[1075,1112],[57,3117,574],{"className":3118},[1075,1112],[57,3120,1183],{"className":3121},[1182],[57,3123,3125],{"className":3124},[1092],[57,3126,3128],{"className":3127,"style":1263},[1096],[57,3129],{},[57,3131],{"className":3132},[1269,1080]," 这个缩放因子。",[14,3135,3136,3137,3140],{},"由于训练与推理时并非所有位置都应进入注意力汇聚（例如 padding 位置、解码器的未来位置），我们先实现一个 ",[54,3138,3139],{},"masked_softmax"," 函数，确保只有有效位置参与计算：",[47,3142,3144],{"className":256,"code":3143,"language":258,"meta":52,"style":52},"def masked_softmax(X, valid_lens):\n    \"\"\"通过在最后一个轴上掩蔽元素来执行softmax操作\"\"\"\n    # X:3D张量，valid_lens:1D或2D张量\n    if valid_lens is None:\n        return nn.functional.softmax(X, dim=-1)\n    else:\n        shape = X.shape\n        if valid_lens.dim() == 1:\n            valid_lens = torch.repeat_interleave(valid_lens, shape[1])\n        else:\n            valid_lens = valid_lens.reshape(-1)\n        # 最后一轴上被掩蔽的元素使用一个非常大的负值替换，从而其softmax输出为0\n        X = d2l.sequence_mask(X.reshape(-1, shape[-1]), valid_lens,\n                              value=-1e6)\n        return nn.functional.softmax(X.reshape(shape), dim=-1)\n",[54,3145,3146,3170,3176,3181,3198,3218,3225,3235,3251,3266,3273,3289,3294,3318,3330],{"__ignoreMap":52},[57,3147,3148,3151,3155,3157,3161,3164,3167],{"class":59,"line":60},[57,3149,3150],{"class":280},"def",[57,3152,3154],{"class":3153},"sCLZk"," masked_softmax",[57,3156,1754],{"class":269},[57,3158,3160],{"class":3159},"syNf4","X",[57,3162,3163],{"class":269},", ",[57,3165,3166],{"class":3159},"valid_lens",[57,3168,3169],{"class":269},"):\n",[57,3171,3172],{"class":59,"line":66},[57,3173,3175],{"class":3174},"seLWX","    \"\"\"通过在最后一个轴上掩蔽元素来执行softmax操作\"\"\"\n",[57,3177,3178],{"class":59,"line":72},[57,3179,3180],{"class":303},"    # X:3D张量，valid_lens:1D或2D张量\n",[57,3182,3183,3186,3189,3192,3195],{"class":59,"line":78},[57,3184,3185],{"class":280},"    if",[57,3187,3188],{"class":269}," valid_lens ",[57,3190,3191],{"class":280},"is",[57,3193,3194],{"class":273}," None",[57,3196,3197],{"class":269},":\n",[57,3199,3200,3203,3206,3210,3213,3215],{"class":59,"line":84},[57,3201,3202],{"class":280},"        return",[57,3204,3205],{"class":269}," nn.functional.softmax(X, ",[57,3207,3209],{"class":3208},"sQkXh","dim",[57,3211,3212],{"class":280},"=-",[57,3214,574],{"class":273},[57,3216,3217],{"class":269},")\n",[57,3219,3220,3223],{"class":59,"line":90},[57,3221,3222],{"class":280},"    else",[57,3224,3197],{"class":269},[57,3226,3227,3230,3232],{"class":59,"line":96},[57,3228,3229],{"class":269},"        shape ",[57,3231,281],{"class":280},[57,3233,3234],{"class":269}," X.shape\n",[57,3236,3237,3240,3243,3246,3249],{"class":59,"line":102},[57,3238,3239],{"class":280},"        if",[57,3241,3242],{"class":269}," valid_lens.dim() ",[57,3244,3245],{"class":280},"==",[57,3247,3248],{"class":273}," 1",[57,3250,3197],{"class":269},[57,3252,3253,3256,3258,3261,3263],{"class":59,"line":108},[57,3254,3255],{"class":269},"            valid_lens ",[57,3257,281],{"class":280},[57,3259,3260],{"class":269}," torch.repeat_interleave(valid_lens, shape[",[57,3262,574],{"class":273},[57,3264,3265],{"class":269},"])\n",[57,3267,3268,3271],{"class":59,"line":114},[57,3269,3270],{"class":280},"        else",[57,3272,3197],{"class":269},[57,3274,3275,3277,3279,3282,3285,3287],{"class":59,"line":120},[57,3276,3255],{"class":269},[57,3278,281],{"class":280},[57,3280,3281],{"class":269}," valid_lens.reshape(",[57,3283,3284],{"class":280},"-",[57,3286,574],{"class":273},[57,3288,3217],{"class":269},[57,3290,3291],{"class":59,"line":126},[57,3292,3293],{"class":303},"        # 最后一轴上被掩蔽的元素使用一个非常大的负值替换，从而其softmax输出为0\n",[57,3295,3296,3299,3301,3304,3306,3308,3311,3313,3315],{"class":59,"line":132},[57,3297,3298],{"class":269},"        X ",[57,3300,281],{"class":280},[57,3302,3303],{"class":269}," d2l.sequence_mask(X.reshape(",[57,3305,3284],{"class":280},[57,3307,574],{"class":273},[57,3309,3310],{"class":269},", shape[",[57,3312,3284],{"class":280},[57,3314,574],{"class":273},[57,3316,3317],{"class":269},"]), valid_lens,\n",[57,3319,3320,3323,3325,3328],{"class":59,"line":138},[57,3321,3322],{"class":3208},"                              value",[57,3324,3212],{"class":280},[57,3326,3327],{"class":273},"1e6",[57,3329,3217],{"class":269},[57,3331,3332,3334,3337,3339,3341,3343],{"class":59,"line":144},[57,3333,3202],{"class":280},[57,3335,3336],{"class":269}," nn.functional.softmax(X.reshape(shape), ",[57,3338,3209],{"class":3208},[57,3340,3212],{"class":280},[57,3342,574],{"class":273},[57,3344,3217],{"class":269},[14,3346,3347],{},"下面实现缩放点积注意力。",[47,3349,3351],{"className":256,"code":3350,"language":258,"meta":52,"style":52},"class DotProductAttention(nn.Module):\n    \"\"\"缩放点积注意力\"\"\"\n    def __init__(self, dropout, **kwargs):\n        super(DotProductAttention, self).__init__(**kwargs)\n        self.dropout = nn.Dropout(dropout)\n\n    # queries的形状：(batch_size，查询的个数，d)\n    # keys的形状：(batch_size，“键－值”对的个数，d)\n    # values的形状：(batch_size，“键－值”对的个数，值的维度)\n    # valid_lens的形状:(batch_size，)或者(batch_size，查询的个数)\n    def forward(self, queries, keys, values, valid_lens=None):\n        d = queries.shape[-1]\n        # 设置transpose_b=True为了交换keys的最后两个维度\n        scores = torch.bmm(queries, keys.transpose(1,2)) / math.sqrt(d)\n        self.attention_weights = masked_softmax(scores, valid_lens)\n        return torch.bmm(self.dropout(self.attention_weights), values)\n",[54,3352,3353,3375,3380,3408,3431,3444,3449,3454,3459,3464,3469,3506,3523,3528,3553,3565],{"__ignoreMap":52},[57,3354,3355,3358,3362,3364,3368,3370,3373],{"class":59,"line":60},[57,3356,3357],{"class":280},"class",[57,3359,3361],{"class":3360},"skCyd"," DotProductAttention",[57,3363,1754],{"class":269},[57,3365,3367],{"class":3366},"sDP9b","nn",[57,3369,1818],{"class":269},[57,3371,3372],{"class":3366},"Module",[57,3374,3169],{"class":269},[57,3376,3377],{"class":59,"line":66},[57,3378,3379],{"class":3174},"    \"\"\"缩放点积注意力\"\"\"\n",[57,3381,3382,3385,3388,3390,3393,3395,3398,3400,3403,3406],{"class":59,"line":72},[57,3383,3384],{"class":280},"    def",[57,3386,3387],{"class":273}," __init__",[57,3389,1754],{"class":269},[57,3391,266],{"class":3392},"sD3jg",[57,3394,3163],{"class":269},[57,3396,3397],{"class":3159},"dropout",[57,3399,3163],{"class":269},[57,3401,3402],{"class":280},"**",[57,3404,3405],{"class":3159},"kwargs",[57,3407,3169],{"class":269},[57,3409,3410,3414,3417,3419,3422,3424,3426,3428],{"class":59,"line":78},[57,3411,3413],{"class":3412},"sPGBF","        super",[57,3415,3416],{"class":269},"(DotProductAttention, ",[57,3418,266],{"class":265},[57,3420,3421],{"class":269},").",[57,3423,869],{"class":273},[57,3425,1754],{"class":269},[57,3427,3402],{"class":280},[57,3429,3430],{"class":269},"kwargs)\n",[57,3432,3433,3436,3439,3441],{"class":59,"line":84},[57,3434,3435],{"class":265},"        self",[57,3437,3438],{"class":269},".dropout ",[57,3440,281],{"class":280},[57,3442,3443],{"class":269}," nn.Dropout(dropout)\n",[57,3445,3446],{"class":59,"line":90},[57,3447,3448],{"emptyLinePlaceholder":963},"\n",[57,3450,3451],{"class":59,"line":96},[57,3452,3453],{"class":303},"    # queries的形状：(batch_size，查询的个数，d)\n",[57,3455,3456],{"class":59,"line":102},[57,3457,3458],{"class":303},"    # keys的形状：(batch_size，“键－值”对的个数，d)\n",[57,3460,3461],{"class":59,"line":108},[57,3462,3463],{"class":303},"    # values的形状：(batch_size，“键－值”对的个数，值的维度)\n",[57,3465,3466],{"class":59,"line":114},[57,3467,3468],{"class":303},"    # valid_lens的形状:(batch_size，)或者(batch_size，查询的个数)\n",[57,3470,3471,3473,3476,3478,3480,3482,3485,3487,3490,3492,3495,3497,3499,3501,3504],{"class":59,"line":120},[57,3472,3384],{"class":280},[57,3474,3475],{"class":3153}," forward",[57,3477,1754],{"class":269},[57,3479,266],{"class":3392},[57,3481,3163],{"class":269},[57,3483,3484],{"class":3159},"queries",[57,3486,3163],{"class":269},[57,3488,3489],{"class":3159},"keys",[57,3491,3163],{"class":269},[57,3493,3494],{"class":3159},"values",[57,3496,3163],{"class":269},[57,3498,3166],{"class":3159},[57,3500,281],{"class":280},[57,3502,3503],{"class":273},"None",[57,3505,3169],{"class":269},[57,3507,3508,3511,3513,3516,3518,3520],{"class":59,"line":126},[57,3509,3510],{"class":269},"        d ",[57,3512,281],{"class":280},[57,3514,3515],{"class":269}," queries.shape[",[57,3517,3284],{"class":280},[57,3519,574],{"class":273},[57,3521,3522],{"class":269},"]\n",[57,3524,3525],{"class":59,"line":132},[57,3526,3527],{"class":303},"        # 设置transpose_b=True为了交换keys的最后两个维度\n",[57,3529,3530,3533,3535,3538,3540,3542,3544,3547,3550],{"class":59,"line":138},[57,3531,3532],{"class":269},"        scores ",[57,3534,281],{"class":280},[57,3536,3537],{"class":269}," torch.bmm(queries, keys.transpose(",[57,3539,574],{"class":273},[57,3541,1759],{"class":269},[57,3543,590],{"class":273},[57,3545,3546],{"class":269},")) ",[57,3548,3549],{"class":280},"/",[57,3551,3552],{"class":269}," math.sqrt(d)\n",[57,3554,3555,3557,3560,3562],{"class":59,"line":144},[57,3556,3435],{"class":265},[57,3558,3559],{"class":269},".attention_weights ",[57,3561,281],{"class":280},[57,3563,3564],{"class":269}," masked_softmax(scores, valid_lens)\n",[57,3566,3567,3569,3572,3574,3577,3579],{"class":59,"line":150},[57,3568,3202],{"class":280},[57,3570,3571],{"class":269}," torch.bmm(",[57,3573,266],{"class":265},[57,3575,3576],{"class":269},".dropout(",[57,3578,266],{"class":265},[57,3580,3581],{"class":269},".attention_weights), values)\n",[14,3583,3584,3585,3588,3589,3592],{},"代码中的 ",[54,3586,3587],{},"**kwargs"," 表示接受任意关键字参数，是为了符合 PyTorch ",[54,3590,3591],{},"nn.Module"," 的继承约定。",[248,3594,3595],{"id":3595},"多头注意力",[14,3597,3598],{},"多头注意力（Multi-Head Attention, MHA）允许模型在不同位置上同时关注来自不同表示子空间的信息。如果只用单一注意力头，加权平均的过程会把这些不同的关注模式平均掉，模型的表达力也会随之下降。",[14,3600,3601],{},[1000,3602],{"alt":52,"src":3603},"transformer/MHA.png",[14,3605,3606],{},[57,3607,3609,3722],{"className":3608},[1019],[57,3610,3612],{"className":3611},[1023],[1025,3613,3614],{"xmlns":1027},[1029,3615,3616,3719],{},[1032,3617,3618,3641,3643,3645,3647,3649,3651,3653,3655,3657,3672,3674,3689,3691,3694,3696,3710,3712],{},[1032,3619,3620,3623,3626,3629,3631,3633,3635,3637,3639],{},[1047,3621,3622],{"mathvariant":1729},"M",[1047,3624,3625],{"mathvariant":1729},"u",[1047,3627,3628],{"mathvariant":1729},"l",[1047,3630,1733],{"mathvariant":1729},[1047,3632,1745],{"mathvariant":1729},[1047,3634,742],{"mathvariant":1729},[1047,3636,1738],{"mathvariant":1729},[1047,3638,18],{"mathvariant":1729},[1047,3640,1049],{"mathvariant":1729},[1414,3642,1754],{"stretchy":1753},[1047,3644,1412],{},[1414,3646,1759],{"separator":1062},[1047,3648,1525],{},[1414,3650,1759],{"separator":1062},[1047,3652,1625],{},[1414,3654,1768],{"stretchy":1753},[1414,3656,281],{},[1032,3658,3659,3661,3663,3665,3668,3670],{},[1047,3660,734],{"mathvariant":1729},[1047,3662,1748],{"mathvariant":1729},[1047,3664,1288],{"mathvariant":1729},[1047,3666,3667],{"mathvariant":1729},"c",[1047,3669,18],{"mathvariant":1729},[1047,3671,1733],{"mathvariant":1729},[1414,3673,1754],{"stretchy":1753},[1044,3675,3676,3687],{},[1032,3677,3678,3681,3683,3685],{},[1047,3679,3680],{"mathvariant":1729},"h",[1047,3682,1738],{"mathvariant":1729},[1047,3684,18],{"mathvariant":1729},[1047,3686,1049],{"mathvariant":1729},[1038,3688,574],{},[1414,3690,1759],{"separator":1062},[1414,3692,3693],{},"…",[1414,3695,1759],{"separator":1062},[1044,3697,3698,3708],{},[1032,3699,3700,3702,3704,3706],{},[1047,3701,3680],{"mathvariant":1729},[1047,3703,1738],{"mathvariant":1729},[1047,3705,18],{"mathvariant":1729},[1047,3707,1049],{"mathvariant":1729},[1047,3709,3680],{},[1414,3711,1768],{"stretchy":1753},[1418,3713,3714,3716],{},[1047,3715,746],{},[1047,3717,3718],{},"O",[1054,3720,3721],{"encoding":1056},"\\mathrm{MultiHead}(Q, K, V) = \\mathrm{Concat}(\\mathrm{head}_1, \\ldots, \\mathrm{head}_h) W^O",[57,3723,3725,3774],{"className":3724,"ariaHidden":1062},[1061],[57,3726,3728,3731,3738,3741,3744,3747,3750,3753,3756,3759,3762,3765,3768,3771],{"className":3727},[1066],[57,3729],{"className":3730,"style":1831},[1070],[57,3732,3734],{"className":3733},[1075],[57,3735,3737],{"className":3736},[1075,1838],"MultiHead",[57,3739,1754],{"className":3740},[1079],[57,3742,1412],{"className":3743},[1075,1148],[57,3745,1759],{"className":3746},[1849],[57,3748],{"className":3749,"style":1853},[1452],[57,3751,1525],{"className":3752,"style":1556},[1075,1148],[57,3754,1759],{"className":3755},[1849],[57,3757],{"className":3758,"style":1853},[1452],[57,3760,1625],{"className":3761,"style":1655},[1075,1148],[57,3763,1768],{"className":3764},[1269],[57,3766],{"className":3767,"style":1453},[1452],[57,3769,281],{"className":3770},[1457],[57,3772],{"className":3773,"style":1453},[1452],[57,3775,3777,3781,3788,3791,3837,3840,3843,3846,3849,3852,3855,3898,3901],{"className":3776},[1066],[57,3778],{"className":3779,"style":3780},[1070],"height:1.0913em;vertical-align:-0.25em;",[57,3782,3784],{"className":3783},[1075],[57,3785,3787],{"className":3786},[1075,1838],"Concat",[57,3789,1754],{"className":3790},[1079],[57,3792,3794,3801],{"className":3793},[1075],[57,3795,3797],{"className":3796},[1075],[57,3798,3800],{"className":3799},[1075,1838],"head",[57,3802,3804],{"className":3803},[1152],[57,3805,3807,3829],{"className":3806},[1087,1088],[57,3808,3810,3826],{"className":3809},[1092],[57,3811,3814],{"className":3812,"style":3813},[1096],"height:0.3011em;",[57,3815,3817,3820],{"style":3816},"top:-2.55em;margin-right:0.05em;",[57,3818],{"className":3819,"style":1493},[1104],[57,3821,3823],{"className":3822},[1109,1110,1111,1112],[57,3824,574],{"className":3825},[1075,1112],[57,3827,1183],{"className":3828},[1182],[57,3830,3832],{"className":3831},[1092],[57,3833,3835],{"className":3834,"style":2230},[1096],[57,3836],{},[57,3838,1759],{"className":3839},[1849],[57,3841],{"className":3842,"style":1853},[1452],[57,3844,3693],{"className":3845},[1898],[57,3847],{"className":3848,"style":1853},[1452],[57,3850,1759],{"className":3851},[1849],[57,3853],{"className":3854,"style":1853},[1452],[57,3856,3858,3864],{"className":3857},[1075],[57,3859,3861],{"className":3860},[1075],[57,3862,3800],{"className":3863},[1075,1838],[57,3865,3867],{"className":3866},[1152],[57,3868,3870,3890],{"className":3869},[1087,1088],[57,3871,3873,3887],{"className":3872},[1092],[57,3874,3876],{"className":3875,"style":2208},[1096],[57,3877,3878,3881],{"style":3816},[57,3879],{"className":3880,"style":1493},[1104],[57,3882,3884],{"className":3883},[1109,1110,1111,1112],[57,3885,3680],{"className":3886},[1075,1148,1112],[57,3888,1183],{"className":3889},[1182],[57,3891,3893],{"className":3892},[1092],[57,3894,3896],{"className":3895,"style":2230},[1096],[57,3897],{},[57,3899,1768],{"className":3900},[1269],[57,3902,3904,3908],{"className":3903},[1075],[57,3905,746],{"className":3906,"style":3907},[1075,1148],"margin-right:0.13889em;",[57,3909,3911],{"className":3910},[1152],[57,3912,3914],{"className":3913},[1087],[57,3915,3917],{"className":3916},[1092],[57,3918,3921],{"className":3919,"style":3920},[1096],"height:0.8413em;",[57,3922,3923,3926],{"style":1489},[57,3924],{"className":3925,"style":1493},[1104],[57,3927,3929],{"className":3928},[1109,1110,1111,1112],[57,3930,3718],{"className":3931,"style":3932},[1075,1148,1112],"margin-right:0.02778em;",[14,3934,3935,3936,4297,4298,369,4547,369,4796,5047,5048,1818],{},"其中 ",[57,3937,3939,4031],{"className":3938},[1019],[57,3940,3942],{"className":3941},[1023],[1025,3943,3944],{"xmlns":1027},[1029,3945,3946,4028],{},[1032,3947,3948,3962,3964,3984,3986,3988,3996,3998,4002,4004,4012,4014,4016,4018,4026],{},[1044,3949,3950,3960],{},[1032,3951,3952,3954,3956,3958],{},[1047,3953,3680],{"mathvariant":1729},[1047,3955,1738],{"mathvariant":1729},[1047,3957,18],{"mathvariant":1729},[1047,3959,1049],{"mathvariant":1729},[1047,3961,1745],{},[1414,3963,281],{},[1032,3965,3966,3968,3970,3972,3974,3976,3978,3980,3982],{},[1047,3967,1730],{"mathvariant":1729},[1047,3969,1733],{"mathvariant":1729},[1047,3971,1733],{"mathvariant":1729},[1047,3973,1738],{"mathvariant":1729},[1047,3975,1288],{"mathvariant":1729},[1047,3977,1733],{"mathvariant":1729},[1047,3979,1745],{"mathvariant":1729},[1047,3981,1748],{"mathvariant":1729},[1047,3983,1288],{"mathvariant":1729},[1414,3985,1754],{"stretchy":1753},[1047,3987,1412],{},[2545,3989,3990,3992,3994],{},[1047,3991,746],{},[1047,3993,1745],{},[1047,3995,1412],{},[1414,3997,1759],{"separator":1062},[3999,4000,4001],"mtext",{},"  ",[1047,4003,1525],{},[2545,4005,4006,4008,4010],{},[1047,4007,746],{},[1047,4009,1745],{},[1047,4011,1525],{},[1414,4013,1759],{"separator":1062},[3999,4015,4001],{},[1047,4017,1625],{},[2545,4019,4020,4022,4024],{},[1047,4021,746],{},[1047,4023,1745],{},[1047,4025,1625],{},[1414,4027,1768],{"stretchy":1753},[1054,4029,4030],{"encoding":1056},"\\mathrm{head}_i=\\mathrm{Attention}(QW_i^Q,\\; KW_i^K,\\; VW_i^V)",[57,4032,4034,4092],{"className":4033,"ariaHidden":1062},[1061],[57,4035,4037,4040,4083,4086,4089],{"className":4036},[1066],[57,4038],{"className":4039,"style":2292},[1070],[57,4041,4043,4049],{"className":4042},[1075],[57,4044,4046],{"className":4045},[1075],[57,4047,3800],{"className":4048},[1075,1838],[57,4050,4052],{"className":4051},[1152],[57,4053,4055,4075],{"className":4054},[1087,1088],[57,4056,4058,4072],{"className":4057},[1092],[57,4059,4061],{"className":4060,"style":2755},[1096],[57,4062,4063,4066],{"style":3816},[57,4064],{"className":4065,"style":1493},[1104],[57,4067,4069],{"className":4068},[1109,1110,1111,1112],[57,4070,1745],{"className":4071},[1075,1148,1112],[57,4073,1183],{"className":4074},[1182],[57,4076,4078],{"className":4077},[1092],[57,4079,4081],{"className":4080,"style":2230},[1096],[57,4082],{},[57,4084],{"className":4085,"style":1453},[1452],[57,4087,281],{"className":4088},[1457],[57,4090],{"className":4091,"style":1453},[1452],[57,4093,4095,4099,4105,4108,4111,4166,4169,4172,4175,4178,4231,4234,4237,4240,4243,4294],{"className":4094},[1066],[57,4096],{"className":4097,"style":4098},[1070],"height:1.2361em;vertical-align:-0.2769em;",[57,4100,4102],{"className":4101},[1075],[57,4103,1839],{"className":4104},[1075,1838],[57,4106,1754],{"className":4107},[1079],[57,4109,1412],{"className":4110},[1075,1148],[57,4112,4114,4117],{"className":4113},[1075],[57,4115,746],{"className":4116,"style":3907},[1075,1148],[57,4118,4120],{"className":4119},[1152],[57,4121,4123,4157],{"className":4122},[1087,1088],[57,4124,4126,4154],{"className":4125},[1092],[57,4127,4130,4142],{"className":4128,"style":4129},[1096],"height:0.9592em;",[57,4131,4133,4136],{"style":4132},"top:-2.4231em;margin-left:-0.1389em;margin-right:0.05em;",[57,4134],{"className":4135,"style":1493},[1104],[57,4137,4139],{"className":4138},[1109,1110,1111,1112],[57,4140,1745],{"className":4141},[1075,1148,1112],[57,4143,4145,4148],{"style":4144},"top:-3.1809em;margin-right:0.05em;",[57,4146],{"className":4147,"style":1493},[1104],[57,4149,4151],{"className":4150},[1109,1110,1111,1112],[57,4152,1412],{"className":4153},[1075,1148,1112],[57,4155,1183],{"className":4156},[1182],[57,4158,4160],{"className":4159},[1092],[57,4161,4164],{"className":4162,"style":4163},[1096],"height:0.2769em;",[57,4165],{},[57,4167,1759],{"className":4168},[1849],[57,4170],{"className":4171,"style":1453},[1452],[57,4173],{"className":4174,"style":1853},[1452],[57,4176,1525],{"className":4177,"style":1556},[1075,1148],[57,4179,4181,4184],{"className":4180},[1075],[57,4182,746],{"className":4183,"style":3907},[1075,1148],[57,4185,4187],{"className":4186},[1152],[57,4188,4190,4222],{"className":4189},[1087,1088],[57,4191,4193,4219],{"className":4192},[1092],[57,4194,4196,4208],{"className":4195,"style":3920},[1096],[57,4197,4199,4202],{"style":4198},"top:-2.4413em;margin-left:-0.1389em;margin-right:0.05em;",[57,4200],{"className":4201,"style":1493},[1104],[57,4203,4205],{"className":4204},[1109,1110,1111,1112],[57,4206,1745],{"className":4207},[1075,1148,1112],[57,4209,4210,4213],{"style":1489},[57,4211],{"className":4212,"style":1493},[1104],[57,4214,4216],{"className":4215},[1109,1110,1111,1112],[57,4217,1525],{"className":4218,"style":1556},[1075,1148,1112],[57,4220,1183],{"className":4221},[1182],[57,4223,4225],{"className":4224},[1092],[57,4226,4229],{"className":4227,"style":4228},[1096],"height:0.2587em;",[57,4230],{},[57,4232,1759],{"className":4233},[1849],[57,4235],{"className":4236,"style":1453},[1452],[57,4238],{"className":4239,"style":1853},[1452],[57,4241,1625],{"className":4242,"style":1655},[1075,1148],[57,4244,4246,4249],{"className":4245},[1075],[57,4247,746],{"className":4248,"style":3907},[1075,1148],[57,4250,4252],{"className":4251},[1152],[57,4253,4255,4286],{"className":4254},[1087,1088],[57,4256,4258,4283],{"className":4257},[1092],[57,4259,4261,4272],{"className":4260,"style":3920},[1096],[57,4262,4263,4266],{"style":4198},[57,4264],{"className":4265,"style":1493},[1104],[57,4267,4269],{"className":4268},[1109,1110,1111,1112],[57,4270,1745],{"className":4271},[1075,1148,1112],[57,4273,4274,4277],{"style":1489},[57,4275],{"className":4276,"style":1493},[1104],[57,4278,4280],{"className":4279},[1109,1110,1111,1112],[57,4281,1625],{"className":4282,"style":1655},[1075,1148,1112],[57,4284,1183],{"className":4285},[1182],[57,4287,4289],{"className":4288},[1092],[57,4290,4292],{"className":4291,"style":4228},[1096],[57,4293],{},[57,4295,1768],{"className":4296},[1269],"，投影参数是矩阵\n",[57,4299,4301,4353],{"className":4300},[1019],[57,4302,4304],{"className":4303},[1023],[1025,4305,4306],{"xmlns":1027},[1029,4307,4308,4350],{},[1032,4309,4310,4318,4320],{},[2545,4311,4312,4314,4316],{},[1047,4313,746],{},[1047,4315,1745],{},[1047,4317,1412],{},[1414,4319,1416],{},[1418,4321,4322,4324],{},[1047,4323,1423],{"mathvariant":1422},[1032,4325,4326,4342,4344],{},[1044,4327,4328,4330],{},[1047,4329,1049],{},[1032,4331,4332,4334,4336,4338,4340],{},[1047,4333,1319],{"mathvariant":1729},[1047,4335,1748],{"mathvariant":1729},[1047,4337,1049],{"mathvariant":1729},[1047,4339,1738],{"mathvariant":1729},[1047,4341,3628],{"mathvariant":1729},[1414,4343,1430],{},[1044,4345,4346,4348],{},[1047,4347,1049],{},[1047,4349,1052],{},[1054,4351,4352],{"encoding":1056},"W_i^Q \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}",[57,4354,4356,4422],{"className":4355,"ariaHidden":1062},[1061],[57,4357,4359,4362,4413,4416,4419],{"className":4358},[1066],[57,4360],{"className":4361,"style":4098},[1070],[57,4363,4365,4368],{"className":4364},[1075],[57,4366,746],{"className":4367,"style":3907},[1075,1148],[57,4369,4371],{"className":4370},[1152],[57,4372,4374,4405],{"className":4373},[1087,1088],[57,4375,4377,4402],{"className":4376},[1092],[57,4378,4380,4391],{"className":4379,"style":4129},[1096],[57,4381,4382,4385],{"style":4132},[57,4383],{"className":4384,"style":1493},[1104],[57,4386,4388],{"className":4387},[1109,1110,1111,1112],[57,4389,1745],{"className":4390},[1075,1148,1112],[57,4392,4393,4396],{"style":4144},[57,4394],{"className":4395,"style":1493},[1104],[57,4397,4399],{"className":4398},[1109,1110,1111,1112],[57,4400,1412],{"className":4401},[1075,1148,1112],[57,4403,1183],{"className":4404},[1182],[57,4406,4408],{"className":4407},[1092],[57,4409,4411],{"className":4410,"style":4163},[1096],[57,4412],{},[57,4414],{"className":4415,"style":1453},[1452],[57,4417,1416],{"className":4418},[1457],[57,4420],{"className":4421,"style":1453},[1452],[57,4423,4425,4428],{"className":4424},[1066],[57,4426],{"className":4427,"style":1467},[1070],[57,4429,4431,4434],{"className":4430},[1075],[57,4432,1423],{"className":4433},[1075,1474],[57,4435,4437],{"className":4436},[1152],[57,4438,4440],{"className":4439},[1087],[57,4441,4443],{"className":4442},[1092],[57,4444,4446],{"className":4445,"style":1467},[1096],[57,4447,4448,4451],{"style":1489},[57,4449],{"className":4450,"style":1493},[1104],[57,4452,4454],{"className":4453},[1109,1110,1111,1112],[57,4455,4457,4504,4507],{"className":4456},[1075,1112],[57,4458,4460,4463],{"className":4459},[1075,1112],[57,4461,1049],{"className":4462},[1075,1148,1112],[57,4464,4466],{"className":4465},[1152],[57,4467,4469,4496],{"className":4468},[1087,1088],[57,4470,4472,4493],{"className":4471},[1092],[57,4473,4475],{"className":4474,"style":1162},[1096],[57,4476,4477,4480],{"style":1165},[57,4478],{"className":4479,"style":1169},[1104],[57,4481,4483],{"className":4482},[1109,1173,1174,1112],[57,4484,4486],{"className":4485},[1075,1112],[57,4487,4489],{"className":4488},[1075,1112],[57,4490,4492],{"className":4491},[1075,1838,1112],"model",[57,4494,1183],{"className":4495},[1182],[57,4497,4499],{"className":4498},[1092],[57,4500,4502],{"className":4501,"style":1190},[1096],[57,4503],{},[57,4505,1430],{"className":4506},[1506,1112],[57,4508,4510,4513],{"className":4509},[1075,1112],[57,4511,1049],{"className":4512},[1075,1148,1112],[57,4514,4516],{"className":4515},[1152],[57,4517,4519,4539],{"className":4518},[1087,1088],[57,4520,4522,4536],{"className":4521},[1092],[57,4523,4525],{"className":4524,"style":1162},[1096],[57,4526,4527,4530],{"style":1165},[57,4528],{"className":4529,"style":1169},[1104],[57,4531,4533],{"className":4532},[1109,1173,1174,1112],[57,4534,1052],{"className":4535,"style":1178},[1075,1148,1112],[57,4537,1183],{"className":4538},[1182],[57,4540,4542],{"className":4541},[1092],[57,4543,4545],{"className":4544,"style":1190},[1096],[57,4546],{},[57,4548,4550,4602],{"className":4549},[1019],[57,4551,4553],{"className":4552},[1023],[1025,4554,4555],{"xmlns":1027},[1029,4556,4557,4599],{},[1032,4558,4559,4567,4569],{},[2545,4560,4561,4563,4565],{},[1047,4562,746],{},[1047,4564,1745],{},[1047,4566,1525],{},[1414,4568,1416],{},[1418,4570,4571,4573],{},[1047,4572,1423],{"mathvariant":1422},[1032,4574,4575,4591,4593],{},[1044,4576,4577,4579],{},[1047,4578,1049],{},[1032,4580,4581,4583,4585,4587,4589],{},[1047,4582,1319],{"mathvariant":1729},[1047,4584,1748],{"mathvariant":1729},[1047,4586,1049],{"mathvariant":1729},[1047,4588,1738],{"mathvariant":1729},[1047,4590,3628],{"mathvariant":1729},[1414,4592,1430],{},[1044,4594,4595,4597],{},[1047,4596,1049],{},[1047,4598,1052],{},[1054,4600,4601],{"encoding":1056},"W_i^K \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_k}",[57,4603,4605,4672],{"className":4604,"ariaHidden":1062},[1061],[57,4606,4608,4612,4663,4666,4669],{"className":4607},[1066],[57,4609],{"className":4610,"style":4611},[1070],"height:1.1em;vertical-align:-0.2587em;",[57,4613,4615,4618],{"className":4614},[1075],[57,4616,746],{"className":4617,"style":3907},[1075,1148],[57,4619,4621],{"className":4620},[1152],[57,4622,4624,4655],{"className":4623},[1087,1088],[57,4625,4627,4652],{"className":4626},[1092],[57,4628,4630,4641],{"className":4629,"style":3920},[1096],[57,4631,4632,4635],{"style":4198},[57,4633],{"className":4634,"style":1493},[1104],[57,4636,4638],{"className":4637},[1109,1110,1111,1112],[57,4639,1745],{"className":4640},[1075,1148,1112],[57,4642,4643,4646],{"style":1489},[57,4644],{"className":4645,"style":1493},[1104],[57,4647,4649],{"className":4648},[1109,1110,1111,1112],[57,4650,1525],{"className":4651,"style":1556},[1075,1148,1112],[57,4653,1183],{"className":4654},[1182],[57,4656,4658],{"className":4657},[1092],[57,4659,4661],{"className":4660,"style":4228},[1096],[57,4662],{},[57,4664],{"className":4665,"style":1453},[1452],[57,4667,1416],{"className":4668},[1457],[57,4670],{"className":4671,"style":1453},[1452],[57,4673,4675,4678],{"className":4674},[1066],[57,4676],{"className":4677,"style":1467},[1070],[57,4679,4681,4684],{"className":4680},[1075],[57,4682,1423],{"className":4683},[1075,1474],[57,4685,4687],{"className":4686},[1152],[57,4688,4690],{"className":4689},[1087],[57,4691,4693],{"className":4692},[1092],[57,4694,4696],{"className":4695,"style":1467},[1096],[57,4697,4698,4701],{"style":1489},[57,4699],{"className":4700,"style":1493},[1104],[57,4702,4704],{"className":4703},[1109,1110,1111,1112],[57,4705,4707,4753,4756],{"className":4706},[1075,1112],[57,4708,4710,4713],{"className":4709},[1075,1112],[57,4711,1049],{"className":4712},[1075,1148,1112],[57,4714,4716],{"className":4715},[1152],[57,4717,4719,4745],{"className":4718},[1087,1088],[57,4720,4722,4742],{"className":4721},[1092],[57,4723,4725],{"className":4724,"style":1162},[1096],[57,4726,4727,4730],{"style":1165},[57,4728],{"className":4729,"style":1169},[1104],[57,4731,4733],{"className":4732},[1109,1173,1174,1112],[57,4734,4736],{"className":4735},[1075,1112],[57,4737,4739],{"className":4738},[1075,1112],[57,4740,4492],{"className":4741},[1075,1838,1112],[57,4743,1183],{"className":4744},[1182],[57,4746,4748],{"className":4747},[1092],[57,4749,4751],{"className":4750,"style":1190},[1096],[57,4752],{},[57,4754,1430],{"className":4755},[1506,1112],[57,4757,4759,4762],{"className":4758},[1075,1112],[57,4760,1049],{"className":4761},[1075,1148,1112],[57,4763,4765],{"className":4764},[1152],[57,4766,4768,4788],{"className":4767},[1087,1088],[57,4769,4771,4785],{"className":4770},[1092],[57,4772,4774],{"className":4773,"style":1162},[1096],[57,4775,4776,4779],{"style":1165},[57,4777],{"className":4778,"style":1169},[1104],[57,4780,4782],{"className":4781},[1109,1173,1174,1112],[57,4783,1052],{"className":4784,"style":1178},[1075,1148,1112],[57,4786,1183],{"className":4787},[1182],[57,4789,4791],{"className":4790},[1092],[57,4792,4794],{"className":4793,"style":1190},[1096],[57,4795],{},[57,4797,4799,4851],{"className":4798},[1019],[57,4800,4802],{"className":4801},[1023],[1025,4803,4804],{"xmlns":1027},[1029,4805,4806,4848],{},[1032,4807,4808,4816,4818],{},[2545,4809,4810,4812,4814],{},[1047,4811,746],{},[1047,4813,1745],{},[1047,4815,1625],{},[1414,4817,1416],{},[1418,4819,4820,4822],{},[1047,4821,1423],{"mathvariant":1422},[1032,4823,4824,4840,4842],{},[1044,4825,4826,4828],{},[1047,4827,1049],{},[1032,4829,4830,4832,4834,4836,4838],{},[1047,4831,1319],{"mathvariant":1729},[1047,4833,1748],{"mathvariant":1729},[1047,4835,1049],{"mathvariant":1729},[1047,4837,1738],{"mathvariant":1729},[1047,4839,3628],{"mathvariant":1729},[1414,4841,1430],{},[1044,4843,4844,4846],{},[1047,4845,1049],{},[1047,4847,1379],{},[1054,4849,4850],{"encoding":1056},"W_i^V \\in \\mathbb{R}^{d_{\\mathrm{model}} \\times d_v}",[57,4852,4854,4920],{"className":4853,"ariaHidden":1062},[1061],[57,4855,4857,4860,4911,4914,4917],{"className":4856},[1066],[57,4858],{"className":4859,"style":4611},[1070],[57,4861,4863,4866],{"className":4862},[1075],[57,4864,746],{"className":4865,"style":3907},[1075,1148],[57,4867,4869],{"className":4868},[1152],[57,4870,4872,4903],{"className":4871},[1087,1088],[57,4873,4875,4900],{"className":4874},[1092],[57,4876,4878,4889],{"className":4877,"style":3920},[1096],[57,4879,4880,4883],{"style":4198},[57,4881],{"className":4882,"style":1493},[1104],[57,4884,4886],{"className":4885},[1109,1110,1111,1112],[57,4887,1745],{"className":4888},[1075,1148,1112],[57,4890,4891,4894],{"style":1489},[57,4892],{"className":4893,"style":1493},[1104],[57,4895,4897],{"className":4896},[1109,1110,1111,1112],[57,4898,1625],{"className":4899,"style":1655},[1075,1148,1112],[57,4901,1183],{"className":4902},[1182],[57,4904,4906],{"className":4905},[1092],[57,4907,4909],{"className":4908,"style":4228},[1096],[57,4910],{},[57,4912],{"className":4913,"style":1453},[1452],[57,4915,1416],{"className":4916},[1457],[57,4918],{"className":4919,"style":1453},[1452],[57,4921,4923,4926],{"className":4922},[1066],[57,4924],{"className":4925,"style":1467},[1070],[57,4927,4929,4932],{"className":4928},[1075],[57,4930,1423],{"className":4931},[1075,1474],[57,4933,4935],{"className":4934},[1152],[57,4936,4938],{"className":4937},[1087],[57,4939,4941],{"className":4940},[1092],[57,4942,4944],{"className":4943,"style":1467},[1096],[57,4945,4946,4949],{"style":1489},[57,4947],{"className":4948,"style":1493},[1104],[57,4950,4952],{"className":4951},[1109,1110,1111,1112],[57,4953,4955,5001,5004],{"className":4954},[1075,1112],[57,4956,4958,4961],{"className":4957},[1075,1112],[57,4959,1049],{"className":4960},[1075,1148,1112],[57,4962,4964],{"className":4963},[1152],[57,4965,4967,4993],{"className":4966},[1087,1088],[57,4968,4970,4990],{"className":4969},[1092],[57,4971,4973],{"className":4972,"style":1162},[1096],[57,4974,4975,4978],{"style":1165},[57,4976],{"className":4977,"style":1169},[1104],[57,4979,4981],{"className":4980},[1109,1173,1174,1112],[57,4982,4984],{"className":4983},[1075,1112],[57,4985,4987],{"className":4986},[1075,1112],[57,4988,4492],{"className":4989},[1075,1838,1112],[57,4991,1183],{"className":4992},[1182],[57,4994,4996],{"className":4995},[1092],[57,4997,4999],{"className":4998,"style":1190},[1096],[57,5000],{},[57,5002,1430],{"className":5003},[1506,1112],[57,5005,5007,5010],{"className":5006},[1075,1112],[57,5008,1049],{"className":5009},[1075,1148,1112],[57,5011,5013],{"className":5012},[1152],[57,5014,5016,5038],{"className":5015},[1087,1088],[57,5017,5019,5035],{"className":5018},[1092],[57,5020,5023],{"className":5021,"style":5022},[1096],"height:0.1645em;",[57,5024,5026,5029],{"style":5025},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[57,5027],{"className":5028,"style":1169},[1104],[57,5030,5032],{"className":5031},[1109,1173,1174,1112],[57,5033,1379],{"className":5034,"style":1394},[1075,1148,1112],[57,5036,1183],{"className":5037},[1182],[57,5039,5041],{"className":5040},[1092],[57,5042,5045],{"className":5043,"style":5044},[1096],"height:0.143em;",[57,5046],{},"\n和\n",[57,5049,5051,5103],{"className":5050},[1019],[57,5052,5054],{"className":5053},[1023],[1025,5055,5056],{"xmlns":1027},[1029,5057,5058,5100],{},[1032,5059,5060,5066,5068],{},[1418,5061,5062,5064],{},[1047,5063,746],{},[1047,5065,3718],{},[1414,5067,1416],{},[1418,5069,5070,5072],{},[1047,5071,1423],{"mathvariant":1422},[1032,5073,5074,5076,5082,5084],{},[1047,5075,3680],{},[1044,5077,5078,5080],{},[1047,5079,1049],{},[1047,5081,1379],{},[1414,5083,1430],{},[1044,5085,5086,5088],{},[1047,5087,1049],{},[1032,5089,5090,5092,5094,5096,5098],{},[1047,5091,1319],{"mathvariant":1729},[1047,5093,1748],{"mathvariant":1729},[1047,5095,1049],{"mathvariant":1729},[1047,5097,1738],{"mathvariant":1729},[1047,5099,3628],{"mathvariant":1729},[1054,5101,5102],{"encoding":1056},"W^O \\in \\mathbb{R}^{h d_v \\times d_{\\mathrm{model}}}",[57,5104,5106,5151],{"className":5105,"ariaHidden":1062},[1061],[57,5107,5109,5113,5142,5145,5148],{"className":5108},[1066],[57,5110],{"className":5111,"style":5112},[1070],"height:0.8804em;vertical-align:-0.0391em;",[57,5114,5116,5119],{"className":5115},[1075],[57,5117,746],{"className":5118,"style":3907},[1075,1148],[57,5120,5122],{"className":5121},[1152],[57,5123,5125],{"className":5124},[1087],[57,5126,5128],{"className":5127},[1092],[57,5129,5131],{"className":5130,"style":3920},[1096],[57,5132,5133,5136],{"style":1489},[57,5134],{"className":5135,"style":1493},[1104],[57,5137,5139],{"className":5138},[1109,1110,1111,1112],[57,5140,3718],{"className":5141,"style":3932},[1075,1148,1112],[57,5143],{"className":5144,"style":1453},[1452],[57,5146,1416],{"className":5147},[1457],[57,5149],{"className":5150,"style":1453},[1452],[57,5152,5154,5157],{"className":5153},[1066],[57,5155],{"className":5156,"style":1467},[1070],[57,5158,5160,5163],{"className":5159},[1075],[57,5161,1423],{"className":5162},[1075,1474],[57,5164,5166],{"className":5165},[1152],[57,5167,5169],{"className":5168},[1087],[57,5170,5172],{"className":5171},[1092],[57,5173,5175],{"className":5174,"style":1467},[1096],[57,5176,5177,5180],{"style":1489},[57,5178],{"className":5179,"style":1493},[1104],[57,5181,5183],{"className":5182},[1109,1110,1111,1112],[57,5184,5186,5189,5229,5232],{"className":5185},[1075,1112],[57,5187,3680],{"className":5188},[1075,1148,1112],[57,5190,5192,5195],{"className":5191},[1075,1112],[57,5193,1049],{"className":5194},[1075,1148,1112],[57,5196,5198],{"className":5197},[1152],[57,5199,5201,5221],{"className":5200},[1087,1088],[57,5202,5204,5218],{"className":5203},[1092],[57,5205,5207],{"className":5206,"style":5022},[1096],[57,5208,5209,5212],{"style":5025},[57,5210],{"className":5211,"style":1169},[1104],[57,5213,5215],{"className":5214},[1109,1173,1174,1112],[57,5216,1379],{"className":5217,"style":1394},[1075,1148,1112],[57,5219,1183],{"className":5220},[1182],[57,5222,5224],{"className":5223},[1092],[57,5225,5227],{"className":5226,"style":5044},[1096],[57,5228],{},[57,5230,1430],{"className":5231},[1506,1112],[57,5233,5235,5238],{"className":5234},[1075,1112],[57,5236,1049],{"className":5237},[1075,1148,1112],[57,5239,5241],{"className":5240},[1152],[57,5242,5244,5270],{"className":5243},[1087,1088],[57,5245,5247,5267],{"className":5246},[1092],[57,5248,5250],{"className":5249,"style":1162},[1096],[57,5251,5252,5255],{"style":1165},[57,5253],{"className":5254,"style":1169},[1104],[57,5256,5258],{"className":5257},[1109,1173,1174,1112],[57,5259,5261],{"className":5260},[1075,1112],[57,5262,5264],{"className":5263},[1075,1112],[57,5265,4492],{"className":5266},[1075,1838,1112],[57,5268,1183],{"className":5269},[1182],[57,5271,5273],{"className":5272},[1092],[57,5274,5276],{"className":5275,"style":1190},[1096],[57,5277],{},[14,5279,5280,5281,743,5284,5287,5288,5291,5292,5295],{},"为了让多个头能够并行计算，我们定义两个变换函数 ",[54,5282,5283],{},"transpose_qkv",[54,5285,5286],{},"transpose_output","，前者将 ",[54,5289,5290],{},"(batch, seq_len, num_hiddens)"," 重排为 ",[54,5293,5294],{},"(batch * num_heads, seq_len, num_hiddens / num_heads)","，后者负责把结果还原回原始形状。",[47,5297,5299],{"className":256,"code":5298,"language":258,"meta":52,"style":52},"def transpose_qkv(X, num_heads):\n    \"\"\"为了多注意力头的并行计算而变换形状\"\"\"\n    # 输入X的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n    # 输出X的形状:(batch_size，查询或者“键－值”对的个数，num_heads，num_hiddens/num_heads)\n    X = X.reshape(X.shape[0], X.shape[1], num_heads, -1)\n\n    # 输出X的形状:(batch_size，num_heads，查询或者“键－值”对的个数, num_hiddens/num_heads)\n    X = X.permute(0, 2, 1, 3)\n\n    # 最终输出的形状:(batch_size*num_heads,查询或者“键－值”对的个数, num_hiddens/num_heads)\n    return X.reshape(-1, X.shape[2], X.shape[3])\n\ndef transpose_output(X, num_heads):\n    \"\"\"逆转transpose_qkv函数的操作\"\"\"\n    X = X.reshape(-1, num_heads, X.shape[1], X.shape[2])\n    X = X.permute(0, 2, 1, 3)\n    return X.reshape(X.shape[0], X.shape[1], -1)\n",[54,5300,5301,5319,5324,5329,5334,5360,5364,5369,5394,5398,5403,5426,5430,5447,5452,5475,5499],{"__ignoreMap":52},[57,5302,5303,5305,5308,5310,5312,5314,5317],{"class":59,"line":60},[57,5304,3150],{"class":280},[57,5306,5307],{"class":3153}," transpose_qkv",[57,5309,1754],{"class":269},[57,5311,3160],{"class":3159},[57,5313,3163],{"class":269},[57,5315,5316],{"class":3159},"num_heads",[57,5318,3169],{"class":269},[57,5320,5321],{"class":59,"line":66},[57,5322,5323],{"class":3174},"    \"\"\"为了多注意力头的并行计算而变换形状\"\"\"\n",[57,5325,5326],{"class":59,"line":72},[57,5327,5328],{"class":303},"    # 输入X的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n",[57,5330,5331],{"class":59,"line":78},[57,5332,5333],{"class":303},"    # 输出X的形状:(batch_size，查询或者“键－值”对的个数，num_heads，num_hiddens/num_heads)\n",[57,5335,5336,5339,5341,5344,5346,5349,5351,5354,5356,5358],{"class":59,"line":84},[57,5337,5338],{"class":269},"    X ",[57,5340,281],{"class":280},[57,5342,5343],{"class":269}," X.reshape(X.shape[",[57,5345,560],{"class":273},[57,5347,5348],{"class":269},"], X.shape[",[57,5350,574],{"class":273},[57,5352,5353],{"class":269},"], num_heads, ",[57,5355,3284],{"class":280},[57,5357,574],{"class":273},[57,5359,3217],{"class":269},[57,5361,5362],{"class":59,"line":90},[57,5363,3448],{"emptyLinePlaceholder":963},[57,5365,5366],{"class":59,"line":96},[57,5367,5368],{"class":303},"    # 输出X的形状:(batch_size，num_heads，查询或者“键－值”对的个数, num_hiddens/num_heads)\n",[57,5370,5371,5373,5375,5378,5380,5382,5384,5386,5388,5390,5392],{"class":59,"line":102},[57,5372,5338],{"class":269},[57,5374,281],{"class":280},[57,5376,5377],{"class":269}," X.permute(",[57,5379,560],{"class":273},[57,5381,3163],{"class":269},[57,5383,590],{"class":273},[57,5385,3163],{"class":269},[57,5387,574],{"class":273},[57,5389,3163],{"class":269},[57,5391,606],{"class":273},[57,5393,3217],{"class":269},[57,5395,5396],{"class":59,"line":108},[57,5397,3448],{"emptyLinePlaceholder":963},[57,5399,5400],{"class":59,"line":114},[57,5401,5402],{"class":303},"    # 最终输出的形状:(batch_size*num_heads,查询或者“键－值”对的个数, num_hiddens/num_heads)\n",[57,5404,5405,5408,5411,5413,5415,5418,5420,5422,5424],{"class":59,"line":120},[57,5406,5407],{"class":280},"    return",[57,5409,5410],{"class":269}," X.reshape(",[57,5412,3284],{"class":280},[57,5414,574],{"class":273},[57,5416,5417],{"class":269},", X.shape[",[57,5419,590],{"class":273},[57,5421,5348],{"class":269},[57,5423,606],{"class":273},[57,5425,3265],{"class":269},[57,5427,5428],{"class":59,"line":126},[57,5429,3448],{"emptyLinePlaceholder":963},[57,5431,5432,5434,5437,5439,5441,5443,5445],{"class":59,"line":132},[57,5433,3150],{"class":280},[57,5435,5436],{"class":3153}," transpose_output",[57,5438,1754],{"class":269},[57,5440,3160],{"class":3159},[57,5442,3163],{"class":269},[57,5444,5316],{"class":3159},[57,5446,3169],{"class":269},[57,5448,5449],{"class":59,"line":138},[57,5450,5451],{"class":3174},"    \"\"\"逆转transpose_qkv函数的操作\"\"\"\n",[57,5453,5454,5456,5458,5460,5462,5464,5467,5469,5471,5473],{"class":59,"line":144},[57,5455,5338],{"class":269},[57,5457,281],{"class":280},[57,5459,5410],{"class":269},[57,5461,3284],{"class":280},[57,5463,574],{"class":273},[57,5465,5466],{"class":269},", num_heads, X.shape[",[57,5468,574],{"class":273},[57,5470,5348],{"class":269},[57,5472,590],{"class":273},[57,5474,3265],{"class":269},[57,5476,5477,5479,5481,5483,5485,5487,5489,5491,5493,5495,5497],{"class":59,"line":150},[57,5478,5338],{"class":269},[57,5480,281],{"class":280},[57,5482,5377],{"class":269},[57,5484,560],{"class":273},[57,5486,3163],{"class":269},[57,5488,590],{"class":273},[57,5490,3163],{"class":269},[57,5492,574],{"class":273},[57,5494,3163],{"class":269},[57,5496,606],{"class":273},[57,5498,3217],{"class":269},[57,5500,5501,5503,5505,5507,5509,5511,5514,5516,5518],{"class":59,"line":156},[57,5502,5407],{"class":280},[57,5504,5343],{"class":269},[57,5506,560],{"class":273},[57,5508,5348],{"class":269},[57,5510,574],{"class":273},[57,5512,5513],{"class":269},"], ",[57,5515,3284],{"class":280},[57,5517,574],{"class":273},[57,5519,3217],{"class":269},[47,5521,5523],{"className":256,"code":5522,"language":258,"meta":52,"style":52},"#@save\nclass MultiHeadAttention(nn.Module):\n    \"\"\"多头注意力\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 num_heads, dropout, bias=False, **kwargs):\n        super(MultiHeadAttention, self).__init__(**kwargs)\n        self.num_heads = num_heads\n        self.attention = d2l.DotProductAttention(dropout)\n        self.W_q = nn.Linear(query_size, num_hiddens, bias=bias)\n        self.W_k = nn.Linear(key_size, num_hiddens, bias=bias)\n        self.W_v = nn.Linear(value_size, num_hiddens, bias=bias)\n        self.W_o = nn.Linear(num_hiddens, num_hiddens, bias=bias)\n\n    def forward(self, queries, keys, values, valid_lens):\n        # queries，keys，values的形状:\n        # (batch_size，查询或者“键－值”对的个数，num_hiddens)\n        # valid_lens　的形状:\n        # (batch_size，)或(batch_size，查询的个数)\n        # 经过变换后，输出的queries，keys，values 的形状:\n        # (batch_size*num_heads，查询或者“键－值”对的个数，num_hiddens/num_heads)\n        queries = transpose_qkv(self.W_q(queries), self.num_heads)\n        keys = transpose_qkv(self.W_k(keys), self.num_heads)\n        values = transpose_qkv(self.W_v(values), self.num_heads)\n\n        if valid_lens is not None:\n            # 在轴0，将第一项（标量或者矢量）复制num_heads次，\n            # 然后如此复制第二项，然后诸如此类。\n            valid_lens = torch.repeat_interleave(\n                valid_lens, repeats=self.num_heads, dim=0)\n\n        # output的形状:(batch_size*num_heads，查询的个数， num_hiddens/num_heads)\n        output = self.attention(queries, keys, values, valid_lens)\n\n        # output_concat的形状:(batch_size，查询的个数，num_hiddens)\n        output_concat = transpose_output(output, self.num_heads)\n        return self.W_o(output_concat)\n",[54,5524,5525,5530,5547,5552,5584,5610,5629,5641,5653,5672,5690,5708,5726,5730,5758,5763,5768,5773,5778,5783,5788,5809,5828,5847,5852,5868,5874,5880,5890,5914,5919,5925,5939,5944,5950,5965],{"__ignoreMap":52},[57,5526,5527],{"class":59,"line":60},[57,5528,5529],{"class":303},"#@save\n",[57,5531,5532,5534,5537,5539,5541,5543,5545],{"class":59,"line":66},[57,5533,3357],{"class":280},[57,5535,5536],{"class":3360}," MultiHeadAttention",[57,5538,1754],{"class":269},[57,5540,3367],{"class":3366},[57,5542,1818],{"class":269},[57,5544,3372],{"class":3366},[57,5546,3169],{"class":269},[57,5548,5549],{"class":59,"line":72},[57,5550,5551],{"class":3174},"    \"\"\"多头注意力\"\"\"\n",[57,5553,5554,5556,5558,5560,5562,5564,5567,5569,5572,5574,5577,5579,5582],{"class":59,"line":78},[57,5555,3384],{"class":280},[57,5557,3387],{"class":273},[57,5559,1754],{"class":269},[57,5561,266],{"class":3392},[57,5563,3163],{"class":269},[57,5565,5566],{"class":3159},"key_size",[57,5568,3163],{"class":269},[57,5570,5571],{"class":3159},"query_size",[57,5573,3163],{"class":269},[57,5575,5576],{"class":3159},"value_size",[57,5578,3163],{"class":269},[57,5580,5581],{"class":3159},"num_hiddens",[57,5583,369],{"class":269},[57,5585,5586,5589,5591,5593,5595,5598,5600,5602,5604,5606,5608],{"class":59,"line":84},[57,5587,5588],{"class":3159},"                 num_heads",[57,5590,3163],{"class":269},[57,5592,3397],{"class":3159},[57,5594,3163],{"class":269},[57,5596,5597],{"class":3159},"bias",[57,5599,281],{"class":280},[57,5601,800],{"class":273},[57,5603,3163],{"class":269},[57,5605,3402],{"class":280},[57,5607,3405],{"class":3159},[57,5609,3169],{"class":269},[57,5611,5612,5614,5617,5619,5621,5623,5625,5627],{"class":59,"line":90},[57,5613,3413],{"class":3412},[57,5615,5616],{"class":269},"(MultiHeadAttention, ",[57,5618,266],{"class":265},[57,5620,3421],{"class":269},[57,5622,869],{"class":273},[57,5624,1754],{"class":269},[57,5626,3402],{"class":280},[57,5628,3430],{"class":269},[57,5630,5631,5633,5636,5638],{"class":59,"line":96},[57,5632,3435],{"class":265},[57,5634,5635],{"class":269},".num_heads ",[57,5637,281],{"class":280},[57,5639,5640],{"class":269}," num_heads\n",[57,5642,5643,5645,5648,5650],{"class":59,"line":102},[57,5644,3435],{"class":265},[57,5646,5647],{"class":269},".attention ",[57,5649,281],{"class":280},[57,5651,5652],{"class":269}," d2l.DotProductAttention(dropout)\n",[57,5654,5655,5657,5660,5662,5665,5667,5669],{"class":59,"line":108},[57,5656,3435],{"class":265},[57,5658,5659],{"class":269},".W_q ",[57,5661,281],{"class":280},[57,5663,5664],{"class":269}," nn.Linear(query_size, num_hiddens, ",[57,5666,5597],{"class":3208},[57,5668,281],{"class":280},[57,5670,5671],{"class":269},"bias)\n",[57,5673,5674,5676,5679,5681,5684,5686,5688],{"class":59,"line":114},[57,5675,3435],{"class":265},[57,5677,5678],{"class":269},".W_k ",[57,5680,281],{"class":280},[57,5682,5683],{"class":269}," nn.Linear(key_size, num_hiddens, ",[57,5685,5597],{"class":3208},[57,5687,281],{"class":280},[57,5689,5671],{"class":269},[57,5691,5692,5694,5697,5699,5702,5704,5706],{"class":59,"line":120},[57,5693,3435],{"class":265},[57,5695,5696],{"class":269},".W_v ",[57,5698,281],{"class":280},[57,5700,5701],{"class":269}," nn.Linear(value_size, num_hiddens, ",[57,5703,5597],{"class":3208},[57,5705,281],{"class":280},[57,5707,5671],{"class":269},[57,5709,5710,5712,5715,5717,5720,5722,5724],{"class":59,"line":126},[57,5711,3435],{"class":265},[57,5713,5714],{"class":269},".W_o ",[57,5716,281],{"class":280},[57,5718,5719],{"class":269}," nn.Linear(num_hiddens, num_hiddens, ",[57,5721,5597],{"class":3208},[57,5723,281],{"class":280},[57,5725,5671],{"class":269},[57,5727,5728],{"class":59,"line":132},[57,5729,3448],{"emptyLinePlaceholder":963},[57,5731,5732,5734,5736,5738,5740,5742,5744,5746,5748,5750,5752,5754,5756],{"class":59,"line":138},[57,5733,3384],{"class":280},[57,5735,3475],{"class":3153},[57,5737,1754],{"class":269},[57,5739,266],{"class":3392},[57,5741,3163],{"class":269},[57,5743,3484],{"class":3159},[57,5745,3163],{"class":269},[57,5747,3489],{"class":3159},[57,5749,3163],{"class":269},[57,5751,3494],{"class":3159},[57,5753,3163],{"class":269},[57,5755,3166],{"class":3159},[57,5757,3169],{"class":269},[57,5759,5760],{"class":59,"line":144},[57,5761,5762],{"class":303},"        # queries，keys，values的形状:\n",[57,5764,5765],{"class":59,"line":150},[57,5766,5767],{"class":303},"        # (batch_size，查询或者“键－值”对的个数，num_hiddens)\n",[57,5769,5770],{"class":59,"line":156},[57,5771,5772],{"class":303},"        # valid_lens　的形状:\n",[57,5774,5775],{"class":59,"line":162},[57,5776,5777],{"class":303},"        # (batch_size，)或(batch_size，查询的个数)\n",[57,5779,5780],{"class":59,"line":168},[57,5781,5782],{"class":303},"        # 经过变换后，输出的queries，keys，values 的形状:\n",[57,5784,5785],{"class":59,"line":174},[57,5786,5787],{"class":303},"        # (batch_size*num_heads，查询或者“键－值”对的个数，num_hiddens/num_heads)\n",[57,5789,5791,5794,5796,5799,5801,5804,5806],{"class":59,"line":5790},21,[57,5792,5793],{"class":269},"        queries ",[57,5795,281],{"class":280},[57,5797,5798],{"class":269}," transpose_qkv(",[57,5800,266],{"class":265},[57,5802,5803],{"class":269},".W_q(queries), ",[57,5805,266],{"class":265},[57,5807,5808],{"class":269},".num_heads)\n",[57,5810,5812,5815,5817,5819,5821,5824,5826],{"class":59,"line":5811},22,[57,5813,5814],{"class":269},"        keys ",[57,5816,281],{"class":280},[57,5818,5798],{"class":269},[57,5820,266],{"class":265},[57,5822,5823],{"class":269},".W_k(keys), ",[57,5825,266],{"class":265},[57,5827,5808],{"class":269},[57,5829,5831,5834,5836,5838,5840,5843,5845],{"class":59,"line":5830},23,[57,5832,5833],{"class":269},"        values ",[57,5835,281],{"class":280},[57,5837,5798],{"class":269},[57,5839,266],{"class":265},[57,5841,5842],{"class":269},".W_v(values), ",[57,5844,266],{"class":265},[57,5846,5808],{"class":269},[57,5848,5850],{"class":59,"line":5849},24,[57,5851,3448],{"emptyLinePlaceholder":963},[57,5853,5855,5857,5859,5861,5864,5866],{"class":59,"line":5854},25,[57,5856,3239],{"class":280},[57,5858,3188],{"class":269},[57,5860,3191],{"class":280},[57,5862,5863],{"class":280}," not",[57,5865,3194],{"class":273},[57,5867,3197],{"class":269},[57,5869,5871],{"class":59,"line":5870},26,[57,5872,5873],{"class":303},"            # 在轴0，将第一项（标量或者矢量）复制num_heads次，\n",[57,5875,5877],{"class":59,"line":5876},27,[57,5878,5879],{"class":303},"            # 然后如此复制第二项，然后诸如此类。\n",[57,5881,5883,5885,5887],{"class":59,"line":5882},28,[57,5884,3255],{"class":269},[57,5886,281],{"class":280},[57,5888,5889],{"class":269}," torch.repeat_interleave(\n",[57,5891,5893,5896,5899,5901,5903,5906,5908,5910,5912],{"class":59,"line":5892},29,[57,5894,5895],{"class":269},"                valid_lens, ",[57,5897,5898],{"class":3208},"repeats",[57,5900,281],{"class":280},[57,5902,266],{"class":265},[57,5904,5905],{"class":269},".num_heads, ",[57,5907,3209],{"class":3208},[57,5909,281],{"class":280},[57,5911,560],{"class":273},[57,5913,3217],{"class":269},[57,5915,5917],{"class":59,"line":5916},30,[57,5918,3448],{"emptyLinePlaceholder":963},[57,5920,5922],{"class":59,"line":5921},31,[57,5923,5924],{"class":303},"        # output的形状:(batch_size*num_heads，查询的个数， num_hiddens/num_heads)\n",[57,5926,5928,5931,5933,5936],{"class":59,"line":5927},32,[57,5929,5930],{"class":269},"        output ",[57,5932,281],{"class":280},[57,5934,5935],{"class":265}," self",[57,5937,5938],{"class":269},".attention(queries, keys, values, valid_lens)\n",[57,5940,5942],{"class":59,"line":5941},33,[57,5943,3448],{"emptyLinePlaceholder":963},[57,5945,5947],{"class":59,"line":5946},34,[57,5948,5949],{"class":303},"        # output_concat的形状:(batch_size，查询的个数，num_hiddens)\n",[57,5951,5953,5956,5958,5961,5963],{"class":59,"line":5952},35,[57,5954,5955],{"class":269},"        output_concat ",[57,5957,281],{"class":280},[57,5959,5960],{"class":269}," transpose_output(output, ",[57,5962,266],{"class":265},[57,5964,5808],{"class":269},[57,5966,5968,5970,5972],{"class":59,"line":5967},36,[57,5969,3202],{"class":280},[57,5971,5935],{"class":265},[57,5973,5974],{"class":269},".W_o(output_concat)\n",[10,5976,5977],{"id":5977},"位置编码",[14,5979,5980,5981,6058],{},"RNN 是按时间步逐个处理 token 的，天然带有顺序信息；而注意力机制为了支持并行计算放弃了顺序处理，因而失去了位置信息。为了让模型仍能感知到 token 在序列中的位置，我们需要把\"位置\"显式地注入到输入表示中——这就是位置编码（Positional Encoding）。位置编码与词嵌入的维度相同，均为 ",[57,5982,5984,6002],{"className":5983},[1019],[57,5985,5987],{"className":5986},[1023],[1025,5988,5989],{"xmlns":1027},[1029,5990,5991,5999],{},[1032,5992,5993],{},[1044,5994,5995,5997],{},[1047,5996,1049],{},[3999,5998,4492],{},[1054,6000,6001],{"encoding":1056},"d_{\\text{model}}",[57,6003,6005],{"className":6004,"ariaHidden":1062},[1061],[57,6006,6008,6011],{"className":6007},[1066],[57,6009],{"className":6010,"style":2292},[1070],[57,6012,6014,6017],{"className":6013},[1075],[57,6015,1049],{"className":6016},[1075,1148],[57,6018,6020],{"className":6019},[1152],[57,6021,6023,6050],{"className":6022},[1087,1088],[57,6024,6026,6047],{"className":6025},[1092],[57,6027,6029],{"className":6028,"style":2208},[1096],[57,6030,6031,6034],{"style":2211},[57,6032],{"className":6033,"style":1493},[1104],[57,6035,6037],{"className":6036},[1109,1110,1111,1112],[57,6038,6040],{"className":6039},[1075,1112],[57,6041,6044],{"className":6042},[1075,6043,1112],"text",[57,6045,4492],{"className":6046},[1075,1112],[57,6048,1183],{"className":6049},[1182],[57,6051,6053],{"className":6052},[1092],[57,6054,6056],{"className":6055,"style":2230},[1096],[57,6057],{},"，因此可以直接相加。它既可以通过参数学习得到，也可以用固定的公式直接计算。",[14,6060,6061],{},"这里我们使用基于正弦和余弦函数的固定位置编码。",[14,6063,6064,6065,6211,6212,6240,6241,6316,6317,6470,6471,6532,6533,6562,6563,743,6600,6656],{},"设输入表示 ",[57,6066,6068,6101],{"className":6067},[1019],[57,6069,6071],{"className":6070},[1023],[1025,6072,6073],{"xmlns":1027},[1029,6074,6075,6098],{},[1032,6076,6077,6080,6082],{},[1047,6078,3160],{"mathvariant":6079},"bold",[1414,6081,1416],{},[1418,6083,6084,6086],{},[1047,6085,1423],{"mathvariant":1422},[1032,6087,6088,6090,6092],{},[1047,6089,1288],{},[1414,6091,1430],{},[1044,6093,6094,6096],{},[1047,6095,1049],{},[3999,6097,4492],{},[1054,6099,6100],{"encoding":1056},"\\mathbf{X} \\in \\mathbb{R}^{n \\times d_{\\text{model}}}",[57,6102,6104,6124],{"className":6103,"ariaHidden":1062},[1061],[57,6105,6107,6111,6115,6118,6121],{"className":6106},[1066],[57,6108],{"className":6109,"style":6110},[1070],"height:0.7252em;vertical-align:-0.0391em;",[57,6112,3160],{"className":6113},[1075,6114],"mathbf",[57,6116],{"className":6117,"style":1453},[1452],[57,6119,1416],{"className":6120},[1457],[57,6122],{"className":6123,"style":1453},[1452],[57,6125,6127,6130],{"className":6126},[1066],[57,6128],{"className":6129,"style":1467},[1070],[57,6131,6133,6136],{"className":6132},[1075],[57,6134,1423],{"className":6135},[1075,1474],[57,6137,6139],{"className":6138},[1152],[57,6140,6142],{"className":6141},[1087],[57,6143,6145],{"className":6144},[1092],[57,6146,6148],{"className":6147,"style":1467},[1096],[57,6149,6150,6153],{"style":1489},[57,6151],{"className":6152,"style":1493},[1104],[57,6154,6156],{"className":6155},[1109,1110,1111,1112],[57,6157,6159,6162,6165],{"className":6158},[1075,1112],[57,6160,1288],{"className":6161},[1075,1148,1112],[57,6163,1430],{"className":6164},[1506,1112],[57,6166,6168,6171],{"className":6167},[1075,1112],[57,6169,1049],{"className":6170},[1075,1148,1112],[57,6172,6174],{"className":6173},[1152],[57,6175,6177,6203],{"className":6176},[1087,1088],[57,6178,6180,6200],{"className":6179},[1092],[57,6181,6183],{"className":6182,"style":1162},[1096],[57,6184,6185,6188],{"style":1165},[57,6186],{"className":6187,"style":1169},[1104],[57,6189,6191],{"className":6190},[1109,1173,1174,1112],[57,6192,6194],{"className":6193},[1075,1112],[57,6195,6197],{"className":6196},[1075,6043,1112],[57,6198,4492],{"className":6199},[1075,1112],[57,6201,1183],{"className":6202},[1182],[57,6204,6206],{"className":6205},[1092],[57,6207,6209],{"className":6208,"style":1190},[1096],[57,6210],{}," 包含一段长度为 ",[57,6213,6215,6228],{"className":6214},[1019],[57,6216,6218],{"className":6217},[1023],[1025,6219,6220],{"xmlns":1027},[1029,6221,6222,6226],{},[1032,6223,6224],{},[1047,6225,1288],{},[1054,6227,1288],{"encoding":1056},[57,6229,6231],{"className":6230,"ariaHidden":1062},[1061],[57,6232,6234,6237],{"className":6233},[1066],[57,6235],{"className":6236,"style":1300},[1070],[57,6238,1288],{"className":6239},[1075,1148]," 的序列、每个 token 用 ",[57,6242,6244,6261],{"className":6243},[1019],[57,6245,6247],{"className":6246},[1023],[1025,6248,6249],{"xmlns":1027},[1029,6250,6251,6259],{},[1032,6252,6253],{},[1044,6254,6255,6257],{},[1047,6256,1049],{},[3999,6258,4492],{},[1054,6260,6001],{"encoding":1056},[57,6262,6264],{"className":6263,"ariaHidden":1062},[1061],[57,6265,6267,6270],{"className":6266},[1066],[57,6268],{"className":6269,"style":2292},[1070],[57,6271,6273,6276],{"className":6272},[1075],[57,6274,1049],{"className":6275},[1075,1148],[57,6277,6279],{"className":6278},[1152],[57,6280,6282,6308],{"className":6281},[1087,1088],[57,6283,6285,6305],{"className":6284},[1092],[57,6286,6288],{"className":6287,"style":2208},[1096],[57,6289,6290,6293],{"style":2211},[57,6291],{"className":6292,"style":1493},[1104],[57,6294,6296],{"className":6295},[1109,1110,1111,1112],[57,6297,6299],{"className":6298},[1075,1112],[57,6300,6302],{"className":6301},[1075,6043,1112],[57,6303,4492],{"className":6304},[1075,1112],[57,6306,1183],{"className":6307},[1182],[57,6309,6311],{"className":6310},[1092],[57,6312,6314],{"className":6313,"style":2230},[1096],[57,6315],{}," 维向量表示。位置编码使用相同形状的矩阵 ",[57,6318,6320,6358],{"className":6319},[1019],[57,6321,6323],{"className":6322},[1023],[1025,6324,6325],{"xmlns":1027},[1029,6326,6327,6355],{},[1032,6328,6329,6337,6339],{},[1032,6330,6331,6334],{},[1047,6332,6333],{"mathvariant":6079},"P",[1047,6335,6336],{"mathvariant":6079},"E",[1414,6338,1416],{},[1418,6340,6341,6343],{},[1047,6342,1423],{"mathvariant":1422},[1032,6344,6345,6347,6349],{},[1047,6346,1288],{},[1414,6348,1430],{},[1044,6350,6351,6353],{},[1047,6352,1049],{},[3999,6354,4492],{},[1054,6356,6357],{"encoding":1056},"\\mathbf{PE} \\in \\mathbb{R}^{n \\times d_{\\text{model}}}",[57,6359,6361,6383],{"className":6360,"ariaHidden":1062},[1061],[57,6362,6364,6367,6374,6377,6380],{"className":6363},[1066],[57,6365],{"className":6366,"style":6110},[1070],[57,6368,6370],{"className":6369},[1075],[57,6371,6373],{"className":6372},[1075,6114],"PE",[57,6375],{"className":6376,"style":1453},[1452],[57,6378,1416],{"className":6379},[1457],[57,6381],{"className":6382,"style":1453},[1452],[57,6384,6386,6389],{"className":6385},[1066],[57,6387],{"className":6388,"style":1467},[1070],[57,6390,6392,6395],{"className":6391},[1075],[57,6393,1423],{"className":6394},[1075,1474],[57,6396,6398],{"className":6397},[1152],[57,6399,6401],{"className":6400},[1087],[57,6402,6404],{"className":6403},[1092],[57,6405,6407],{"className":6406,"style":1467},[1096],[57,6408,6409,6412],{"style":1489},[57,6410],{"className":6411,"style":1493},[1104],[57,6413,6415],{"className":6414},[1109,1110,1111,1112],[57,6416,6418,6421,6424],{"className":6417},[1075,1112],[57,6419,1288],{"className":6420},[1075,1148,1112],[57,6422,1430],{"className":6423},[1506,1112],[57,6425,6427,6430],{"className":6426},[1075,1112],[57,6428,1049],{"className":6429},[1075,1148,1112],[57,6431,6433],{"className":6432},[1152],[57,6434,6436,6462],{"className":6435},[1087,1088],[57,6437,6439,6459],{"className":6438},[1092],[57,6440,6442],{"className":6441,"style":1162},[1096],[57,6443,6444,6447],{"style":1165},[57,6445],{"className":6446,"style":1169},[1104],[57,6448,6450],{"className":6449},[1109,1173,1174,1112],[57,6451,6453],{"className":6452},[1075,1112],[57,6454,6456],{"className":6455},[1075,6043,1112],[57,6457,4492],{"className":6458},[1075,1112],[57,6460,1183],{"className":6461},[1182],[57,6463,6465],{"className":6464},[1092],[57,6466,6468],{"className":6467,"style":1190},[1096],[57,6469],{},"，最终输出为 ",[57,6472,6474,6497],{"className":6473},[1019],[57,6475,6477],{"className":6476},[1023],[1025,6478,6479],{"xmlns":1027},[1029,6480,6481,6494],{},[1032,6482,6483,6485,6488],{},[1047,6484,3160],{"mathvariant":6079},[1414,6486,6487],{},"+",[1032,6489,6490,6492],{},[1047,6491,6333],{"mathvariant":6079},[1047,6493,6336],{"mathvariant":6079},[1054,6495,6496],{"encoding":1056},"\\mathbf{X} + \\mathbf{PE}",[57,6498,6500,6519],{"className":6499,"ariaHidden":1062},[1061],[57,6501,6503,6507,6510,6513,6516],{"className":6502},[1066],[57,6504],{"className":6505,"style":6506},[1070],"height:0.7694em;vertical-align:-0.0833em;",[57,6508,3160],{"className":6509},[1075,6114],[57,6511],{"className":6512,"style":2595},[1452],[57,6514,6487],{"className":6515},[1506],[57,6517],{"className":6518,"style":2595},[1452],[57,6520,6522,6526],{"className":6521},[1066],[57,6523],{"className":6524,"style":6525},[1070],"height:0.6861em;",[57,6527,6529],{"className":6528},[1075],[57,6530,6373],{"className":6531},[1075,6114],"。其中第 ",[57,6534,6536,6549],{"className":6535},[1019],[57,6537,6539],{"className":6538},[1023],[1025,6540,6541],{"xmlns":1027},[1029,6542,6543,6547],{},[1032,6544,6545],{},[1047,6546,1745],{},[1054,6548,1745],{"encoding":1056},[57,6550,6552],{"className":6551,"ariaHidden":1062},[1061],[57,6553,6555,6559],{"className":6554},[1066],[57,6556],{"className":6557,"style":6558},[1070],"height:0.6595em;",[57,6560,1745],{"className":6561},[1075,1148]," 行、第 ",[57,6564,6566,6583],{"className":6565},[1019],[57,6567,6569],{"className":6568},[1023],[1025,6570,6571],{"xmlns":1027},[1029,6572,6573,6580],{},[1032,6574,6575,6577],{},[1038,6576,590],{},[1047,6578,6579],{},"j",[1054,6581,6582],{"encoding":1056},"2j",[57,6584,6586],{"className":6585,"ariaHidden":1062},[1061],[57,6587,6589,6593,6596],{"className":6588},[1066],[57,6590],{"className":6591,"style":6592},[1070],"height:0.854em;vertical-align:-0.1944em;",[57,6594,590],{"className":6595},[1075],[57,6597,6579],{"className":6598,"style":6599},[1075,1148],"margin-right:0.05724em;",[57,6601,6603,6623],{"className":6602},[1019],[57,6604,6606],{"className":6605},[1023],[1025,6607,6608],{"xmlns":1027},[1029,6609,6610,6620],{},[1032,6611,6612,6614,6616,6618],{},[1038,6613,590],{},[1047,6615,6579],{},[1414,6617,6487],{},[1038,6619,574],{},[1054,6621,6622],{"encoding":1056},"2j+1",[57,6624,6626,6647],{"className":6625,"ariaHidden":1062},[1061],[57,6627,6629,6632,6635,6638,6641,6644],{"className":6628},[1066],[57,6630],{"className":6631,"style":6592},[1070],[57,6633,590],{"className":6634},[1075],[57,6636,6579],{"className":6637,"style":6599},[1075,1148],[57,6639],{"className":6640,"style":2595},[1452],[57,6642,6487],{"className":6643},[1506],[57,6645],{"className":6646,"style":2595},[1452],[57,6648,6650,6653],{"className":6649},[1066],[57,6651],{"className":6652,"style":2489},[1070],[57,6654,574],{"className":6655},[1075]," 列上的元素分别为：",[14,6658,6659],{},[57,6660,6662,6730],{"className":6661},[1019],[57,6663,6665],{"className":6664},[1023],[1025,6666,6667],{"xmlns":1027},[1029,6668,6669,6727],{},[1032,6670,6671,6673,6691,6693,6696,6698],{},[1047,6672,6333],{},[1044,6674,6675,6677],{},[1047,6676,6336],{},[1032,6678,6679,6681,6683,6685,6687,6689],{},[1414,6680,1754],{"stretchy":1753},[1047,6682,1745],{},[1414,6684,1759],{"separator":1062},[1038,6686,590],{},[1047,6688,6579],{},[1414,6690,1768],{"stretchy":1753},[1414,6692,281],{},[1047,6694,6695],{},"sin",[1414,6697,1776],{},[1032,6699,6700,6702,6725],{},[1414,6701,1754],{"fence":1062},[1035,6703,6704,6706],{},[1047,6705,1745],{},[1418,6707,6708,6711],{},[1038,6709,6710],{},"10000",[1032,6712,6713,6715,6717,6719],{},[1038,6714,590],{},[1047,6716,6579],{},[1047,6718,3549],{"mathvariant":1729},[1044,6720,6721,6723],{},[1047,6722,1049],{},[3999,6724,4492],{},[1414,6726,1768],{"fence":1062},[1054,6728,6729],{"encoding":1056}," PE_{(i,2j)} = \\sin\\left(\\frac{i}{10000^{2j/d_{\\text{model}}}}\\right)",[57,6731,6733,6813],{"className":6732,"ariaHidden":1062},[1061],[57,6734,6736,6740,6743,6804,6807,6810],{"className":6735},[1066],[57,6737],{"className":6738,"style":6739},[1070],"height:1.0385em;vertical-align:-0.3552em;",[57,6741,6333],{"className":6742,"style":3907},[1075,1148],[57,6744,6746,6750],{"className":6745},[1075],[57,6747,6336],{"className":6748,"style":6749},[1075,1148],"margin-right:0.05764em;",[57,6751,6753],{"className":6752},[1152],[57,6754,6756,6795],{"className":6755},[1087,1088],[57,6757,6759,6792],{"className":6758},[1092],[57,6760,6762],{"className":6761,"style":1162},[1096],[57,6763,6765,6768],{"style":6764},"top:-2.5198em;margin-left:-0.0576em;margin-right:0.05em;",[57,6766],{"className":6767,"style":1493},[1104],[57,6769,6771],{"className":6770},[1109,1110,1111,1112],[57,6772,6774,6777,6780,6783,6786,6789],{"className":6773},[1075,1112],[57,6775,1754],{"className":6776},[1079,1112],[57,6778,1745],{"className":6779},[1075,1148,1112],[57,6781,1759],{"className":6782},[1849,1112],[57,6784,590],{"className":6785},[1075,1112],[57,6787,6579],{"className":6788,"style":6599},[1075,1148,1112],[57,6790,1768],{"className":6791},[1269,1112],[57,6793,1183],{"className":6794},[1182],[57,6796,6798],{"className":6797},[1092],[57,6799,6802],{"className":6800,"style":6801},[1096],"height:0.3552em;",[57,6803],{},[57,6805],{"className":6806,"style":1453},[1452],[57,6808,281],{"className":6809},[1457],[57,6811],{"className":6812,"style":1453},[1452],[57,6814,6816,6819,6822,6825],{"className":6815},[1066],[57,6817],{"className":6818,"style":1884},[1070],[57,6820,6695],{"className":6821},[1888],[57,6823],{"className":6824,"style":1853},[1452],[57,6826,6828,6834,6993],{"className":6827},[1898],[57,6829,6831],{"className":6830,"style":1903},[1079,1902],[57,6832,1754],{"className":6833},[1907,1908],[57,6835,6837,6840,6990],{"className":6836},[1075],[57,6838],{"className":6839},[1079,1080],[57,6841,6843],{"className":6842},[1035],[57,6844,6846,6981],{"className":6845},[1087,1088],[57,6847,6849,6978],{"className":6848},[1092],[57,6850,6853,6956,6964],{"className":6851,"style":6852},[1096],"height:0.8557em;",[57,6854,6856,6859],{"style":6855},"top:-2.5648em;",[57,6857],{"className":6858,"style":1105},[1104],[57,6860,6862],{"className":6861},[1109,1110,1111,1112],[57,6863,6865,6869],{"className":6864},[1075,1112],[57,6866,6868],{"className":6867},[1075,1112],"1000",[57,6870,6872,6875],{"className":6871},[1075,1112],[57,6873,560],{"className":6874},[1075,1112],[57,6876,6878],{"className":6877},[1152],[57,6879,6881],{"className":6880},[1087],[57,6882,6884],{"className":6883},[1092],[57,6885,6888],{"className":6886,"style":6887},[1096],"height:0.8932em;",[57,6889,6891,6895],{"style":6890},"top:-2.8932em;margin-right:0.0714em;",[57,6892],{"className":6893,"style":6894},[1104],"height:2.5357em;",[57,6896,6898],{"className":6897},[1109,1173,1174,1112],[57,6899,6901,6904,6907,6910],{"className":6900},[1075,1112],[57,6902,590],{"className":6903},[1075,1112],[57,6905,6579],{"className":6906,"style":6599},[1075,1148,1112],[57,6908,3549],{"className":6909},[1075,1112],[57,6911,6913,6916],{"className":6912},[1075,1112],[57,6914,1049],{"className":6915},[1075,1148,1112],[57,6917,6919],{"className":6918},[1152],[57,6920,6922,6947],{"className":6921},[1087,1088],[57,6923,6925,6944],{"className":6924},[1092],[57,6926,6928],{"className":6927,"style":1162},[1096],[57,6929,6931,6935],{"style":6930},"top:-2.3448em;margin-left:0em;margin-right:0.1em;",[57,6932],{"className":6933,"style":6934},[1104],"height:2.6944em;",[57,6936,6938],{"className":6937},[1075,1112],[57,6939,6941],{"className":6940},[1075,6043,1112],[57,6942,4492],{"className":6943},[1075,1112],[57,6945,1183],{"className":6946},[1182],[57,6948,6950],{"className":6949},[1092],[57,6951,6954],{"className":6952,"style":6953},[1096],"height:0.3496em;",[57,6955],{},[57,6957,6958,6961],{"style":1230},[57,6959],{"className":6960,"style":1105},[1104],[57,6962],{"className":6963,"style":1238},[1237],[57,6965,6966,6969],{"style":1241},[57,6967],{"className":6968,"style":1105},[1104],[57,6970,6972],{"className":6971},[1109,1110,1111,1112],[57,6973,6975],{"className":6974},[1075,1112],[57,6976,1745],{"className":6977},[1075,1148,1112],[57,6979,1183],{"className":6980},[1182],[57,6982,6984],{"className":6983},[1092],[57,6985,6988],{"className":6986,"style":6987},[1096],"height:0.4352em;",[57,6989],{},[57,6991],{"className":6992},[1269,1080],[57,6994,6996],{"className":6995,"style":1903},[1269,1902],[57,6997,1768],{"className":6998},[1907,1908],[14,7000,7001],{},[57,7002,7004,7075],{"className":7003},[1019],[57,7005,7007],{"className":7006},[1023],[1025,7008,7009],{"xmlns":1027},[1029,7010,7011,7072],{},[1032,7012,7013,7015,7037,7039,7042,7044],{},[1047,7014,6333],{},[1044,7016,7017,7019],{},[1047,7018,6336],{},[1032,7020,7021,7023,7025,7027,7029,7031,7033,7035],{},[1414,7022,1754],{"stretchy":1753},[1047,7024,1745],{},[1414,7026,1759],{"separator":1062},[1038,7028,590],{},[1047,7030,6579],{},[1414,7032,6487],{},[1038,7034,574],{},[1414,7036,1768],{"stretchy":1753},[1414,7038,281],{},[1047,7040,7041],{},"cos",[1414,7043,1776],{},[1032,7045,7046,7048,7070],{},[1414,7047,1754],{"fence":1062},[1035,7049,7050,7052],{},[1047,7051,1745],{},[1418,7053,7054,7056],{},[1038,7055,6710],{},[1032,7057,7058,7060,7062,7064],{},[1038,7059,590],{},[1047,7061,6579],{},[1047,7063,3549],{"mathvariant":1729},[1044,7065,7066,7068],{},[1047,7067,1049],{},[3999,7069,4492],{},[1414,7071,1768],{"fence":1062},[1054,7073,7074],{"encoding":1056}," PE_{(i,2j+1)} = \\cos\\left(\\frac{i}{10000^{2j/d_{\\text{model}}}}\\right)",[57,7076,7078,7160],{"className":7077,"ariaHidden":1062},[1061],[57,7079,7081,7084,7087,7151,7154,7157],{"className":7080},[1066],[57,7082],{"className":7083,"style":6739},[1070],[57,7085,6333],{"className":7086,"style":3907},[1075,1148],[57,7088,7090,7093],{"className":7089},[1075],[57,7091,6336],{"className":7092,"style":6749},[1075,1148],[57,7094,7096],{"className":7095},[1152],[57,7097,7099,7143],{"className":7098},[1087,1088],[57,7100,7102,7140],{"className":7101},[1092],[57,7103,7105],{"className":7104,"style":1162},[1096],[57,7106,7107,7110],{"style":6764},[57,7108],{"className":7109,"style":1493},[1104],[57,7111,7113],{"className":7112},[1109,1110,1111,1112],[57,7114,7116,7119,7122,7125,7128,7131,7134,7137],{"className":7115},[1075,1112],[57,7117,1754],{"className":7118},[1079,1112],[57,7120,1745],{"className":7121},[1075,1148,1112],[57,7123,1759],{"className":7124},[1849,1112],[57,7126,590],{"className":7127},[1075,1112],[57,7129,6579],{"className":7130,"style":6599},[1075,1148,1112],[57,7132,6487],{"className":7133},[1506,1112],[57,7135,574],{"className":7136},[1075,1112],[57,7138,1768],{"className":7139},[1269,1112],[57,7141,1183],{"className":7142},[1182],[57,7144,7146],{"className":7145},[1092],[57,7147,7149],{"className":7148,"style":6801},[1096],[57,7150],{},[57,7152],{"className":7153,"style":1453},[1452],[57,7155,281],{"className":7156},[1457],[57,7158],{"className":7159,"style":1453},[1452],[57,7161,7163,7166,7169,7172],{"className":7162},[1066],[57,7164],{"className":7165,"style":1884},[1070],[57,7167,7041],{"className":7168},[1888],[57,7170],{"className":7171,"style":1853},[1452],[57,7173,7175,7181,7330],{"className":7174},[1898],[57,7176,7178],{"className":7177,"style":1903},[1079,1902],[57,7179,1754],{"className":7180},[1907,1908],[57,7182,7184,7187,7327],{"className":7183},[1075],[57,7185],{"className":7186},[1079,1080],[57,7188,7190],{"className":7189},[1035],[57,7191,7193,7319],{"className":7192},[1087,1088],[57,7194,7196,7316],{"className":7195},[1092],[57,7197,7199,7294,7302],{"className":7198,"style":6852},[1096],[57,7200,7201,7204],{"style":6855},[57,7202],{"className":7203,"style":1105},[1104],[57,7205,7207],{"className":7206},[1109,1110,1111,1112],[57,7208,7210,7213],{"className":7209},[1075,1112],[57,7211,6868],{"className":7212},[1075,1112],[57,7214,7216,7219],{"className":7215},[1075,1112],[57,7217,560],{"className":7218},[1075,1112],[57,7220,7222],{"className":7221},[1152],[57,7223,7225],{"className":7224},[1087],[57,7226,7228],{"className":7227},[1092],[57,7229,7231],{"className":7230,"style":6887},[1096],[57,7232,7233,7236],{"style":6890},[57,7234],{"className":7235,"style":6894},[1104],[57,7237,7239],{"className":7238},[1109,1173,1174,1112],[57,7240,7242,7245,7248,7251],{"className":7241},[1075,1112],[57,7243,590],{"className":7244},[1075,1112],[57,7246,6579],{"className":7247,"style":6599},[1075,1148,1112],[57,7249,3549],{"className":7250},[1075,1112],[57,7252,7254,7257],{"className":7253},[1075,1112],[57,7255,1049],{"className":7256},[1075,1148,1112],[57,7258,7260],{"className":7259},[1152],[57,7261,7263,7286],{"className":7262},[1087,1088],[57,7264,7266,7283],{"className":7265},[1092],[57,7267,7269],{"className":7268,"style":1162},[1096],[57,7270,7271,7274],{"style":6930},[57,7272],{"className":7273,"style":6934},[1104],[57,7275,7277],{"className":7276},[1075,1112],[57,7278,7280],{"className":7279},[1075,6043,1112],[57,7281,4492],{"className":7282},[1075,1112],[57,7284,1183],{"className":7285},[1182],[57,7287,7289],{"className":7288},[1092],[57,7290,7292],{"className":7291,"style":6953},[1096],[57,7293],{},[57,7295,7296,7299],{"style":1230},[57,7297],{"className":7298,"style":1105},[1104],[57,7300],{"className":7301,"style":1238},[1237],[57,7303,7304,7307],{"style":1241},[57,7305],{"className":7306,"style":1105},[1104],[57,7308,7310],{"className":7309},[1109,1110,1111,1112],[57,7311,7313],{"className":7312},[1075,1112],[57,7314,1745],{"className":7315},[1075,1148,1112],[57,7317,1183],{"className":7318},[1182],[57,7320,7322],{"className":7321},[1092],[57,7323,7325],{"className":7324,"style":6987},[1096],[57,7326],{},[57,7328],{"className":7329},[1269,1080],[57,7331,7333],{"className":7332,"style":1903},[1269,1902],[57,7334,1768],{"className":7335},[1907,1908],[14,7337,3935,7338,7366,7367,7395,7396,7431,7432,7488,7489,7517,7518,7611,7612,7688],{},[57,7339,7341,7354],{"className":7340},[1019],[57,7342,7344],{"className":7343},[1023],[1025,7345,7346],{"xmlns":1027},[1029,7347,7348,7352],{},[1032,7349,7350],{},[1047,7351,1745],{},[1054,7353,1745],{"encoding":1056},[57,7355,7357],{"className":7356,"ariaHidden":1062},[1061],[57,7358,7360,7363],{"className":7359},[1066],[57,7361],{"className":7362,"style":6558},[1070],[57,7364,1745],{"className":7365},[1075,1148]," 表示位置，",[57,7368,7370,7383],{"className":7369},[1019],[57,7371,7373],{"className":7372},[1023],[1025,7374,7375],{"xmlns":1027},[1029,7376,7377,7381],{},[1032,7378,7379],{},[1047,7380,6579],{},[1054,7382,6579],{"encoding":1056},[57,7384,7386],{"className":7385,"ariaHidden":1062},[1061],[57,7387,7389,7392],{"className":7388},[1066],[57,7390],{"className":7391,"style":6592},[1070],[57,7393,6579],{"className":7394,"style":6599},[1075,1148]," 表示维度索引。也就是说，位置编码的每一维都对应一条不同频率的正弦/余弦曲线，波长从 ",[57,7397,7399,7416],{"className":7398},[1019],[57,7400,7402],{"className":7401},[1023],[1025,7403,7404],{"xmlns":1027},[1029,7405,7406,7413],{},[1032,7407,7408,7410],{},[1038,7409,590],{},[1047,7411,7412],{},"π",[1054,7414,7415],{"encoding":1056},"2\\pi",[57,7417,7419],{"className":7418,"ariaHidden":1062},[1061],[57,7420,7422,7425,7428],{"className":7421},[1066],[57,7423],{"className":7424,"style":2489},[1070],[57,7426,590],{"className":7427},[1075],[57,7429,7412],{"className":7430,"style":1394},[1075,1148]," 到 ",[57,7433,7435,7455],{"className":7434},[1019],[57,7436,7438],{"className":7437},[1023],[1025,7439,7440],{"xmlns":1027},[1029,7441,7442,7452],{},[1032,7443,7444,7446,7448,7450],{},[1038,7445,6710],{},[1414,7447,2539],{},[1038,7449,590],{},[1047,7451,7412],{},[1054,7453,7454],{"encoding":1056},"10000 \\cdot 2\\pi",[57,7456,7458,7476],{"className":7457,"ariaHidden":1062},[1061],[57,7459,7461,7464,7467,7470,7473],{"className":7460},[1066],[57,7462],{"className":7463,"style":2489},[1070],[57,7465,6710],{"className":7466},[1075],[57,7468],{"className":7469,"style":2595},[1452],[57,7471,2539],{"className":7472},[1506],[57,7474],{"className":7475,"style":2595},[1452],[57,7477,7479,7482,7485],{"className":7478},[1066],[57,7480],{"className":7481,"style":2489},[1070],[57,7483,590],{"className":7484},[1075],[57,7486,7412],{"className":7487,"style":1394},[1075,1148]," 按几何级数增长。选择这一形式的直觉在于：对于任意固定偏移 ",[57,7490,7492,7505],{"className":7491},[1019],[57,7493,7495],{"className":7494},[1023],[1025,7496,7497],{"xmlns":1027},[1029,7498,7499,7503],{},[1032,7500,7501],{},[1047,7502,1052],{},[1054,7504,1052],{"encoding":1056},[57,7506,7508],{"className":7507,"ariaHidden":1062},[1061],[57,7509,7511,7514],{"className":7510},[1066],[57,7512],{"className":7513,"style":1360},[1070],[57,7515,1052],{"className":7516,"style":1178},[1075,1148],"，",[57,7519,7521,7547],{"className":7520},[1019],[57,7522,7524],{"className":7523},[1023],[1025,7525,7526],{"xmlns":1027},[1029,7527,7528,7544],{},[1032,7529,7530,7532],{},[1047,7531,6333],{},[1044,7533,7534,7536],{},[1047,7535,6336],{},[1032,7537,7538,7540,7542],{},[1047,7539,1745],{},[1414,7541,6487],{},[1047,7543,1052],{},[1054,7545,7546],{"encoding":1056},"PE_{i+k}",[57,7548,7550],{"className":7549,"ariaHidden":1062},[1061],[57,7551,7553,7557,7560],{"className":7552},[1066],[57,7554],{"className":7555,"style":7556},[1070],"height:0.8917em;vertical-align:-0.2083em;",[57,7558,6333],{"className":7559,"style":3907},[1075,1148],[57,7561,7563,7566],{"className":7562},[1075],[57,7564,6336],{"className":7565,"style":6749},[1075,1148],[57,7567,7569],{"className":7568},[1152],[57,7570,7572,7602],{"className":7571},[1087,1088],[57,7573,7575,7599],{"className":7574},[1092],[57,7576,7578],{"className":7577,"style":2208},[1096],[57,7579,7581,7584],{"style":7580},"top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;",[57,7582],{"className":7583,"style":1493},[1104],[57,7585,7587],{"className":7586},[1109,1110,1111,1112],[57,7588,7590,7593,7596],{"className":7589},[1075,1112],[57,7591,1745],{"className":7592},[1075,1148,1112],[57,7594,6487],{"className":7595},[1506,1112],[57,7597,1052],{"className":7598,"style":1178},[1075,1148,1112],[57,7600,1183],{"className":7601},[1182],[57,7603,7605],{"className":7604},[1092],[57,7606,7609],{"className":7607,"style":7608},[1096],"height:0.2083em;",[57,7610],{}," 都可以表示为 ",[57,7613,7615,7635],{"className":7614},[1019],[57,7616,7618],{"className":7617},[1023],[1025,7619,7620],{"xmlns":1027},[1029,7621,7622,7632],{},[1032,7623,7624,7626],{},[1047,7625,6333],{},[1044,7627,7628,7630],{},[1047,7629,6336],{},[1047,7631,1745],{},[1054,7633,7634],{"encoding":1056},"PE_i",[57,7636,7638],{"className":7637,"ariaHidden":1062},[1061],[57,7639,7641,7645,7648],{"className":7640},[1066],[57,7642],{"className":7643,"style":7644},[1070],"height:0.8333em;vertical-align:-0.15em;",[57,7646,6333],{"className":7647,"style":3907},[1075,1148],[57,7649,7651,7654],{"className":7650},[1075],[57,7652,6336],{"className":7653,"style":6749},[1075,1148],[57,7655,7657],{"className":7656},[1152],[57,7658,7660,7680],{"className":7659},[1087,1088],[57,7661,7663,7677],{"className":7662},[1092],[57,7664,7666],{"className":7665,"style":2755},[1096],[57,7667,7668,7671],{"style":7580},[57,7669],{"className":7670,"style":1493},[1104],[57,7672,7674],{"className":7673},[1109,1110,1111,1112],[57,7675,1745],{"className":7676},[1075,1148,1112],[57,7678,1183],{"className":7679},[1182],[57,7681,7683],{"className":7682},[1092],[57,7684,7686],{"className":7685,"style":2230},[1096],[57,7687],{}," 的线性变换，从而让模型更容易学到\"相对位置\"这种关系。",[47,7690,7692],{"className":256,"code":7691,"language":258,"meta":52,"style":52},"class PositionalEncoding(nn.Block):\n    \"\"\"位置编码\"\"\"\n    def __init__(self, num_hiddens, dropout, max_len=1000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(dropout)\n        # 创建一个足够长的P\n        self.P = np.zeros((1, max_len, num_hiddens))\n        X = np.arange(max_len).reshape(-1, 1) / np.power(\n            10000, np.arange(0, num_hiddens, 2) / num_hiddens)\n        self.P[:, :, 0::2] = np.sin(X)\n        self.P[:, :, 1::2] = np.cos(X)\n\n    def forward(self, X):\n        X = X + self.P[:, :X.shape[1], :].as_in_ctx(X.ctx)\n        return self.dropout(X)\n",[54,7693,7694,7712,7717,7746,7762,7772,7777,7794,7819,7841,7872,7899,7903,7919,7952],{"__ignoreMap":52},[57,7695,7696,7698,7701,7703,7705,7707,7710],{"class":59,"line":60},[57,7697,3357],{"class":280},[57,7699,7700],{"class":3360}," PositionalEncoding",[57,7702,1754],{"class":269},[57,7704,3367],{"class":3366},[57,7706,1818],{"class":269},[57,7708,7709],{"class":3366},"Block",[57,7711,3169],{"class":269},[57,7713,7714],{"class":59,"line":66},[57,7715,7716],{"class":3174},"    \"\"\"位置编码\"\"\"\n",[57,7718,7719,7721,7723,7725,7727,7729,7731,7733,7735,7737,7740,7742,7744],{"class":59,"line":72},[57,7720,3384],{"class":280},[57,7722,3387],{"class":273},[57,7724,1754],{"class":269},[57,7726,266],{"class":3392},[57,7728,3163],{"class":269},[57,7730,5581],{"class":3159},[57,7732,3163],{"class":269},[57,7734,3397],{"class":3159},[57,7736,3163],{"class":269},[57,7738,7739],{"class":3159},"max_len",[57,7741,281],{"class":280},[57,7743,6868],{"class":273},[57,7745,3169],{"class":269},[57,7747,7748,7750,7753,7755,7757,7759],{"class":59,"line":78},[57,7749,3413],{"class":3412},[57,7751,7752],{"class":269},"(PositionalEncoding, ",[57,7754,266],{"class":265},[57,7756,3421],{"class":269},[57,7758,869],{"class":273},[57,7760,7761],{"class":269},"()\n",[57,7763,7764,7766,7768,7770],{"class":59,"line":84},[57,7765,3435],{"class":265},[57,7767,3438],{"class":269},[57,7769,281],{"class":280},[57,7771,3443],{"class":269},[57,7773,7774],{"class":59,"line":90},[57,7775,7776],{"class":303},"        # 创建一个足够长的P\n",[57,7778,7779,7781,7784,7786,7789,7791],{"class":59,"line":96},[57,7780,3435],{"class":265},[57,7782,7783],{"class":269},".P ",[57,7785,281],{"class":280},[57,7787,7788],{"class":269}," np.zeros((",[57,7790,574],{"class":273},[57,7792,7793],{"class":269},", max_len, num_hiddens))\n",[57,7795,7796,7798,7800,7803,7805,7807,7809,7811,7814,7816],{"class":59,"line":102},[57,7797,3298],{"class":269},[57,7799,281],{"class":280},[57,7801,7802],{"class":269}," np.arange(max_len).reshape(",[57,7804,3284],{"class":280},[57,7806,574],{"class":273},[57,7808,3163],{"class":269},[57,7810,574],{"class":273},[57,7812,7813],{"class":269},") ",[57,7815,3549],{"class":280},[57,7817,7818],{"class":269}," np.power(\n",[57,7820,7821,7824,7827,7829,7832,7834,7836,7838],{"class":59,"line":108},[57,7822,7823],{"class":273},"            10000",[57,7825,7826],{"class":269},", np.arange(",[57,7828,560],{"class":273},[57,7830,7831],{"class":269},", num_hiddens, ",[57,7833,590],{"class":273},[57,7835,7813],{"class":269},[57,7837,3549],{"class":280},[57,7839,7840],{"class":269}," num_hiddens)\n",[57,7842,7843,7845,7848,7852,7854,7856,7858,7860,7863,7865,7867,7869],{"class":59,"line":114},[57,7844,3435],{"class":265},[57,7846,7847],{"class":269},".P[",[57,7849,7851],{"class":7850},"sDoOe",":",[57,7853,3163],{"class":269},[57,7855,7851],{"class":7850},[57,7857,3163],{"class":269},[57,7859,560],{"class":273},[57,7861,7862],{"class":7850},"::",[57,7864,590],{"class":273},[57,7866,277],{"class":269},[57,7868,281],{"class":280},[57,7870,7871],{"class":269}," np.sin(X)\n",[57,7873,7874,7876,7878,7880,7882,7884,7886,7888,7890,7892,7894,7896],{"class":59,"line":120},[57,7875,3435],{"class":265},[57,7877,7847],{"class":269},[57,7879,7851],{"class":7850},[57,7881,3163],{"class":269},[57,7883,7851],{"class":7850},[57,7885,3163],{"class":269},[57,7887,574],{"class":273},[57,7889,7862],{"class":7850},[57,7891,590],{"class":273},[57,7893,277],{"class":269},[57,7895,281],{"class":280},[57,7897,7898],{"class":269}," np.cos(X)\n",[57,7900,7901],{"class":59,"line":126},[57,7902,3448],{"emptyLinePlaceholder":963},[57,7904,7905,7907,7909,7911,7913,7915,7917],{"class":59,"line":132},[57,7906,3384],{"class":280},[57,7908,3475],{"class":3153},[57,7910,1754],{"class":269},[57,7912,266],{"class":3392},[57,7914,3163],{"class":269},[57,7916,3160],{"class":3159},[57,7918,3169],{"class":269},[57,7920,7921,7923,7925,7928,7930,7932,7934,7936,7938,7940,7943,7945,7947,7949],{"class":59,"line":138},[57,7922,3298],{"class":269},[57,7924,281],{"class":280},[57,7926,7927],{"class":269}," X ",[57,7929,6487],{"class":280},[57,7931,5935],{"class":265},[57,7933,7847],{"class":269},[57,7935,7851],{"class":7850},[57,7937,3163],{"class":269},[57,7939,7851],{"class":7850},[57,7941,7942],{"class":269},"X.shape[",[57,7944,574],{"class":273},[57,7946,5513],{"class":269},[57,7948,7851],{"class":7850},[57,7950,7951],{"class":269},"].as_in_ctx(X.ctx)\n",[57,7953,7954,7956,7958],{"class":59,"line":144},[57,7955,3202],{"class":280},[57,7957,5935],{"class":265},[57,7959,7960],{"class":269},".dropout(X)\n",[10,7962,7963],{"id":7963},"前馈神经网络",[14,7965,7966],{},"除了注意力子层，编码器与解码器的每一层都包含一个 position-wise 的全连接前馈网络（FFN）：它对序列中每个位置独立、且共享权重地施加同一组变换，包含两个线性层与中间的 ReLU 激活：",[14,7968,7969],{},[57,7970,7972,8050],{"className":7971},[1019],[57,7973,7975],{"className":7974},[1023],[1025,7976,7977],{"xmlns":1027},[1029,7978,7979,8047],{},[1032,7980,7981,7991,7993,7996,7998,8000,8003,8005,8007,8009,8011,8014,8016,8022,8024,8031,8033,8039,8041],{},[1032,7982,7983,7986,7988],{},[1047,7984,7985],{"mathvariant":1729},"F",[1047,7987,7985],{"mathvariant":1729},[1047,7989,7990],{"mathvariant":1729},"N",[1414,7992,1754],{"stretchy":1753},[1047,7994,7995],{},"x",[1414,7997,1768],{"stretchy":1753},[1414,7999,281],{},[1047,8001,8002],{},"max",[1414,8004,1776],{},[1414,8006,1754],{"stretchy":1753},[1038,8008,560],{},[1414,8010,1759],{"separator":1062},[3999,8012,8013],{}," ",[1047,8015,7995],{},[1044,8017,8018,8020],{},[1047,8019,746],{},[1038,8021,574],{},[1414,8023,6487],{},[1044,8025,8026,8029],{},[1047,8027,8028],{},"b",[1038,8030,574],{},[1414,8032,1768],{"stretchy":1753},[1044,8034,8035,8037],{},[1047,8036,746],{},[1038,8038,590],{},[1414,8040,6487],{},[1044,8042,8043,8045],{},[1047,8044,8028],{},[1038,8046,590],{},[1054,8048,8049],{"encoding":1056}," \\mathrm{FFN}(x) = \\max(0,\\,xW_1+b_1)W_2+b_2",[57,8051,8053,8084,8161,8259],{"className":8052,"ariaHidden":1062},[1061],[57,8054,8056,8059,8066,8069,8072,8075,8078,8081],{"className":8055},[1066],[57,8057],{"className":8058,"style":1831},[1070],[57,8060,8062],{"className":8061},[1075],[57,8063,8065],{"className":8064},[1075,1838],"FFN",[57,8067,1754],{"className":8068},[1079],[57,8070,7995],{"className":8071},[1075,1148],[57,8073,1768],{"className":8074},[1269],[57,8076],{"className":8077,"style":1453},[1452],[57,8079,281],{"className":8080},[1457],[57,8082],{"className":8083,"style":1453},[1452],[57,8085,8087,8090,8093,8096,8099,8102,8105,8108,8111,8152,8155,8158],{"className":8086},[1066],[57,8088],{"className":8089,"style":1831},[1070],[57,8091,8002],{"className":8092},[1888],[57,8094,1754],{"className":8095},[1079],[57,8097,560],{"className":8098},[1075],[57,8100,1759],{"className":8101},[1849],[57,8103],{"className":8104,"style":1853},[1452],[57,8106],{"className":8107,"style":1853},[1452],[57,8109,7995],{"className":8110},[1075,1148],[57,8112,8114,8117],{"className":8113},[1075],[57,8115,746],{"className":8116,"style":3907},[1075,1148],[57,8118,8120],{"className":8119},[1152],[57,8121,8123,8144],{"className":8122},[1087,1088],[57,8124,8126,8141],{"className":8125},[1092],[57,8127,8129],{"className":8128,"style":3813},[1096],[57,8130,8132,8135],{"style":8131},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[57,8133],{"className":8134,"style":1493},[1104],[57,8136,8138],{"className":8137},[1109,1110,1111,1112],[57,8139,574],{"className":8140},[1075,1112],[57,8142,1183],{"className":8143},[1182],[57,8145,8147],{"className":8146},[1092],[57,8148,8150],{"className":8149,"style":2230},[1096],[57,8151],{},[57,8153],{"className":8154,"style":2595},[1452],[57,8156,6487],{"className":8157},[1506],[57,8159],{"className":8160,"style":2595},[1452],[57,8162,8164,8167,8207,8210,8250,8253,8256],{"className":8163},[1066],[57,8165],{"className":8166,"style":1831},[1070],[57,8168,8170,8173],{"className":8169},[1075],[57,8171,8028],{"className":8172},[1075,1148],[57,8174,8176],{"className":8175},[1152],[57,8177,8179,8199],{"className":8178},[1087,1088],[57,8180,8182,8196],{"className":8181},[1092],[57,8183,8185],{"className":8184,"style":3813},[1096],[57,8186,8187,8190],{"style":2211},[57,8188],{"className":8189,"style":1493},[1104],[57,8191,8193],{"className":8192},[1109,1110,1111,1112],[57,8194,574],{"className":8195},[1075,1112],[57,8197,1183],{"className":8198},[1182],[57,8200,8202],{"className":8201},[1092],[57,8203,8205],{"className":8204,"style":2230},[1096],[57,8206],{},[57,8208,1768],{"className":8209},[1269],[57,8211,8213,8216],{"className":8212},[1075],[57,8214,746],{"className":8215,"style":3907},[1075,1148],[57,8217,8219],{"className":8218},[1152],[57,8220,8222,8242],{"className":8221},[1087,1088],[57,8223,8225,8239],{"className":8224},[1092],[57,8226,8228],{"className":8227,"style":3813},[1096],[57,8229,8230,8233],{"style":8131},[57,8231],{"className":8232,"style":1493},[1104],[57,8234,8236],{"className":8235},[1109,1110,1111,1112],[57,8237,590],{"className":8238},[1075,1112],[57,8240,1183],{"className":8241},[1182],[57,8243,8245],{"className":8244},[1092],[57,8246,8248],{"className":8247,"style":2230},[1096],[57,8249],{},[57,8251],{"className":8252,"style":2595},[1452],[57,8254,6487],{"className":8255},[1506],[57,8257],{"className":8258,"style":2595},[1452],[57,8260,8262,8265],{"className":8261},[1066],[57,8263],{"className":8264,"style":2292},[1070],[57,8266,8268,8271],{"className":8267},[1075],[57,8269,8028],{"className":8270},[1075,1148],[57,8272,8274],{"className":8273},[1152],[57,8275,8277,8297],{"className":8276},[1087,1088],[57,8278,8280,8294],{"className":8279},[1092],[57,8281,8283],{"className":8282,"style":3813},[1096],[57,8284,8285,8288],{"style":2211},[57,8286],{"className":8287,"style":1493},[1104],[57,8289,8291],{"className":8290},[1109,1110,1111,1112],[57,8292,590],{"className":8293},[1075,1112],[57,8295,1183],{"className":8296},[1182],[57,8298,8300],{"className":8299},[1092],[57,8301,8303],{"className":8302,"style":2230},[1096],[57,8304],{},[14,8306,8307],{},"直观上，注意力层负责\"在序列内交换信息\"，而 FFN 负责\"对每个位置做非线性变换\"，二者交替堆叠，使模型既能整合上下文又能拟合复杂的特征映射。",[47,8309,8311],{"className":256,"code":8310,"language":258,"meta":52,"style":52},"class PositionWiseFFN(nn.Module):\n    \"\"\"基于位置的前馈网络\"\"\"\n    def __init__(self, ffn_num_input, ffn_num_hiddens, ffn_num_outputs,\n                 **kwargs):\n        super(PositionWiseFFN, self).__init__(**kwargs)\n        self.dense1 = nn.Linear(ffn_num_input, ffn_num_hiddens)\n        self.relu = nn.ReLU()\n        self.dense2 = nn.Linear(ffn_num_hiddens, ffn_num_outputs)\n\n    def forward(self, X):\n        return self.dense2(self.relu(self.dense1(X)))\n",[54,8312,8313,8330,8335,8362,8371,8390,8402,8414,8426,8430,8446],{"__ignoreMap":52},[57,8314,8315,8317,8320,8322,8324,8326,8328],{"class":59,"line":60},[57,8316,3357],{"class":280},[57,8318,8319],{"class":3360}," PositionWiseFFN",[57,8321,1754],{"class":269},[57,8323,3367],{"class":3366},[57,8325,1818],{"class":269},[57,8327,3372],{"class":3366},[57,8329,3169],{"class":269},[57,8331,8332],{"class":59,"line":66},[57,8333,8334],{"class":3174},"    \"\"\"基于位置的前馈网络\"\"\"\n",[57,8336,8337,8339,8341,8343,8345,8347,8350,8352,8355,8357,8360],{"class":59,"line":72},[57,8338,3384],{"class":280},[57,8340,3387],{"class":273},[57,8342,1754],{"class":269},[57,8344,266],{"class":3392},[57,8346,3163],{"class":269},[57,8348,8349],{"class":3159},"ffn_num_input",[57,8351,3163],{"class":269},[57,8353,8354],{"class":3159},"ffn_num_hiddens",[57,8356,3163],{"class":269},[57,8358,8359],{"class":3159},"ffn_num_outputs",[57,8361,369],{"class":269},[57,8363,8364,8367,8369],{"class":59,"line":78},[57,8365,8366],{"class":280},"                 **",[57,8368,3405],{"class":3159},[57,8370,3169],{"class":269},[57,8372,8373,8375,8378,8380,8382,8384,8386,8388],{"class":59,"line":84},[57,8374,3413],{"class":3412},[57,8376,8377],{"class":269},"(PositionWiseFFN, ",[57,8379,266],{"class":265},[57,8381,3421],{"class":269},[57,8383,869],{"class":273},[57,8385,1754],{"class":269},[57,8387,3402],{"class":280},[57,8389,3430],{"class":269},[57,8391,8392,8394,8397,8399],{"class":59,"line":90},[57,8393,3435],{"class":265},[57,8395,8396],{"class":269},".dense1 ",[57,8398,281],{"class":280},[57,8400,8401],{"class":269}," nn.Linear(ffn_num_input, ffn_num_hiddens)\n",[57,8403,8404,8406,8409,8411],{"class":59,"line":96},[57,8405,3435],{"class":265},[57,8407,8408],{"class":269},".relu ",[57,8410,281],{"class":280},[57,8412,8413],{"class":269}," nn.ReLU()\n",[57,8415,8416,8418,8421,8423],{"class":59,"line":102},[57,8417,3435],{"class":265},[57,8419,8420],{"class":269},".dense2 ",[57,8422,281],{"class":280},[57,8424,8425],{"class":269}," nn.Linear(ffn_num_hiddens, ffn_num_outputs)\n",[57,8427,8428],{"class":59,"line":108},[57,8429,3448],{"emptyLinePlaceholder":963},[57,8431,8432,8434,8436,8438,8440,8442,8444],{"class":59,"line":114},[57,8433,3384],{"class":280},[57,8435,3475],{"class":3153},[57,8437,1754],{"class":269},[57,8439,266],{"class":3392},[57,8441,3163],{"class":269},[57,8443,3160],{"class":3159},[57,8445,3169],{"class":269},[57,8447,8448,8450,8452,8455,8457,8460,8462],{"class":59,"line":120},[57,8449,3202],{"class":280},[57,8451,5935],{"class":265},[57,8453,8454],{"class":269},".dense2(",[57,8456,266],{"class":265},[57,8458,8459],{"class":269},".relu(",[57,8461,266],{"class":265},[57,8463,8464],{"class":269},".dense1(X)))\n",[10,8466,8467],{"id":8467},"残差连接和层规范化",[14,8469,8470],{},"每个子层（自注意力或 FFN）外面都包了一层残差连接（residual connection）和层归一化（LayerNorm）。残差连接缓解了深层网络的梯度消失问题，让信息和梯度能够沿\"恒等通路\"穿透多层；而 LayerNorm 沿特征维度做归一化，相比 BatchNorm 不依赖 batch 大小，更适合变长序列的场景。",[14,8472,8473,8474,8599,8600,8719],{},"在原始论文（Post-LN）中，每个子层的输出形式为 ",[57,8475,8477,8545],{"className":8476},[1019],[57,8478,8480],{"className":8479},[1023],[1025,8481,8482],{"xmlns":1027},[1029,8483,8484,8542],{},[1032,8485,8486,8509,8511,8513,8515,8534,8536,8538,8540],{},[1032,8487,8488,8491,8493,8496,8498,8501,8503,8505,8507],{},[1047,8489,8490],{"mathvariant":1729},"L",[1047,8492,18],{"mathvariant":1729},[1047,8494,8495],{"mathvariant":1729},"y",[1047,8497,1738],{"mathvariant":1729},[1047,8499,8500],{"mathvariant":1729},"r",[1047,8502,7990],{"mathvariant":1729},[1047,8504,1748],{"mathvariant":1729},[1047,8506,8500],{"mathvariant":1729},[1047,8508,1319],{"mathvariant":1729},[1414,8510,1754],{"stretchy":1753},[1047,8512,7995],{},[1414,8514,6487],{},[1032,8516,8517,8520,8522,8524,8526,8528,8530,8532],{},[1047,8518,8519],{"mathvariant":1729},"S",[1047,8521,3625],{"mathvariant":1729},[1047,8523,8028],{"mathvariant":1729},[1047,8525,3628],{"mathvariant":1729},[1047,8527,18],{"mathvariant":1729},[1047,8529,8495],{"mathvariant":1729},[1047,8531,1738],{"mathvariant":1729},[1047,8533,8500],{"mathvariant":1729},[1414,8535,1754],{"stretchy":1753},[1047,8537,7995],{},[1414,8539,1768],{"stretchy":1753},[1414,8541,1768],{"stretchy":1753},[1054,8543,8544],{"encoding":1056},"\\mathrm{LayerNorm}(x + \\mathrm{Sublayer}(x))",[57,8546,8548,8576],{"className":8547,"ariaHidden":1062},[1061],[57,8549,8551,8554,8561,8564,8567,8570,8573],{"className":8550},[1066],[57,8552],{"className":8553,"style":1831},[1070],[57,8555,8557],{"className":8556},[1075],[57,8558,8560],{"className":8559},[1075,1838],"LayerNorm",[57,8562,1754],{"className":8563},[1079],[57,8565,7995],{"className":8566},[1075,1148],[57,8568],{"className":8569,"style":2595},[1452],[57,8571,6487],{"className":8572},[1506],[57,8574],{"className":8575,"style":2595},[1452],[57,8577,8579,8582,8589,8592,8595],{"className":8578},[1066],[57,8580],{"className":8581,"style":1831},[1070],[57,8583,8585],{"className":8584},[1075],[57,8586,8588],{"className":8587},[1075,1838],"Sublayer",[57,8590,1754],{"className":8591},[1079],[57,8593,7995],{"className":8594},[1075,1148],[57,8596,8598],{"className":8597},[1269],"))","。我们在子层输出处先应用 dropout，再加回输入并归一化。注：现代实现中更常用 Pre-LN（即 ",[57,8601,8603,8667],{"className":8602},[1019],[57,8604,8606],{"className":8605},[1023],[1025,8607,8608],{"xmlns":1027},[1029,8609,8610,8664],{},[1032,8611,8612,8614,8616,8634,8636,8656,8658,8660,8662],{},[1047,8613,7995],{},[1414,8615,6487],{},[1032,8617,8618,8620,8622,8624,8626,8628,8630,8632],{},[1047,8619,8519],{"mathvariant":1729},[1047,8621,3625],{"mathvariant":1729},[1047,8623,8028],{"mathvariant":1729},[1047,8625,3628],{"mathvariant":1729},[1047,8627,18],{"mathvariant":1729},[1047,8629,8495],{"mathvariant":1729},[1047,8631,1738],{"mathvariant":1729},[1047,8633,8500],{"mathvariant":1729},[1414,8635,1754],{"stretchy":1753},[1032,8637,8638,8640,8642,8644,8646,8648,8650,8652,8654],{},[1047,8639,8490],{"mathvariant":1729},[1047,8641,18],{"mathvariant":1729},[1047,8643,8495],{"mathvariant":1729},[1047,8645,1738],{"mathvariant":1729},[1047,8647,8500],{"mathvariant":1729},[1047,8649,7990],{"mathvariant":1729},[1047,8651,1748],{"mathvariant":1729},[1047,8653,8500],{"mathvariant":1729},[1047,8655,1319],{"mathvariant":1729},[1414,8657,1754],{"stretchy":1753},[1047,8659,7995],{},[1414,8661,1768],{"stretchy":1753},[1414,8663,1768],{"stretchy":1753},[1054,8665,8666],{"encoding":1056},"x + \\mathrm{Sublayer}(\\mathrm{LayerNorm}(x))",[57,8668,8670,8689],{"className":8669,"ariaHidden":1062},[1061],[57,8671,8673,8677,8680,8683,8686],{"className":8672},[1066],[57,8674],{"className":8675,"style":8676},[1070],"height:0.6667em;vertical-align:-0.0833em;",[57,8678,7995],{"className":8679},[1075,1148],[57,8681],{"className":8682,"style":2595},[1452],[57,8684,6487],{"className":8685},[1506],[57,8687],{"className":8688,"style":2595},[1452],[57,8690,8692,8695,8701,8704,8710,8713,8716],{"className":8691},[1066],[57,8693],{"className":8694,"style":1831},[1070],[57,8696,8698],{"className":8697},[1075],[57,8699,8588],{"className":8700},[1075,1838],[57,8702,1754],{"className":8703},[1079],[57,8705,8707],{"className":8706},[1075],[57,8708,8560],{"className":8709},[1075,1838],[57,8711,1754],{"className":8712},[1079],[57,8714,7995],{"className":8715},[1075,1148],[57,8717,8598],{"className":8718},[1269],"），训练更稳定，对 warmup 也不那么敏感。",[47,8721,8723],{"className":256,"code":8722,"language":258,"meta":52,"style":52},"class AddNorm(nn.Module):\n    \"\"\"残差连接后进行层规范化\"\"\"\n    def __init__(self, normalized_shape, dropout, **kwargs):\n        super(AddNorm, self).__init__(**kwargs)\n        self.dropout = nn.Dropout(dropout)\n        self.ln = nn.LayerNorm(normalized_shape)\n\n    def forward(self, X, Y):\n        return self.ln(self.dropout(Y) + X)\n",[54,8724,8725,8742,8747,8774,8793,8803,8815,8819,8840],{"__ignoreMap":52},[57,8726,8727,8729,8732,8734,8736,8738,8740],{"class":59,"line":60},[57,8728,3357],{"class":280},[57,8730,8731],{"class":3360}," AddNorm",[57,8733,1754],{"class":269},[57,8735,3367],{"class":3366},[57,8737,1818],{"class":269},[57,8739,3372],{"class":3366},[57,8741,3169],{"class":269},[57,8743,8744],{"class":59,"line":66},[57,8745,8746],{"class":3174},"    \"\"\"残差连接后进行层规范化\"\"\"\n",[57,8748,8749,8751,8753,8755,8757,8759,8762,8764,8766,8768,8770,8772],{"class":59,"line":72},[57,8750,3384],{"class":280},[57,8752,3387],{"class":273},[57,8754,1754],{"class":269},[57,8756,266],{"class":3392},[57,8758,3163],{"class":269},[57,8760,8761],{"class":3159},"normalized_shape",[57,8763,3163],{"class":269},[57,8765,3397],{"class":3159},[57,8767,3163],{"class":269},[57,8769,3402],{"class":280},[57,8771,3405],{"class":3159},[57,8773,3169],{"class":269},[57,8775,8776,8778,8781,8783,8785,8787,8789,8791],{"class":59,"line":78},[57,8777,3413],{"class":3412},[57,8779,8780],{"class":269},"(AddNorm, ",[57,8782,266],{"class":265},[57,8784,3421],{"class":269},[57,8786,869],{"class":273},[57,8788,1754],{"class":269},[57,8790,3402],{"class":280},[57,8792,3430],{"class":269},[57,8794,8795,8797,8799,8801],{"class":59,"line":84},[57,8796,3435],{"class":265},[57,8798,3438],{"class":269},[57,8800,281],{"class":280},[57,8802,3443],{"class":269},[57,8804,8805,8807,8810,8812],{"class":59,"line":90},[57,8806,3435],{"class":265},[57,8808,8809],{"class":269},".ln ",[57,8811,281],{"class":280},[57,8813,8814],{"class":269}," nn.LayerNorm(normalized_shape)\n",[57,8816,8817],{"class":59,"line":96},[57,8818,3448],{"emptyLinePlaceholder":963},[57,8820,8821,8823,8825,8827,8829,8831,8833,8835,8838],{"class":59,"line":102},[57,8822,3384],{"class":280},[57,8824,3475],{"class":3153},[57,8826,1754],{"class":269},[57,8828,266],{"class":3392},[57,8830,3163],{"class":269},[57,8832,3160],{"class":3159},[57,8834,3163],{"class":269},[57,8836,8837],{"class":3159},"Y",[57,8839,3169],{"class":269},[57,8841,8842,8844,8846,8849,8851,8854,8856],{"class":59,"line":108},[57,8843,3202],{"class":280},[57,8845,5935],{"class":265},[57,8847,8848],{"class":269},".ln(",[57,8850,266],{"class":265},[57,8852,8853],{"class":269},".dropout(Y) ",[57,8855,6487],{"class":280},[57,8857,8858],{"class":269}," X)\n",[10,8860,8861],{"id":8861},"编码器-解码器架构",[14,8863,8864,8865,9016,9017,9193,9194,9223,9224,9374],{},"主流的神经序列转换模型大多采用编码器-解码器结构。编码器把输入符号序列 ",[57,8866,8868,8902],{"className":8867},[1019],[57,8869,8871],{"className":8870},[1023],[1025,8872,8873],{"xmlns":1027},[1029,8874,8875,8899],{},[1032,8876,8877,8879,8885,8887,8889,8891,8897],{},[1414,8878,1754],{"stretchy":1753},[1044,8880,8881,8883],{},[1047,8882,7995],{},[1038,8884,574],{},[1414,8886,1759],{"separator":1062},[1414,8888,3693],{},[1414,8890,1759],{"separator":1062},[1044,8892,8893,8895],{},[1047,8894,7995],{},[1047,8896,1288],{},[1414,8898,1768],{"stretchy":1753},[1054,8900,8901],{"encoding":1056},"(x_1, \\ldots, x_n)",[57,8903,8905],{"className":8904,"ariaHidden":1062},[1061],[57,8906,8908,8911,8914,8954,8957,8960,8963,8966,8969,8972,9013],{"className":8907},[1066],[57,8909],{"className":8910,"style":1831},[1070],[57,8912,1754],{"className":8913},[1079],[57,8915,8917,8920],{"className":8916},[1075],[57,8918,7995],{"className":8919},[1075,1148],[57,8921,8923],{"className":8922},[1152],[57,8924,8926,8946],{"className":8925},[1087,1088],[57,8927,8929,8943],{"className":8928},[1092],[57,8930,8932],{"className":8931,"style":3813},[1096],[57,8933,8934,8937],{"style":2211},[57,8935],{"className":8936,"style":1493},[1104],[57,8938,8940],{"className":8939},[1109,1110,1111,1112],[57,8941,574],{"className":8942},[1075,1112],[57,8944,1183],{"className":8945},[1182],[57,8947,8949],{"className":8948},[1092],[57,8950,8952],{"className":8951,"style":2230},[1096],[57,8953],{},[57,8955,1759],{"className":8956},[1849],[57,8958],{"className":8959,"style":1853},[1452],[57,8961,3693],{"className":8962},[1898],[57,8964],{"className":8965,"style":1853},[1452],[57,8967,1759],{"className":8968},[1849],[57,8970],{"className":8971,"style":1853},[1452],[57,8973,8975,8978],{"className":8974},[1075],[57,8976,7995],{"className":8977},[1075,1148],[57,8979,8981],{"className":8980},[1152],[57,8982,8984,9005],{"className":8983},[1087,1088],[57,8985,8987,9002],{"className":8986},[1092],[57,8988,8991],{"className":8989,"style":8990},[1096],"height:0.1514em;",[57,8992,8993,8996],{"style":2211},[57,8994],{"className":8995,"style":1493},[1104],[57,8997,8999],{"className":8998},[1109,1110,1111,1112],[57,9000,1288],{"className":9001},[1075,1148,1112],[57,9003,1183],{"className":9004},[1182],[57,9006,9008],{"className":9007},[1092],[57,9009,9011],{"className":9010,"style":2230},[1096],[57,9012],{},[57,9014,1768],{"className":9015},[1269]," 映射成连续表示 ",[57,9018,9020,9059],{"className":9019},[1019],[57,9021,9023],{"className":9022},[1023],[1025,9024,9025],{"xmlns":1027},[1029,9026,9027,9056],{},[1032,9028,9029,9032,9034,9036,9042,9044,9046,9048,9054],{},[1047,9030,9031],{"mathvariant":6079},"z",[1414,9033,281],{},[1414,9035,1754],{"stretchy":1753},[1044,9037,9038,9040],{},[1047,9039,9031],{},[1038,9041,574],{},[1414,9043,1759],{"separator":1062},[1414,9045,3693],{},[1414,9047,1759],{"separator":1062},[1044,9049,9050,9052],{},[1047,9051,9031],{},[1047,9053,1288],{},[1414,9055,1768],{"stretchy":1753},[1054,9057,9058],{"encoding":1056},"\\mathbf{z} = (z_1, \\ldots, z_n)",[57,9060,9062,9081],{"className":9061,"ariaHidden":1062},[1061],[57,9063,9065,9069,9072,9075,9078],{"className":9064},[1066],[57,9066],{"className":9067,"style":9068},[1070],"height:0.4444em;",[57,9070,9031],{"className":9071},[1075,6114],[57,9073],{"className":9074,"style":1453},[1452],[57,9076,281],{"className":9077},[1457],[57,9079],{"className":9080,"style":1453},[1452],[57,9082,9084,9087,9090,9132,9135,9138,9141,9144,9147,9150,9190],{"className":9083},[1066],[57,9085],{"className":9086,"style":1831},[1070],[57,9088,1754],{"className":9089},[1079],[57,9091,9093,9097],{"className":9092},[1075],[57,9094,9031],{"className":9095,"style":9096},[1075,1148],"margin-right:0.04398em;",[57,9098,9100],{"className":9099},[1152],[57,9101,9103,9124],{"className":9102},[1087,1088],[57,9104,9106,9121],{"className":9105},[1092],[57,9107,9109],{"className":9108,"style":3813},[1096],[57,9110,9112,9115],{"style":9111},"top:-2.55em;margin-left:-0.044em;margin-right:0.05em;",[57,9113],{"className":9114,"style":1493},[1104],[57,9116,9118],{"className":9117},[1109,1110,1111,1112],[57,9119,574],{"className":9120},[1075,1112],[57,9122,1183],{"className":9123},[1182],[57,9125,9127],{"className":9126},[1092],[57,9128,9130],{"className":9129,"style":2230},[1096],[57,9131],{},[57,9133,1759],{"className":9134},[1849],[57,9136],{"className":9137,"style":1853},[1452],[57,9139,3693],{"className":9140},[1898],[57,9142],{"className":9143,"style":1853},[1452],[57,9145,1759],{"className":9146},[1849],[57,9148],{"className":9149,"style":1853},[1452],[57,9151,9153,9156],{"className":9152},[1075],[57,9154,9031],{"className":9155,"style":9096},[1075,1148],[57,9157,9159],{"className":9158},[1152],[57,9160,9162,9182],{"className":9161},[1087,1088],[57,9163,9165,9179],{"className":9164},[1092],[57,9166,9168],{"className":9167,"style":8990},[1096],[57,9169,9170,9173],{"style":9111},[57,9171],{"className":9172,"style":1493},[1104],[57,9174,9176],{"className":9175},[1109,1110,1111,1112],[57,9177,1288],{"className":9178},[1075,1148,1112],[57,9180,1183],{"className":9181},[1182],[57,9183,9185],{"className":9184},[1092],[57,9186,9188],{"className":9187,"style":2230},[1096],[57,9189],{},[57,9191,1768],{"className":9192},[1269],"；解码器再以 ",[57,9195,9197,9211],{"className":9196},[1019],[57,9198,9200],{"className":9199},[1023],[1025,9201,9202],{"xmlns":1027},[1029,9203,9204,9208],{},[1032,9205,9206],{},[1047,9207,9031],{"mathvariant":6079},[1054,9209,9210],{"encoding":1056},"\\mathbf{z}",[57,9212,9214],{"className":9213,"ariaHidden":1062},[1061],[57,9215,9217,9220],{"className":9216},[1066],[57,9218],{"className":9219,"style":9068},[1070],[57,9221,9031],{"className":9222},[1075,6114]," 为条件，自回归地一步步生成输出序列 ",[57,9225,9227,9261],{"className":9226},[1019],[57,9228,9230],{"className":9229},[1023],[1025,9231,9232],{"xmlns":1027},[1029,9233,9234,9258],{},[1032,9235,9236,9238,9244,9246,9248,9250,9256],{},[1414,9237,1754],{"stretchy":1753},[1044,9239,9240,9242],{},[1047,9241,8495],{},[1038,9243,574],{},[1414,9245,1759],{"separator":1062},[1414,9247,3693],{},[1414,9249,1759],{"separator":1062},[1044,9251,9252,9254],{},[1047,9253,8495],{},[1047,9255,1319],{},[1414,9257,1768],{"stretchy":1753},[1054,9259,9260],{"encoding":1056},"(y_1, \\ldots, y_m)",[57,9262,9264],{"className":9263,"ariaHidden":1062},[1061],[57,9265,9267,9270,9273,9313,9316,9319,9322,9325,9328,9331,9371],{"className":9266},[1066],[57,9268],{"className":9269,"style":1831},[1070],[57,9271,1754],{"className":9272},[1079],[57,9274,9276,9279],{"className":9275},[1075],[57,9277,8495],{"className":9278,"style":1394},[1075,1148],[57,9280,9282],{"className":9281},[1152],[57,9283,9285,9305],{"className":9284},[1087,1088],[57,9286,9288,9302],{"className":9287},[1092],[57,9289,9291],{"className":9290,"style":3813},[1096],[57,9292,9293,9296],{"style":2758},[57,9294],{"className":9295,"style":1493},[1104],[57,9297,9299],{"className":9298},[1109,1110,1111,1112],[57,9300,574],{"className":9301},[1075,1112],[57,9303,1183],{"className":9304},[1182],[57,9306,9308],{"className":9307},[1092],[57,9309,9311],{"className":9310,"style":2230},[1096],[57,9312],{},[57,9314,1759],{"className":9315},[1849],[57,9317],{"className":9318,"style":1853},[1452],[57,9320,3693],{"className":9321},[1898],[57,9323],{"className":9324,"style":1853},[1452],[57,9326,1759],{"className":9327},[1849],[57,9329],{"className":9330,"style":1853},[1452],[57,9332,9334,9337],{"className":9333},[1075],[57,9335,8495],{"className":9336,"style":1394},[1075,1148],[57,9338,9340],{"className":9339},[1152],[57,9341,9343,9363],{"className":9342},[1087,1088],[57,9344,9346,9360],{"className":9345},[1092],[57,9347,9349],{"className":9348,"style":8990},[1096],[57,9350,9351,9354],{"style":2758},[57,9352],{"className":9353,"style":1493},[1104],[57,9355,9357],{"className":9356},[1109,1110,1111,1112],[57,9358,1319],{"className":9359},[1075,1148,1112],[57,9361,1183],{"className":9362},[1182],[57,9364,9366],{"className":9365},[1092],[57,9367,9369],{"className":9368,"style":2230},[1096],[57,9370],{},[57,9372,1768],{"className":9373},[1269],"——也就是说，每生成一个新词，都会把之前已经生成的所有词作为额外输入。",[47,9376,9378],{"className":256,"code":9377,"language":258,"meta":52,"style":52},"class Encoder(nn.Module):\n    \"\"\"编码器-解码器架构的基本编码器接口\"\"\"\n    def __init__(self, **kwargs):\n        super(Encoder, self).__init__(**kwargs)\n\n    def forward(self, X, *args):\n        raise NotImplementedError\n\nclass Decoder(nn.Module):\n    \"\"\"编码器-解码器架构的基本解码器接口\"\"\"\n    def __init__(self, **kwargs):\n        super(Decoder, self).__init__(**kwargs)\n\n    def init_state(self, enc_outputs, *args):\n        raise NotImplementedError\n\n    def forward(self, X, state):\n        raise NotImplementedError\n\nclass EncoderDecoder(nn.Module):\n    \"\"\"编码器-解码器架构的基类\"\"\"\n    def __init__(self, encoder, decoder, **kwargs):\n        super(EncoderDecoder, self).__init__(**kwargs)\n        self.encoder = encoder\n        self.decoder = decoder\n\n    def forward(self, enc_X, dec_X, *args):\n        enc_outputs = self.encoder(enc_X, *args)\n        dec_state = self.decoder.init_state(enc_outputs, *args)\n        return self.decoder(dec_X, dec_state)\n",[54,9379,9380,9397,9402,9420,9439,9443,9467,9475,9479,9496,9501,9519,9538,9542,9566,9572,9576,9597,9603,9607,9624,9629,9657,9676,9688,9700,9704,9732,9749,9765],{"__ignoreMap":52},[57,9381,9382,9384,9387,9389,9391,9393,9395],{"class":59,"line":60},[57,9383,3357],{"class":280},[57,9385,9386],{"class":3360}," Encoder",[57,9388,1754],{"class":269},[57,9390,3367],{"class":3366},[57,9392,1818],{"class":269},[57,9394,3372],{"class":3366},[57,9396,3169],{"class":269},[57,9398,9399],{"class":59,"line":66},[57,9400,9401],{"class":3174},"    \"\"\"编码器-解码器架构的基本编码器接口\"\"\"\n",[57,9403,9404,9406,9408,9410,9412,9414,9416,9418],{"class":59,"line":72},[57,9405,3384],{"class":280},[57,9407,3387],{"class":273},[57,9409,1754],{"class":269},[57,9411,266],{"class":3392},[57,9413,3163],{"class":269},[57,9415,3402],{"class":280},[57,9417,3405],{"class":3159},[57,9419,3169],{"class":269},[57,9421,9422,9424,9427,9429,9431,9433,9435,9437],{"class":59,"line":78},[57,9423,3413],{"class":3412},[57,9425,9426],{"class":269},"(Encoder, ",[57,9428,266],{"class":265},[57,9430,3421],{"class":269},[57,9432,869],{"class":273},[57,9434,1754],{"class":269},[57,9436,3402],{"class":280},[57,9438,3430],{"class":269},[57,9440,9441],{"class":59,"line":84},[57,9442,3448],{"emptyLinePlaceholder":963},[57,9444,9445,9447,9449,9451,9453,9455,9457,9459,9462,9465],{"class":59,"line":90},[57,9446,3384],{"class":280},[57,9448,3475],{"class":3153},[57,9450,1754],{"class":269},[57,9452,266],{"class":3392},[57,9454,3163],{"class":269},[57,9456,3160],{"class":3159},[57,9458,3163],{"class":269},[57,9460,9461],{"class":280},"*",[57,9463,9464],{"class":3159},"args",[57,9466,3169],{"class":269},[57,9468,9469,9472],{"class":59,"line":96},[57,9470,9471],{"class":280},"        raise",[57,9473,9474],{"class":3412}," NotImplementedError\n",[57,9476,9477],{"class":59,"line":102},[57,9478,3448],{"emptyLinePlaceholder":963},[57,9480,9481,9483,9486,9488,9490,9492,9494],{"class":59,"line":108},[57,9482,3357],{"class":280},[57,9484,9485],{"class":3360}," Decoder",[57,9487,1754],{"class":269},[57,9489,3367],{"class":3366},[57,9491,1818],{"class":269},[57,9493,3372],{"class":3366},[57,9495,3169],{"class":269},[57,9497,9498],{"class":59,"line":114},[57,9499,9500],{"class":3174},"    \"\"\"编码器-解码器架构的基本解码器接口\"\"\"\n",[57,9502,9503,9505,9507,9509,9511,9513,9515,9517],{"class":59,"line":120},[57,9504,3384],{"class":280},[57,9506,3387],{"class":273},[57,9508,1754],{"class":269},[57,9510,266],{"class":3392},[57,9512,3163],{"class":269},[57,9514,3402],{"class":280},[57,9516,3405],{"class":3159},[57,9518,3169],{"class":269},[57,9520,9521,9523,9526,9528,9530,9532,9534,9536],{"class":59,"line":126},[57,9522,3413],{"class":3412},[57,9524,9525],{"class":269},"(Decoder, ",[57,9527,266],{"class":265},[57,9529,3421],{"class":269},[57,9531,869],{"class":273},[57,9533,1754],{"class":269},[57,9535,3402],{"class":280},[57,9537,3430],{"class":269},[57,9539,9540],{"class":59,"line":132},[57,9541,3448],{"emptyLinePlaceholder":963},[57,9543,9544,9546,9549,9551,9553,9555,9558,9560,9562,9564],{"class":59,"line":138},[57,9545,3384],{"class":280},[57,9547,9548],{"class":3153}," init_state",[57,9550,1754],{"class":269},[57,9552,266],{"class":3392},[57,9554,3163],{"class":269},[57,9556,9557],{"class":3159},"enc_outputs",[57,9559,3163],{"class":269},[57,9561,9461],{"class":280},[57,9563,9464],{"class":3159},[57,9565,3169],{"class":269},[57,9567,9568,9570],{"class":59,"line":144},[57,9569,9471],{"class":280},[57,9571,9474],{"class":3412},[57,9573,9574],{"class":59,"line":150},[57,9575,3448],{"emptyLinePlaceholder":963},[57,9577,9578,9580,9582,9584,9586,9588,9590,9592,9595],{"class":59,"line":156},[57,9579,3384],{"class":280},[57,9581,3475],{"class":3153},[57,9583,1754],{"class":269},[57,9585,266],{"class":3392},[57,9587,3163],{"class":269},[57,9589,3160],{"class":3159},[57,9591,3163],{"class":269},[57,9593,9594],{"class":3159},"state",[57,9596,3169],{"class":269},[57,9598,9599,9601],{"class":59,"line":162},[57,9600,9471],{"class":280},[57,9602,9474],{"class":3412},[57,9604,9605],{"class":59,"line":168},[57,9606,3448],{"emptyLinePlaceholder":963},[57,9608,9609,9611,9614,9616,9618,9620,9622],{"class":59,"line":174},[57,9610,3357],{"class":280},[57,9612,9613],{"class":3360}," EncoderDecoder",[57,9615,1754],{"class":269},[57,9617,3367],{"class":3366},[57,9619,1818],{"class":269},[57,9621,3372],{"class":3366},[57,9623,3169],{"class":269},[57,9625,9626],{"class":59,"line":5790},[57,9627,9628],{"class":3174},"    \"\"\"编码器-解码器架构的基类\"\"\"\n",[57,9630,9631,9633,9635,9637,9639,9641,9644,9646,9649,9651,9653,9655],{"class":59,"line":5811},[57,9632,3384],{"class":280},[57,9634,3387],{"class":273},[57,9636,1754],{"class":269},[57,9638,266],{"class":3392},[57,9640,3163],{"class":269},[57,9642,9643],{"class":3159},"encoder",[57,9645,3163],{"class":269},[57,9647,9648],{"class":3159},"decoder",[57,9650,3163],{"class":269},[57,9652,3402],{"class":280},[57,9654,3405],{"class":3159},[57,9656,3169],{"class":269},[57,9658,9659,9661,9664,9666,9668,9670,9672,9674],{"class":59,"line":5830},[57,9660,3413],{"class":3412},[57,9662,9663],{"class":269},"(EncoderDecoder, ",[57,9665,266],{"class":265},[57,9667,3421],{"class":269},[57,9669,869],{"class":273},[57,9671,1754],{"class":269},[57,9673,3402],{"class":280},[57,9675,3430],{"class":269},[57,9677,9678,9680,9683,9685],{"class":59,"line":5849},[57,9679,3435],{"class":265},[57,9681,9682],{"class":269},".encoder ",[57,9684,281],{"class":280},[57,9686,9687],{"class":269}," encoder\n",[57,9689,9690,9692,9695,9697],{"class":59,"line":5854},[57,9691,3435],{"class":265},[57,9693,9694],{"class":269},".decoder ",[57,9696,281],{"class":280},[57,9698,9699],{"class":269}," decoder\n",[57,9701,9702],{"class":59,"line":5870},[57,9703,3448],{"emptyLinePlaceholder":963},[57,9705,9706,9708,9710,9712,9714,9716,9719,9721,9724,9726,9728,9730],{"class":59,"line":5876},[57,9707,3384],{"class":280},[57,9709,3475],{"class":3153},[57,9711,1754],{"class":269},[57,9713,266],{"class":3392},[57,9715,3163],{"class":269},[57,9717,9718],{"class":3159},"enc_X",[57,9720,3163],{"class":269},[57,9722,9723],{"class":3159},"dec_X",[57,9725,3163],{"class":269},[57,9727,9461],{"class":280},[57,9729,9464],{"class":3159},[57,9731,3169],{"class":269},[57,9733,9734,9737,9739,9741,9744,9746],{"class":59,"line":5882},[57,9735,9736],{"class":269},"        enc_outputs ",[57,9738,281],{"class":280},[57,9740,5935],{"class":265},[57,9742,9743],{"class":269},".encoder(enc_X, ",[57,9745,9461],{"class":280},[57,9747,9748],{"class":269},"args)\n",[57,9750,9751,9754,9756,9758,9761,9763],{"class":59,"line":5892},[57,9752,9753],{"class":269},"        dec_state ",[57,9755,281],{"class":280},[57,9757,5935],{"class":265},[57,9759,9760],{"class":269},".decoder.init_state(enc_outputs, ",[57,9762,9461],{"class":280},[57,9764,9748],{"class":269},[57,9766,9767,9769,9771],{"class":59,"line":5916},[57,9768,3202],{"class":280},[57,9770,5935],{"class":265},[57,9772,9773],{"class":269},".decoder(dec_X, dec_state)\n",[10,9775,9777],{"id":9776},"transformer-编码器","transformer 编码器",[14,9779,9780],{},"代码实现如下：",[47,9782,9784],{"className":256,"code":9783,"language":258,"meta":52,"style":52},"class EncoderBlock(nn.Module):\n    \"\"\"Transformer编码器块\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n                 dropout, use_bias=False, **kwargs):\n        super(EncoderBlock, self).__init__(**kwargs)\n        self.attention = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout,\n            use_bias)\n        self.addnorm1 = AddNorm(norm_shape, dropout)\n        self.ffn = PositionWiseFFN(\n            ffn_num_input, ffn_num_hiddens, num_hiddens)\n        self.addnorm2 = AddNorm(norm_shape, dropout)\n\n    def forward(self, X, valid_lens):\n        Y = self.addnorm1(X, self.attention(X, X, X, valid_lens))\n        return self.addnorm2(Y, self.ffn(Y))\n\nclass TransformerEncoder(Encoder):\n    \"\"\"Transformer编码器\"\"\"\n    def __init__(self, vocab_size, key_size, query_size, value_size,\n                 num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,\n                 num_heads, num_layers, dropout, use_bias=False, **kwargs):\n        super(TransformerEncoder, self).__init__(**kwargs)\n        self.num_hiddens = num_hiddens\n        self.embedding = nn.Embedding(vocab_size, num_hiddens)\n        self.pos_encoding = PositionalEncoding(num_hiddens, dropout)\n        self.blks = nn.Sequential()\n        for i in range(num_layers):\n            self.blks.add_module(\"block\"+str(i),\n                EncoderBlock(key_size, query_size, value_size, num_hiddens,\n                             norm_shape, ffn_num_input, ffn_num_hiddens,\n                             num_heads, dropout, use_bias))\n\n    def forward(self, X, valid_lens, *args):\n        # 因为位置编码值在-1和1之间，\n        # 因此嵌入值乘以嵌入维度的平方根进行缩放，\n        # 然后再与位置编码相加。\n        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))\n        self.attention_weights = [None] * len(self.blks)\n        for i, blk in enumerate(self.blks):\n            X = blk(X, valid_lens)\n            self.attention_weights[i] = blk.attention.attention.attention_weights\n        return X\n",[54,9785,9786,9803,9808,9836,9855,9877,9896,9907,9912,9917,9929,9941,9946,9957,9961,9981,9998,10012,10016,10030,10035,10064,10084,10113,10132,10144,10156,10168,10180,10198,10221,10226,10231,10236,10240,10266,10271,10277,10283,10310,10338,10358,10369,10382],{"__ignoreMap":52},[57,9787,9788,9790,9793,9795,9797,9799,9801],{"class":59,"line":60},[57,9789,3357],{"class":280},[57,9791,9792],{"class":3360}," EncoderBlock",[57,9794,1754],{"class":269},[57,9796,3367],{"class":3366},[57,9798,1818],{"class":269},[57,9800,3372],{"class":3366},[57,9802,3169],{"class":269},[57,9804,9805],{"class":59,"line":66},[57,9806,9807],{"class":3174},"    \"\"\"Transformer编码器块\"\"\"\n",[57,9809,9810,9812,9814,9816,9818,9820,9822,9824,9826,9828,9830,9832,9834],{"class":59,"line":72},[57,9811,3384],{"class":280},[57,9813,3387],{"class":273},[57,9815,1754],{"class":269},[57,9817,266],{"class":3392},[57,9819,3163],{"class":269},[57,9821,5566],{"class":3159},[57,9823,3163],{"class":269},[57,9825,5571],{"class":3159},[57,9827,3163],{"class":269},[57,9829,5576],{"class":3159},[57,9831,3163],{"class":269},[57,9833,5581],{"class":3159},[57,9835,369],{"class":269},[57,9837,9838,9841,9843,9845,9847,9849,9851,9853],{"class":59,"line":78},[57,9839,9840],{"class":3159},"                 norm_shape",[57,9842,3163],{"class":269},[57,9844,8349],{"class":3159},[57,9846,3163],{"class":269},[57,9848,8354],{"class":3159},[57,9850,3163],{"class":269},[57,9852,5316],{"class":3159},[57,9854,369],{"class":269},[57,9856,9857,9860,9862,9865,9867,9869,9871,9873,9875],{"class":59,"line":84},[57,9858,9859],{"class":3159},"                 dropout",[57,9861,3163],{"class":269},[57,9863,9864],{"class":3159},"use_bias",[57,9866,281],{"class":280},[57,9868,800],{"class":273},[57,9870,3163],{"class":269},[57,9872,3402],{"class":280},[57,9874,3405],{"class":3159},[57,9876,3169],{"class":269},[57,9878,9879,9881,9884,9886,9888,9890,9892,9894],{"class":59,"line":90},[57,9880,3413],{"class":3412},[57,9882,9883],{"class":269},"(EncoderBlock, ",[57,9885,266],{"class":265},[57,9887,3421],{"class":269},[57,9889,869],{"class":273},[57,9891,1754],{"class":269},[57,9893,3402],{"class":280},[57,9895,3430],{"class":269},[57,9897,9898,9900,9902,9904],{"class":59,"line":96},[57,9899,3435],{"class":265},[57,9901,5647],{"class":269},[57,9903,281],{"class":280},[57,9905,9906],{"class":269}," d2l.MultiHeadAttention(\n",[57,9908,9909],{"class":59,"line":102},[57,9910,9911],{"class":269},"            key_size, query_size, value_size, num_hiddens, num_heads, dropout,\n",[57,9913,9914],{"class":59,"line":108},[57,9915,9916],{"class":269},"            use_bias)\n",[57,9918,9919,9921,9924,9926],{"class":59,"line":114},[57,9920,3435],{"class":265},[57,9922,9923],{"class":269},".addnorm1 ",[57,9925,281],{"class":280},[57,9927,9928],{"class":269}," AddNorm(norm_shape, dropout)\n",[57,9930,9931,9933,9936,9938],{"class":59,"line":120},[57,9932,3435],{"class":265},[57,9934,9935],{"class":269},".ffn ",[57,9937,281],{"class":280},[57,9939,9940],{"class":269}," PositionWiseFFN(\n",[57,9942,9943],{"class":59,"line":126},[57,9944,9945],{"class":269},"            ffn_num_input, ffn_num_hiddens, num_hiddens)\n",[57,9947,9948,9950,9953,9955],{"class":59,"line":132},[57,9949,3435],{"class":265},[57,9951,9952],{"class":269},".addnorm2 ",[57,9954,281],{"class":280},[57,9956,9928],{"class":269},[57,9958,9959],{"class":59,"line":138},[57,9960,3448],{"emptyLinePlaceholder":963},[57,9962,9963,9965,9967,9969,9971,9973,9975,9977,9979],{"class":59,"line":144},[57,9964,3384],{"class":280},[57,9966,3475],{"class":3153},[57,9968,1754],{"class":269},[57,9970,266],{"class":3392},[57,9972,3163],{"class":269},[57,9974,3160],{"class":3159},[57,9976,3163],{"class":269},[57,9978,3166],{"class":3159},[57,9980,3169],{"class":269},[57,9982,9983,9986,9988,9990,9993,9995],{"class":59,"line":150},[57,9984,9985],{"class":269},"        Y ",[57,9987,281],{"class":280},[57,9989,5935],{"class":265},[57,9991,9992],{"class":269},".addnorm1(X, ",[57,9994,266],{"class":265},[57,9996,9997],{"class":269},".attention(X, X, X, valid_lens))\n",[57,9999,10000,10002,10004,10007,10009],{"class":59,"line":156},[57,10001,3202],{"class":280},[57,10003,5935],{"class":265},[57,10005,10006],{"class":269},".addnorm2(Y, ",[57,10008,266],{"class":265},[57,10010,10011],{"class":269},".ffn(Y))\n",[57,10013,10014],{"class":59,"line":162},[57,10015,3448],{"emptyLinePlaceholder":963},[57,10017,10018,10020,10023,10025,10028],{"class":59,"line":168},[57,10019,3357],{"class":280},[57,10021,10022],{"class":3360}," TransformerEncoder",[57,10024,1754],{"class":269},[57,10026,10027],{"class":3366},"Encoder",[57,10029,3169],{"class":269},[57,10031,10032],{"class":59,"line":174},[57,10033,10034],{"class":3174},"    \"\"\"Transformer编码器\"\"\"\n",[57,10036,10037,10039,10041,10043,10045,10047,10050,10052,10054,10056,10058,10060,10062],{"class":59,"line":5790},[57,10038,3384],{"class":280},[57,10040,3387],{"class":273},[57,10042,1754],{"class":269},[57,10044,266],{"class":3392},[57,10046,3163],{"class":269},[57,10048,10049],{"class":3159},"vocab_size",[57,10051,3163],{"class":269},[57,10053,5566],{"class":3159},[57,10055,3163],{"class":269},[57,10057,5571],{"class":3159},[57,10059,3163],{"class":269},[57,10061,5576],{"class":3159},[57,10063,369],{"class":269},[57,10065,10066,10069,10071,10074,10076,10078,10080,10082],{"class":59,"line":5811},[57,10067,10068],{"class":3159},"                 num_hiddens",[57,10070,3163],{"class":269},[57,10072,10073],{"class":3159},"norm_shape",[57,10075,3163],{"class":269},[57,10077,8349],{"class":3159},[57,10079,3163],{"class":269},[57,10081,8354],{"class":3159},[57,10083,369],{"class":269},[57,10085,10086,10088,10090,10093,10095,10097,10099,10101,10103,10105,10107,10109,10111],{"class":59,"line":5830},[57,10087,5588],{"class":3159},[57,10089,3163],{"class":269},[57,10091,10092],{"class":3159},"num_layers",[57,10094,3163],{"class":269},[57,10096,3397],{"class":3159},[57,10098,3163],{"class":269},[57,10100,9864],{"class":3159},[57,10102,281],{"class":280},[57,10104,800],{"class":273},[57,10106,3163],{"class":269},[57,10108,3402],{"class":280},[57,10110,3405],{"class":3159},[57,10112,3169],{"class":269},[57,10114,10115,10117,10120,10122,10124,10126,10128,10130],{"class":59,"line":5849},[57,10116,3413],{"class":3412},[57,10118,10119],{"class":269},"(TransformerEncoder, ",[57,10121,266],{"class":265},[57,10123,3421],{"class":269},[57,10125,869],{"class":273},[57,10127,1754],{"class":269},[57,10129,3402],{"class":280},[57,10131,3430],{"class":269},[57,10133,10134,10136,10139,10141],{"class":59,"line":5854},[57,10135,3435],{"class":265},[57,10137,10138],{"class":269},".num_hiddens ",[57,10140,281],{"class":280},[57,10142,10143],{"class":269}," num_hiddens\n",[57,10145,10146,10148,10151,10153],{"class":59,"line":5870},[57,10147,3435],{"class":265},[57,10149,10150],{"class":269},".embedding ",[57,10152,281],{"class":280},[57,10154,10155],{"class":269}," nn.Embedding(vocab_size, num_hiddens)\n",[57,10157,10158,10160,10163,10165],{"class":59,"line":5876},[57,10159,3435],{"class":265},[57,10161,10162],{"class":269},".pos_encoding ",[57,10164,281],{"class":280},[57,10166,10167],{"class":269}," PositionalEncoding(num_hiddens, dropout)\n",[57,10169,10170,10172,10175,10177],{"class":59,"line":5882},[57,10171,3435],{"class":265},[57,10173,10174],{"class":269},".blks ",[57,10176,281],{"class":280},[57,10178,10179],{"class":269}," nn.Sequential()\n",[57,10181,10182,10185,10188,10191,10195],{"class":59,"line":5892},[57,10183,10184],{"class":280},"        for",[57,10186,10187],{"class":269}," i ",[57,10189,10190],{"class":280},"in",[57,10192,10194],{"class":10193},"sDgm9"," range",[57,10196,10197],{"class":269},"(num_layers):\n",[57,10199,10200,10203,10206,10208,10211,10213,10215,10218],{"class":59,"line":5916},[57,10201,10202],{"class":265},"            self",[57,10204,10205],{"class":269},".blks.add_module(",[57,10207,297],{"class":289},[57,10209,10210],{"class":293},"block",[57,10212,297],{"class":289},[57,10214,6487],{"class":280},[57,10216,10217],{"class":3412},"str",[57,10219,10220],{"class":269},"(i),\n",[57,10222,10223],{"class":59,"line":5921},[57,10224,10225],{"class":269},"                EncoderBlock(key_size, query_size, value_size, num_hiddens,\n",[57,10227,10228],{"class":59,"line":5927},[57,10229,10230],{"class":269},"                             norm_shape, ffn_num_input, ffn_num_hiddens,\n",[57,10232,10233],{"class":59,"line":5941},[57,10234,10235],{"class":269},"                             num_heads, dropout, use_bias))\n",[57,10237,10238],{"class":59,"line":5946},[57,10239,3448],{"emptyLinePlaceholder":963},[57,10241,10242,10244,10246,10248,10250,10252,10254,10256,10258,10260,10262,10264],{"class":59,"line":5952},[57,10243,3384],{"class":280},[57,10245,3475],{"class":3153},[57,10247,1754],{"class":269},[57,10249,266],{"class":3392},[57,10251,3163],{"class":269},[57,10253,3160],{"class":3159},[57,10255,3163],{"class":269},[57,10257,3166],{"class":3159},[57,10259,3163],{"class":269},[57,10261,9461],{"class":280},[57,10263,9464],{"class":3159},[57,10265,3169],{"class":269},[57,10267,10268],{"class":59,"line":5967},[57,10269,10270],{"class":303},"        # 因为位置编码值在-1和1之间，\n",[57,10272,10274],{"class":59,"line":10273},37,[57,10275,10276],{"class":303},"        # 因此嵌入值乘以嵌入维度的平方根进行缩放，\n",[57,10278,10280],{"class":59,"line":10279},38,[57,10281,10282],{"class":303},"        # 然后再与位置编码相加。\n",[57,10284,10286,10288,10290,10292,10295,10297,10300,10302,10305,10307],{"class":59,"line":10285},39,[57,10287,3298],{"class":269},[57,10289,281],{"class":280},[57,10291,5935],{"class":265},[57,10293,10294],{"class":269},".pos_encoding(",[57,10296,266],{"class":265},[57,10298,10299],{"class":269},".embedding(X) ",[57,10301,9461],{"class":280},[57,10303,10304],{"class":269}," math.sqrt(",[57,10306,266],{"class":265},[57,10308,10309],{"class":269},".num_hiddens))\n",[57,10311,10313,10315,10317,10319,10322,10324,10326,10328,10331,10333,10335],{"class":59,"line":10312},40,[57,10314,3435],{"class":265},[57,10316,3559],{"class":269},[57,10318,281],{"class":280},[57,10320,10321],{"class":269}," [",[57,10323,3503],{"class":273},[57,10325,277],{"class":269},[57,10327,9461],{"class":280},[57,10329,10330],{"class":10193}," len",[57,10332,1754],{"class":269},[57,10334,266],{"class":265},[57,10336,10337],{"class":269},".blks)\n",[57,10339,10341,10343,10346,10348,10351,10353,10355],{"class":59,"line":10340},41,[57,10342,10184],{"class":280},[57,10344,10345],{"class":269}," i, blk ",[57,10347,10190],{"class":280},[57,10349,10350],{"class":10193}," enumerate",[57,10352,1754],{"class":269},[57,10354,266],{"class":265},[57,10356,10357],{"class":269},".blks):\n",[57,10359,10361,10364,10366],{"class":59,"line":10360},42,[57,10362,10363],{"class":269},"            X ",[57,10365,281],{"class":280},[57,10367,10368],{"class":269}," blk(X, valid_lens)\n",[57,10370,10372,10374,10377,10379],{"class":59,"line":10371},43,[57,10373,10202],{"class":265},[57,10375,10376],{"class":269},".attention_weights[i] ",[57,10378,281],{"class":280},[57,10380,10381],{"class":269}," blk.attention.attention.attention_weights\n",[57,10383,10385,10387],{"class":59,"line":10384},44,[57,10386,3202],{"class":280},[57,10388,10389],{"class":269}," X\n",[10,10391,10393],{"id":10392},"transformer-解码器","transformer 解码器",[14,10395,10396],{},"代码如下：",[47,10398,10400],{"className":256,"code":10399,"language":258,"meta":52,"style":52},"class DecoderBlock(nn.Module):\n    \"\"\"解码器中第i个块\"\"\"\n    def __init__(self, key_size, query_size, value_size, num_hiddens,\n                 norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n                 dropout, i, **kwargs):\n        super(DecoderBlock, self).__init__(**kwargs)\n        self.i = i\n        self.attention1 = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n        self.addnorm1 = AddNorm(norm_shape, dropout)\n        self.attention2 = d2l.MultiHeadAttention(\n            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n        self.addnorm2 = AddNorm(norm_shape, dropout)\n        self.ffn = PositionWiseFFN(ffn_num_input, ffn_num_hiddens,\n                                   num_hiddens)\n        self.addnorm3 = AddNorm(norm_shape, dropout)\n\n    def forward(self, X, state):\n        enc_outputs, enc_valid_lens = state[0], state[1]\n        # 训练阶段，输出序列的所有词元都在同一时间处理，\n        # 因此state[2][self.i]初始化为None。\n        # 预测阶段，输出序列是通过词元一个接着一个解码的，\n        # 因此state[2][self.i]包含着直到当前时间步第i个块解码的输出表示\n        if state[2][self.i] is None:\n            key_values = X\n        else:\n            key_values = torch.cat((state[2][self.i], X), axis=1)\n        state[2][self.i] = key_values\n        if self.training:\n            batch_size, num_steps, _ = X.shape\n            # dec_valid_lens的开头:(batch_size,num_steps),\n            # 其中每一行是[1,2,...,num_steps]\n            dec_valid_lens = torch.arange(\n                1, num_steps + 1, device=X.device).repeat(batch_size, 1)\n        else:\n            dec_valid_lens = None\n\n        # 自注意力\n        X2 = self.attention1(X, key_values, key_values, dec_valid_lens)\n        Y = self.addnorm1(X, X2)\n        # 编码器－解码器注意力。\n        # enc_outputs的开头:(batch_size,num_steps,num_hiddens)\n        Y2 = self.attention2(Y, enc_outputs, enc_outputs, enc_valid_lens)\n        Z = self.addnorm2(Y, Y2)\n        return self.addnorm3(Z, self.ffn(Z)), state\n\nclass TransformerDecoder(AttentionDecoder):\n    def __init__(self, vocab_size, key_size, query_size, value_size,\n                 num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,\n                 num_heads, num_layers, dropout, **kwargs):\n        super(TransformerDecoder, self).__init__(**kwargs)\n        self.num_hiddens = num_hiddens\n        self.num_layers = num_layers\n        self.embedding = nn.Embedding(vocab_size, num_hiddens)\n        self.pos_encoding = PositionalEncoding(num_hiddens, dropout)\n        self.blks = nn.Sequential()\n        for i in range(num_layers):\n            self.blks.add_module(\"block\"+str(i),\n                DecoderBlock(key_size, query_size, value_size, num_hiddens,\n                             norm_shape, ffn_num_input, ffn_num_hiddens,\n                             num_heads, dropout, i))\n        self.dense = nn.Linear(num_hiddens, vocab_size)\n\n    def init_state(self, enc_outputs, enc_valid_lens, *args):\n        return [enc_outputs, enc_valid_lens, [None] * self.num_layers]\n\n    def forward(self, X, state):\n        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))\n        self._attention_weights = [[None] * len(self.blks) for _ in range (2)]\n        for i, blk in enumerate(self.blks):\n            X, state = blk(X, state)\n            # 解码器自注意力权重\n            self._attention_weights[0][i] = blk.attention1.attention.attention_weights\n            # “编码器－解码器”自注意力权重\n            self._attention_weights[1][i] = blk.attention2.attention.attention_weights\n        return self.dense(X), state\n\n    @property\n    def attention_weights(self):\n        return self._attention_weights\n",[54,10401,10402,10419,10424,10452,10470,10486,10505,10517,10528,10533,10543,10554,10558,10568,10579,10584,10595,10599,10619,10638,10643,10648,10653,10658,10680,10689,10695,10722,10740,10749,10758,10763,10768,10778,10804,10810,10819,10823,10828,10840,10851,10856,10861,10873,10885,10900,10905,10920,10949,10968,10989,11009,11020,11033,11044,11055,11066,11079,11098,11104,11109,11115,11128,11133,11161,11180,11185,11206,11229,11275,11292,11303,11309,11327,11333,11349,11359,11364,11373,11387],{"__ignoreMap":52},[57,10403,10404,10406,10409,10411,10413,10415,10417],{"class":59,"line":60},[57,10405,3357],{"class":280},[57,10407,10408],{"class":3360}," DecoderBlock",[57,10410,1754],{"class":269},[57,10412,3367],{"class":3366},[57,10414,1818],{"class":269},[57,10416,3372],{"class":3366},[57,10418,3169],{"class":269},[57,10420,10421],{"class":59,"line":66},[57,10422,10423],{"class":3174},"    \"\"\"解码器中第i个块\"\"\"\n",[57,10425,10426,10428,10430,10432,10434,10436,10438,10440,10442,10444,10446,10448,10450],{"class":59,"line":72},[57,10427,3384],{"class":280},[57,10429,3387],{"class":273},[57,10431,1754],{"class":269},[57,10433,266],{"class":3392},[57,10435,3163],{"class":269},[57,10437,5566],{"class":3159},[57,10439,3163],{"class":269},[57,10441,5571],{"class":3159},[57,10443,3163],{"class":269},[57,10445,5576],{"class":3159},[57,10447,3163],{"class":269},[57,10449,5581],{"class":3159},[57,10451,369],{"class":269},[57,10453,10454,10456,10458,10460,10462,10464,10466,10468],{"class":59,"line":78},[57,10455,9840],{"class":3159},[57,10457,3163],{"class":269},[57,10459,8349],{"class":3159},[57,10461,3163],{"class":269},[57,10463,8354],{"class":3159},[57,10465,3163],{"class":269},[57,10467,5316],{"class":3159},[57,10469,369],{"class":269},[57,10471,10472,10474,10476,10478,10480,10482,10484],{"class":59,"line":84},[57,10473,9859],{"class":3159},[57,10475,3163],{"class":269},[57,10477,1745],{"class":3159},[57,10479,3163],{"class":269},[57,10481,3402],{"class":280},[57,10483,3405],{"class":3159},[57,10485,3169],{"class":269},[57,10487,10488,10490,10493,10495,10497,10499,10501,10503],{"class":59,"line":90},[57,10489,3413],{"class":3412},[57,10491,10492],{"class":269},"(DecoderBlock, ",[57,10494,266],{"class":265},[57,10496,3421],{"class":269},[57,10498,869],{"class":273},[57,10500,1754],{"class":269},[57,10502,3402],{"class":280},[57,10504,3430],{"class":269},[57,10506,10507,10509,10512,10514],{"class":59,"line":96},[57,10508,3435],{"class":265},[57,10510,10511],{"class":269},".i ",[57,10513,281],{"class":280},[57,10515,10516],{"class":269}," i\n",[57,10518,10519,10521,10524,10526],{"class":59,"line":102},[57,10520,3435],{"class":265},[57,10522,10523],{"class":269},".attention1 ",[57,10525,281],{"class":280},[57,10527,9906],{"class":269},[57,10529,10530],{"class":59,"line":108},[57,10531,10532],{"class":269},"            key_size, query_size, value_size, num_hiddens, num_heads, dropout)\n",[57,10534,10535,10537,10539,10541],{"class":59,"line":114},[57,10536,3435],{"class":265},[57,10538,9923],{"class":269},[57,10540,281],{"class":280},[57,10542,9928],{"class":269},[57,10544,10545,10547,10550,10552],{"class":59,"line":120},[57,10546,3435],{"class":265},[57,10548,10549],{"class":269},".attention2 ",[57,10551,281],{"class":280},[57,10553,9906],{"class":269},[57,10555,10556],{"class":59,"line":126},[57,10557,10532],{"class":269},[57,10559,10560,10562,10564,10566],{"class":59,"line":132},[57,10561,3435],{"class":265},[57,10563,9952],{"class":269},[57,10565,281],{"class":280},[57,10567,9928],{"class":269},[57,10569,10570,10572,10574,10576],{"class":59,"line":138},[57,10571,3435],{"class":265},[57,10573,9935],{"class":269},[57,10575,281],{"class":280},[57,10577,10578],{"class":269}," PositionWiseFFN(ffn_num_input, ffn_num_hiddens,\n",[57,10580,10581],{"class":59,"line":144},[57,10582,10583],{"class":269},"                                   num_hiddens)\n",[57,10585,10586,10588,10591,10593],{"class":59,"line":150},[57,10587,3435],{"class":265},[57,10589,10590],{"class":269},".addnorm3 ",[57,10592,281],{"class":280},[57,10594,9928],{"class":269},[57,10596,10597],{"class":59,"line":156},[57,10598,3448],{"emptyLinePlaceholder":963},[57,10600,10601,10603,10605,10607,10609,10611,10613,10615,10617],{"class":59,"line":162},[57,10602,3384],{"class":280},[57,10604,3475],{"class":3153},[57,10606,1754],{"class":269},[57,10608,266],{"class":3392},[57,10610,3163],{"class":269},[57,10612,3160],{"class":3159},[57,10614,3163],{"class":269},[57,10616,9594],{"class":3159},[57,10618,3169],{"class":269},[57,10620,10621,10624,10626,10629,10631,10634,10636],{"class":59,"line":168},[57,10622,10623],{"class":269},"        enc_outputs, enc_valid_lens ",[57,10625,281],{"class":280},[57,10627,10628],{"class":269}," state[",[57,10630,560],{"class":273},[57,10632,10633],{"class":269},"], state[",[57,10635,574],{"class":273},[57,10637,3522],{"class":269},[57,10639,10640],{"class":59,"line":174},[57,10641,10642],{"class":303},"        # 训练阶段，输出序列的所有词元都在同一时间处理，\n",[57,10644,10645],{"class":59,"line":5790},[57,10646,10647],{"class":303},"        # 因此state[2][self.i]初始化为None。\n",[57,10649,10650],{"class":59,"line":5811},[57,10651,10652],{"class":303},"        # 预测阶段，输出序列是通过词元一个接着一个解码的，\n",[57,10654,10655],{"class":59,"line":5830},[57,10656,10657],{"class":303},"        # 因此state[2][self.i]包含着直到当前时间步第i个块解码的输出表示\n",[57,10659,10660,10662,10664,10666,10669,10671,10674,10676,10678],{"class":59,"line":5849},[57,10661,3239],{"class":280},[57,10663,10628],{"class":269},[57,10665,590],{"class":273},[57,10667,10668],{"class":269},"][",[57,10670,266],{"class":265},[57,10672,10673],{"class":269},".i] ",[57,10675,3191],{"class":280},[57,10677,3194],{"class":273},[57,10679,3197],{"class":269},[57,10681,10682,10685,10687],{"class":59,"line":5854},[57,10683,10684],{"class":269},"            key_values ",[57,10686,281],{"class":280},[57,10688,10389],{"class":269},[57,10690,10691,10693],{"class":59,"line":5870},[57,10692,3270],{"class":280},[57,10694,3197],{"class":269},[57,10696,10697,10699,10701,10704,10706,10708,10710,10713,10716,10718,10720],{"class":59,"line":5876},[57,10698,10684],{"class":269},[57,10700,281],{"class":280},[57,10702,10703],{"class":269}," torch.cat((state[",[57,10705,590],{"class":273},[57,10707,10668],{"class":269},[57,10709,266],{"class":265},[57,10711,10712],{"class":269},".i], X), ",[57,10714,10715],{"class":3208},"axis",[57,10717,281],{"class":280},[57,10719,574],{"class":273},[57,10721,3217],{"class":269},[57,10723,10724,10727,10729,10731,10733,10735,10737],{"class":59,"line":5882},[57,10725,10726],{"class":269},"        state[",[57,10728,590],{"class":273},[57,10730,10668],{"class":269},[57,10732,266],{"class":265},[57,10734,10673],{"class":269},[57,10736,281],{"class":280},[57,10738,10739],{"class":269}," key_values\n",[57,10741,10742,10744,10746],{"class":59,"line":5892},[57,10743,3239],{"class":280},[57,10745,5935],{"class":265},[57,10747,10748],{"class":269},".training:\n",[57,10750,10751,10754,10756],{"class":59,"line":5916},[57,10752,10753],{"class":269},"            batch_size, num_steps, _ ",[57,10755,281],{"class":280},[57,10757,3234],{"class":269},[57,10759,10760],{"class":59,"line":5921},[57,10761,10762],{"class":303},"            # dec_valid_lens的开头:(batch_size,num_steps),\n",[57,10764,10765],{"class":59,"line":5927},[57,10766,10767],{"class":303},"            # 其中每一行是[1,2,...,num_steps]\n",[57,10769,10770,10773,10775],{"class":59,"line":5941},[57,10771,10772],{"class":269},"            dec_valid_lens ",[57,10774,281],{"class":280},[57,10776,10777],{"class":269}," torch.arange(\n",[57,10779,10780,10783,10786,10788,10790,10792,10795,10797,10800,10802],{"class":59,"line":5946},[57,10781,10782],{"class":273},"                1",[57,10784,10785],{"class":269},", num_steps ",[57,10787,6487],{"class":280},[57,10789,3248],{"class":273},[57,10791,3163],{"class":269},[57,10793,10794],{"class":3208},"device",[57,10796,281],{"class":280},[57,10798,10799],{"class":269},"X.device).repeat(batch_size, ",[57,10801,574],{"class":273},[57,10803,3217],{"class":269},[57,10805,10806,10808],{"class":59,"line":5952},[57,10807,3270],{"class":280},[57,10809,3197],{"class":269},[57,10811,10812,10814,10816],{"class":59,"line":5967},[57,10813,10772],{"class":269},[57,10815,281],{"class":280},[57,10817,10818],{"class":273}," None\n",[57,10820,10821],{"class":59,"line":10273},[57,10822,3448],{"emptyLinePlaceholder":963},[57,10824,10825],{"class":59,"line":10279},[57,10826,10827],{"class":303},"        # 自注意力\n",[57,10829,10830,10833,10835,10837],{"class":59,"line":10285},[57,10831,10832],{"class":269},"        X2 ",[57,10834,281],{"class":280},[57,10836,5935],{"class":265},[57,10838,10839],{"class":269},".attention1(X, key_values, key_values, dec_valid_lens)\n",[57,10841,10842,10844,10846,10848],{"class":59,"line":10312},[57,10843,9985],{"class":269},[57,10845,281],{"class":280},[57,10847,5935],{"class":265},[57,10849,10850],{"class":269},".addnorm1(X, X2)\n",[57,10852,10853],{"class":59,"line":10340},[57,10854,10855],{"class":303},"        # 编码器－解码器注意力。\n",[57,10857,10858],{"class":59,"line":10360},[57,10859,10860],{"class":303},"        # enc_outputs的开头:(batch_size,num_steps,num_hiddens)\n",[57,10862,10863,10866,10868,10870],{"class":59,"line":10371},[57,10864,10865],{"class":269},"        Y2 ",[57,10867,281],{"class":280},[57,10869,5935],{"class":265},[57,10871,10872],{"class":269},".attention2(Y, enc_outputs, enc_outputs, enc_valid_lens)\n",[57,10874,10875,10878,10880,10882],{"class":59,"line":10384},[57,10876,10877],{"class":269},"        Z ",[57,10879,281],{"class":280},[57,10881,5935],{"class":265},[57,10883,10884],{"class":269},".addnorm2(Y, Y2)\n",[57,10886,10888,10890,10892,10895,10897],{"class":59,"line":10887},45,[57,10889,3202],{"class":280},[57,10891,5935],{"class":265},[57,10893,10894],{"class":269},".addnorm3(Z, ",[57,10896,266],{"class":265},[57,10898,10899],{"class":269},".ffn(Z)), state\n",[57,10901,10903],{"class":59,"line":10902},46,[57,10904,3448],{"emptyLinePlaceholder":963},[57,10906,10908,10910,10913,10915,10918],{"class":59,"line":10907},47,[57,10909,3357],{"class":280},[57,10911,10912],{"class":3360}," TransformerDecoder",[57,10914,1754],{"class":269},[57,10916,10917],{"class":3366},"AttentionDecoder",[57,10919,3169],{"class":269},[57,10921,10923,10925,10927,10929,10931,10933,10935,10937,10939,10941,10943,10945,10947],{"class":59,"line":10922},48,[57,10924,3384],{"class":280},[57,10926,3387],{"class":273},[57,10928,1754],{"class":269},[57,10930,266],{"class":3392},[57,10932,3163],{"class":269},[57,10934,10049],{"class":3159},[57,10936,3163],{"class":269},[57,10938,5566],{"class":3159},[57,10940,3163],{"class":269},[57,10942,5571],{"class":3159},[57,10944,3163],{"class":269},[57,10946,5576],{"class":3159},[57,10948,369],{"class":269},[57,10950,10952,10954,10956,10958,10960,10962,10964,10966],{"class":59,"line":10951},49,[57,10953,10068],{"class":3159},[57,10955,3163],{"class":269},[57,10957,10073],{"class":3159},[57,10959,3163],{"class":269},[57,10961,8349],{"class":3159},[57,10963,3163],{"class":269},[57,10965,8354],{"class":3159},[57,10967,369],{"class":269},[57,10969,10971,10973,10975,10977,10979,10981,10983,10985,10987],{"class":59,"line":10970},50,[57,10972,5588],{"class":3159},[57,10974,3163],{"class":269},[57,10976,10092],{"class":3159},[57,10978,3163],{"class":269},[57,10980,3397],{"class":3159},[57,10982,3163],{"class":269},[57,10984,3402],{"class":280},[57,10986,3405],{"class":3159},[57,10988,3169],{"class":269},[57,10990,10992,10994,10997,10999,11001,11003,11005,11007],{"class":59,"line":10991},51,[57,10993,3413],{"class":3412},[57,10995,10996],{"class":269},"(TransformerDecoder, ",[57,10998,266],{"class":265},[57,11000,3421],{"class":269},[57,11002,869],{"class":273},[57,11004,1754],{"class":269},[57,11006,3402],{"class":280},[57,11008,3430],{"class":269},[57,11010,11012,11014,11016,11018],{"class":59,"line":11011},52,[57,11013,3435],{"class":265},[57,11015,10138],{"class":269},[57,11017,281],{"class":280},[57,11019,10143],{"class":269},[57,11021,11023,11025,11028,11030],{"class":59,"line":11022},53,[57,11024,3435],{"class":265},[57,11026,11027],{"class":269},".num_layers ",[57,11029,281],{"class":280},[57,11031,11032],{"class":269}," num_layers\n",[57,11034,11036,11038,11040,11042],{"class":59,"line":11035},54,[57,11037,3435],{"class":265},[57,11039,10150],{"class":269},[57,11041,281],{"class":280},[57,11043,10155],{"class":269},[57,11045,11047,11049,11051,11053],{"class":59,"line":11046},55,[57,11048,3435],{"class":265},[57,11050,10162],{"class":269},[57,11052,281],{"class":280},[57,11054,10167],{"class":269},[57,11056,11058,11060,11062,11064],{"class":59,"line":11057},56,[57,11059,3435],{"class":265},[57,11061,10174],{"class":269},[57,11063,281],{"class":280},[57,11065,10179],{"class":269},[57,11067,11069,11071,11073,11075,11077],{"class":59,"line":11068},57,[57,11070,10184],{"class":280},[57,11072,10187],{"class":269},[57,11074,10190],{"class":280},[57,11076,10194],{"class":10193},[57,11078,10197],{"class":269},[57,11080,11082,11084,11086,11088,11090,11092,11094,11096],{"class":59,"line":11081},58,[57,11083,10202],{"class":265},[57,11085,10205],{"class":269},[57,11087,297],{"class":289},[57,11089,10210],{"class":293},[57,11091,297],{"class":289},[57,11093,6487],{"class":280},[57,11095,10217],{"class":3412},[57,11097,10220],{"class":269},[57,11099,11101],{"class":59,"line":11100},59,[57,11102,11103],{"class":269},"                DecoderBlock(key_size, query_size, value_size, num_hiddens,\n",[57,11105,11107],{"class":59,"line":11106},60,[57,11108,10230],{"class":269},[57,11110,11112],{"class":59,"line":11111},61,[57,11113,11114],{"class":269},"                             num_heads, dropout, i))\n",[57,11116,11118,11120,11123,11125],{"class":59,"line":11117},62,[57,11119,3435],{"class":265},[57,11121,11122],{"class":269},".dense ",[57,11124,281],{"class":280},[57,11126,11127],{"class":269}," nn.Linear(num_hiddens, vocab_size)\n",[57,11129,11131],{"class":59,"line":11130},63,[57,11132,3448],{"emptyLinePlaceholder":963},[57,11134,11136,11138,11140,11142,11144,11146,11148,11150,11153,11155,11157,11159],{"class":59,"line":11135},64,[57,11137,3384],{"class":280},[57,11139,9548],{"class":3153},[57,11141,1754],{"class":269},[57,11143,266],{"class":3392},[57,11145,3163],{"class":269},[57,11147,9557],{"class":3159},[57,11149,3163],{"class":269},[57,11151,11152],{"class":3159},"enc_valid_lens",[57,11154,3163],{"class":269},[57,11156,9461],{"class":280},[57,11158,9464],{"class":3159},[57,11160,3169],{"class":269},[57,11162,11164,11166,11169,11171,11173,11175,11177],{"class":59,"line":11163},65,[57,11165,3202],{"class":280},[57,11167,11168],{"class":269}," [enc_outputs, enc_valid_lens, [",[57,11170,3503],{"class":273},[57,11172,277],{"class":269},[57,11174,9461],{"class":280},[57,11176,5935],{"class":265},[57,11178,11179],{"class":269},".num_layers]\n",[57,11181,11183],{"class":59,"line":11182},66,[57,11184,3448],{"emptyLinePlaceholder":963},[57,11186,11188,11190,11192,11194,11196,11198,11200,11202,11204],{"class":59,"line":11187},67,[57,11189,3384],{"class":280},[57,11191,3475],{"class":3153},[57,11193,1754],{"class":269},[57,11195,266],{"class":3392},[57,11197,3163],{"class":269},[57,11199,3160],{"class":3159},[57,11201,3163],{"class":269},[57,11203,9594],{"class":3159},[57,11205,3169],{"class":269},[57,11207,11209,11211,11213,11215,11217,11219,11221,11223,11225,11227],{"class":59,"line":11208},68,[57,11210,3298],{"class":269},[57,11212,281],{"class":280},[57,11214,5935],{"class":265},[57,11216,10294],{"class":269},[57,11218,266],{"class":265},[57,11220,10299],{"class":269},[57,11222,9461],{"class":280},[57,11224,10304],{"class":269},[57,11226,266],{"class":265},[57,11228,10309],{"class":269},[57,11230,11232,11234,11237,11239,11242,11244,11246,11248,11250,11252,11254,11257,11260,11263,11265,11267,11270,11272],{"class":59,"line":11231},69,[57,11233,3435],{"class":265},[57,11235,11236],{"class":269},"._attention_weights ",[57,11238,281],{"class":280},[57,11240,11241],{"class":269}," [[",[57,11243,3503],{"class":273},[57,11245,277],{"class":269},[57,11247,9461],{"class":280},[57,11249,10330],{"class":10193},[57,11251,1754],{"class":269},[57,11253,266],{"class":265},[57,11255,11256],{"class":269},".blks) ",[57,11258,11259],{"class":280},"for",[57,11261,11262],{"class":269}," _ ",[57,11264,10190],{"class":280},[57,11266,10194],{"class":10193},[57,11268,11269],{"class":269}," (",[57,11271,590],{"class":273},[57,11273,11274],{"class":269},")]\n",[57,11276,11278,11280,11282,11284,11286,11288,11290],{"class":59,"line":11277},70,[57,11279,10184],{"class":280},[57,11281,10345],{"class":269},[57,11283,10190],{"class":280},[57,11285,10350],{"class":10193},[57,11287,1754],{"class":269},[57,11289,266],{"class":265},[57,11291,10357],{"class":269},[57,11293,11295,11298,11300],{"class":59,"line":11294},71,[57,11296,11297],{"class":269},"            X, state ",[57,11299,281],{"class":280},[57,11301,11302],{"class":269}," blk(X, state)\n",[57,11304,11306],{"class":59,"line":11305},72,[57,11307,11308],{"class":303},"            # 解码器自注意力权重\n",[57,11310,11312,11314,11317,11319,11322,11324],{"class":59,"line":11311},73,[57,11313,10202],{"class":265},[57,11315,11316],{"class":269},"._attention_weights[",[57,11318,560],{"class":273},[57,11320,11321],{"class":269},"][i] ",[57,11323,281],{"class":280},[57,11325,11326],{"class":269}," blk.attention1.attention.attention_weights\n",[57,11328,11330],{"class":59,"line":11329},74,[57,11331,11332],{"class":303},"            # “编码器－解码器”自注意力权重\n",[57,11334,11336,11338,11340,11342,11344,11346],{"class":59,"line":11335},75,[57,11337,10202],{"class":265},[57,11339,11316],{"class":269},[57,11341,574],{"class":273},[57,11343,11321],{"class":269},[57,11345,281],{"class":280},[57,11347,11348],{"class":269}," blk.attention2.attention.attention_weights\n",[57,11350,11352,11354,11356],{"class":59,"line":11351},76,[57,11353,3202],{"class":280},[57,11355,5935],{"class":265},[57,11357,11358],{"class":269},".dense(X), state\n",[57,11360,11362],{"class":59,"line":11361},77,[57,11363,3448],{"emptyLinePlaceholder":963},[57,11365,11367,11370],{"class":59,"line":11366},78,[57,11368,11369],{"class":3153},"    @",[57,11371,11372],{"class":3412},"property\n",[57,11374,11376,11378,11381,11383,11385],{"class":59,"line":11375},79,[57,11377,3384],{"class":280},[57,11379,11380],{"class":3153}," attention_weights",[57,11382,1754],{"class":269},[57,11384,266],{"class":3392},[57,11386,3169],{"class":269},[57,11388,11390,11392,11394],{"class":59,"line":11389},80,[57,11391,3202],{"class":280},[57,11393,5935],{"class":265},[57,11395,11396],{"class":269},"._attention_weights\n",[10,11398,11399],{"id":11399},"训练",[14,11401,11402],{},"训练时同样需要屏蔽 padding 等无效位置，避免它们污染损失值：",[47,11404,11406],{"className":256,"code":11405,"language":258,"meta":52,"style":52},"def sequence_mask(X, valid_len, value=0):\n    \"\"\"在序列中屏蔽不相关的项\"\"\"\n    maxlen = X.size(1)\n    mask = torch.arange((maxlen), dtype=torch.float32,\n                        device=X.device)[None, :] \u003C valid_len[:, None]\n    X[~mask] = value\n    return X\n\nclass MaskedSoftmaxCELoss(nn.CrossEntropyLoss):\n    \"\"\"带遮蔽的softmax交叉熵损失函数\"\"\"\n    # pred的形状：(batch_size,num_steps,vocab_size)\n    # label的形状：(batch_size,num_steps)\n    # valid_len的形状：(batch_size,)\n    def forward(self, pred, label, valid_len):\n        weights = torch.ones_like(label)\n        weights = sequence_mask(weights, valid_len)\n        self.reduction='none'\n        unweighted_loss = super(MaskedSoftmaxCELoss, self).forward(\n            pred.permute(0, 2, 1), label)\n        weighted_loss = (unweighted_loss * weights).mean(dim=1)\n        return weighted_loss\n",[54,11407,11408,11434,11439,11453,11471,11500,11516,11522,11526,11544,11549,11554,11559,11564,11590,11600,11609,11627,11645,11663,11686],{"__ignoreMap":52},[57,11409,11410,11412,11415,11417,11419,11421,11424,11426,11428,11430,11432],{"class":59,"line":60},[57,11411,3150],{"class":280},[57,11413,11414],{"class":3153}," sequence_mask",[57,11416,1754],{"class":269},[57,11418,3160],{"class":3159},[57,11420,3163],{"class":269},[57,11422,11423],{"class":3159},"valid_len",[57,11425,3163],{"class":269},[57,11427,687],{"class":3159},[57,11429,281],{"class":280},[57,11431,560],{"class":273},[57,11433,3169],{"class":269},[57,11435,11436],{"class":59,"line":66},[57,11437,11438],{"class":3174},"    \"\"\"在序列中屏蔽不相关的项\"\"\"\n",[57,11440,11441,11444,11446,11449,11451],{"class":59,"line":72},[57,11442,11443],{"class":269},"    maxlen ",[57,11445,281],{"class":280},[57,11447,11448],{"class":269}," X.size(",[57,11450,574],{"class":273},[57,11452,3217],{"class":269},[57,11454,11455,11458,11460,11463,11466,11468],{"class":59,"line":78},[57,11456,11457],{"class":269},"    mask ",[57,11459,281],{"class":280},[57,11461,11462],{"class":269}," torch.arange((maxlen), ",[57,11464,11465],{"class":3208},"dtype",[57,11467,281],{"class":280},[57,11469,11470],{"class":269},"torch.float32,\n",[57,11472,11473,11476,11478,11481,11483,11486,11489,11492,11494,11496,11498],{"class":59,"line":84},[57,11474,11475],{"class":3208},"                        device",[57,11477,281],{"class":280},[57,11479,11480],{"class":269},"X.device)[",[57,11482,3503],{"class":273},[57,11484,11485],{"class":269},", :] ",[57,11487,11488],{"class":280},"\u003C",[57,11490,11491],{"class":269}," valid_len[",[57,11493,7851],{"class":7850},[57,11495,3163],{"class":269},[57,11497,3503],{"class":273},[57,11499,3522],{"class":269},[57,11501,11502,11505,11508,11511,11513],{"class":59,"line":90},[57,11503,11504],{"class":269},"    X[",[57,11506,11507],{"class":280},"~",[57,11509,11510],{"class":269},"mask] ",[57,11512,281],{"class":280},[57,11514,11515],{"class":269}," value\n",[57,11517,11518,11520],{"class":59,"line":96},[57,11519,5407],{"class":280},[57,11521,10389],{"class":269},[57,11523,11524],{"class":59,"line":102},[57,11525,3448],{"emptyLinePlaceholder":963},[57,11527,11528,11530,11533,11535,11537,11539,11542],{"class":59,"line":108},[57,11529,3357],{"class":280},[57,11531,11532],{"class":3360}," MaskedSoftmaxCELoss",[57,11534,1754],{"class":269},[57,11536,3367],{"class":3366},[57,11538,1818],{"class":269},[57,11540,11541],{"class":3366},"CrossEntropyLoss",[57,11543,3169],{"class":269},[57,11545,11546],{"class":59,"line":114},[57,11547,11548],{"class":3174},"    \"\"\"带遮蔽的softmax交叉熵损失函数\"\"\"\n",[57,11550,11551],{"class":59,"line":120},[57,11552,11553],{"class":303},"    # pred的形状：(batch_size,num_steps,vocab_size)\n",[57,11555,11556],{"class":59,"line":126},[57,11557,11558],{"class":303},"    # label的形状：(batch_size,num_steps)\n",[57,11560,11561],{"class":59,"line":132},[57,11562,11563],{"class":303},"    # valid_len的形状：(batch_size,)\n",[57,11565,11566,11568,11570,11572,11574,11576,11579,11581,11584,11586,11588],{"class":59,"line":138},[57,11567,3384],{"class":280},[57,11569,3475],{"class":3153},[57,11571,1754],{"class":269},[57,11573,266],{"class":3392},[57,11575,3163],{"class":269},[57,11577,11578],{"class":3159},"pred",[57,11580,3163],{"class":269},[57,11582,11583],{"class":3159},"label",[57,11585,3163],{"class":269},[57,11587,11423],{"class":3159},[57,11589,3169],{"class":269},[57,11591,11592,11595,11597],{"class":59,"line":144},[57,11593,11594],{"class":269},"        weights ",[57,11596,281],{"class":280},[57,11598,11599],{"class":269}," torch.ones_like(label)\n",[57,11601,11602,11604,11606],{"class":59,"line":150},[57,11603,11594],{"class":269},[57,11605,281],{"class":280},[57,11607,11608],{"class":269}," sequence_mask(weights, valid_len)\n",[57,11610,11611,11613,11616,11618,11621,11624],{"class":59,"line":156},[57,11612,3435],{"class":265},[57,11614,11615],{"class":269},".reduction",[57,11617,281],{"class":280},[57,11619,11620],{"class":289},"'",[57,11622,11623],{"class":293},"none",[57,11625,11626],{"class":289},"'\n",[57,11628,11629,11632,11634,11637,11640,11642],{"class":59,"line":162},[57,11630,11631],{"class":269},"        unweighted_loss ",[57,11633,281],{"class":280},[57,11635,11636],{"class":3412}," super",[57,11638,11639],{"class":269},"(MaskedSoftmaxCELoss, ",[57,11641,266],{"class":265},[57,11643,11644],{"class":269},").forward(\n",[57,11646,11647,11650,11652,11654,11656,11658,11660],{"class":59,"line":168},[57,11648,11649],{"class":269},"            pred.permute(",[57,11651,560],{"class":273},[57,11653,3163],{"class":269},[57,11655,590],{"class":273},[57,11657,3163],{"class":269},[57,11659,574],{"class":273},[57,11661,11662],{"class":269},"), label)\n",[57,11664,11665,11668,11670,11673,11675,11678,11680,11682,11684],{"class":59,"line":174},[57,11666,11667],{"class":269},"        weighted_loss ",[57,11669,281],{"class":280},[57,11671,11672],{"class":269}," (unweighted_loss ",[57,11674,9461],{"class":280},[57,11676,11677],{"class":269}," weights).mean(",[57,11679,3209],{"class":3208},[57,11681,281],{"class":280},[57,11683,574],{"class":273},[57,11685,3217],{"class":269},[57,11687,11688,11690],{"class":59,"line":5790},[57,11689,3202],{"class":280},[57,11691,11692],{"class":269}," weighted_loss\n",[14,11694,11695],{},"梯度裁剪函数：",[47,11697,11699],{"className":256,"code":11698,"language":258,"meta":52,"style":52},"def grad_clipping(net, theta):\n    \"\"\"裁剪梯度\"\"\"\n    if isinstance(net, nn.Module):\n        params = [p for p in net.parameters() if p.requires_grad]\n    else:\n        params = net.params\n    norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params))\n    if norm > theta:\n        for param in params:\n            param.grad[:] *= theta / norm\n",[54,11700,11701,11720,11725,11735,11761,11767,11776,11808,11821,11833],{"__ignoreMap":52},[57,11702,11703,11705,11708,11710,11713,11715,11718],{"class":59,"line":60},[57,11704,3150],{"class":280},[57,11706,11707],{"class":3153}," grad_clipping",[57,11709,1754],{"class":269},[57,11711,11712],{"class":3159},"net",[57,11714,3163],{"class":269},[57,11716,11717],{"class":3159},"theta",[57,11719,3169],{"class":269},[57,11721,11722],{"class":59,"line":66},[57,11723,11724],{"class":3174},"    \"\"\"裁剪梯度\"\"\"\n",[57,11726,11727,11729,11732],{"class":59,"line":72},[57,11728,3185],{"class":280},[57,11730,11731],{"class":10193}," isinstance",[57,11733,11734],{"class":269},"(net, nn.Module):\n",[57,11736,11737,11740,11742,11745,11747,11750,11752,11755,11758],{"class":59,"line":78},[57,11738,11739],{"class":269},"        params ",[57,11741,281],{"class":280},[57,11743,11744],{"class":269}," [p ",[57,11746,11259],{"class":280},[57,11748,11749],{"class":269}," p ",[57,11751,10190],{"class":280},[57,11753,11754],{"class":269}," net.parameters() ",[57,11756,11757],{"class":280},"if",[57,11759,11760],{"class":269}," p.requires_grad]\n",[57,11762,11763,11765],{"class":59,"line":84},[57,11764,3222],{"class":280},[57,11766,3197],{"class":269},[57,11768,11769,11771,11773],{"class":59,"line":90},[57,11770,11739],{"class":269},[57,11772,281],{"class":280},[57,11774,11775],{"class":269}," net.params\n",[57,11777,11778,11781,11783,11786,11789,11792,11794,11797,11799,11801,11803,11805],{"class":59,"line":96},[57,11779,11780],{"class":269},"    norm ",[57,11782,281],{"class":280},[57,11784,11785],{"class":269}," torch.sqrt(",[57,11787,11788],{"class":10193},"sum",[57,11790,11791],{"class":269},"(torch.sum((p.grad ",[57,11793,3402],{"class":280},[57,11795,11796],{"class":273}," 2",[57,11798,3546],{"class":269},[57,11800,11259],{"class":280},[57,11802,11749],{"class":269},[57,11804,10190],{"class":280},[57,11806,11807],{"class":269}," params))\n",[57,11809,11810,11812,11815,11818],{"class":59,"line":102},[57,11811,3185],{"class":280},[57,11813,11814],{"class":269}," norm ",[57,11816,11817],{"class":280},">",[57,11819,11820],{"class":269}," theta:\n",[57,11822,11823,11825,11828,11830],{"class":59,"line":108},[57,11824,10184],{"class":280},[57,11826,11827],{"class":269}," param ",[57,11829,10190],{"class":280},[57,11831,11832],{"class":269}," params:\n",[57,11834,11835,11838,11840,11842,11845,11848,11850],{"class":59,"line":114},[57,11836,11837],{"class":269},"            param.grad[",[57,11839,7851],{"class":7850},[57,11841,277],{"class":269},[57,11843,11844],{"class":280},"*=",[57,11846,11847],{"class":269}," theta ",[57,11849,3549],{"class":280},[57,11851,11852],{"class":269}," norm\n",[14,11854,11855,11856,11859,11860,11863,11864,11868],{},"在训练时，会把特殊的开始符 ",[54,11857,11858],{},"\u003Cbos>"," 与原始输出序列（去掉结束符 ",[54,11861,11862],{},"\u003Ceos>","）拼接，作为解码器的输入——这种做法称为 ",[11865,11866,11867],"strong",{},"强制教学（teacher forcing）","：直接将真实标签喂给解码器，避免训练早期因预测漂移导致的误差累积；与之相对的另一种选择，是让上一时间步预测出的 token 作为下一时间步的输入。",[47,11870,11872],{"className":256,"code":11871,"language":258,"meta":52,"style":52},"def train_seq2seq(net, data_iter, lr, num_epochs, tgt_vocab, device):\n    \"\"\"训练序列到序列模型\"\"\"\n    def xavier_init_weights(m):\n        if type(m) == nn.Linear:\n            nn.init.xavier_uniform_(m.weight)\n        if type(m) == nn.GRU:\n            for param in m._flat_weights_names:\n                if \"weight\" in param:\n                    nn.init.xavier_uniform_(m._parameters[param])\n\n    net.apply(xavier_init_weights)\n    net.to(device)\n    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n    loss = MaskedSoftmaxCELoss()\n    net.train()\n    animator = d2l.Animator(xlabel='epoch', ylabel='loss',\n                     xlim=[10, num_epochs])\n    for epoch in range(num_epochs):\n        timer = d2l.Timer()\n        metric = d2l.Accumulator(2)  # 训练损失总和，词元数量\n        for batch in data_iter:\n            optimizer.zero_grad()\n            X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n            bos = torch.tensor([tgt_vocab['\u003Cbos>']] * Y.shape[0],\n                          device=device).reshape(-1, 1)\n            dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学\n            Y_hat, _ = net(X, dec_input, X_valid_len)\n            l = loss(Y_hat, Y, Y_valid_len)\n            l.sum().backward()      # 损失函数的标量进行“反向传播”\n            grad_clipping(net, 1)\n            num_tokens = Y_valid_len.sum()\n            optimizer.step()\n            with torch.no_grad():\n                metric.add(l.sum(), num_tokens)\n        if (epoch + 1) % 10 == 0:\n            animator.add(epoch + 1, (metric[0] / metric[1],))\n    print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} '\n        f'tokens/sec on {str(device)}')\n",[54,11873,11874,11911,11916,11929,11944,11949,11967,11979,11998,12003,12007,12012,12017,12034,12044,12049,12087,12103,12118,12128,12146,12158,12163,12183,12212,12232,12262,12272,12282,12290,12299,12309,12314,12322,12327,12354,12380,12441],{"__ignoreMap":52},[57,11875,11876,11878,11881,11883,11885,11887,11890,11892,11895,11897,11900,11902,11905,11907,11909],{"class":59,"line":60},[57,11877,3150],{"class":280},[57,11879,11880],{"class":3153}," train_seq2seq",[57,11882,1754],{"class":269},[57,11884,11712],{"class":3159},[57,11886,3163],{"class":269},[57,11888,11889],{"class":3159},"data_iter",[57,11891,3163],{"class":269},[57,11893,11894],{"class":3159},"lr",[57,11896,3163],{"class":269},[57,11898,11899],{"class":3159},"num_epochs",[57,11901,3163],{"class":269},[57,11903,11904],{"class":3159},"tgt_vocab",[57,11906,3163],{"class":269},[57,11908,10794],{"class":3159},[57,11910,3169],{"class":269},[57,11912,11913],{"class":59,"line":66},[57,11914,11915],{"class":3174},"    \"\"\"训练序列到序列模型\"\"\"\n",[57,11917,11918,11920,11923,11925,11927],{"class":59,"line":72},[57,11919,3384],{"class":280},[57,11921,11922],{"class":3153}," xavier_init_weights",[57,11924,1754],{"class":269},[57,11926,1319],{"class":3159},[57,11928,3169],{"class":269},[57,11930,11931,11933,11936,11939,11941],{"class":59,"line":78},[57,11932,3239],{"class":280},[57,11934,11935],{"class":3412}," type",[57,11937,11938],{"class":269},"(m) ",[57,11940,3245],{"class":280},[57,11942,11943],{"class":269}," nn.Linear:\n",[57,11945,11946],{"class":59,"line":84},[57,11947,11948],{"class":269},"            nn.init.xavier_uniform_(m.weight)\n",[57,11950,11951,11953,11955,11957,11959,11962,11965],{"class":59,"line":90},[57,11952,3239],{"class":280},[57,11954,11935],{"class":3412},[57,11956,11938],{"class":269},[57,11958,3245],{"class":280},[57,11960,11961],{"class":269}," nn.",[57,11963,11964],{"class":273},"GRU",[57,11966,3197],{"class":269},[57,11968,11969,11972,11974,11976],{"class":59,"line":96},[57,11970,11971],{"class":280},"            for",[57,11973,11827],{"class":269},[57,11975,10190],{"class":280},[57,11977,11978],{"class":269}," m._flat_weights_names:\n",[57,11980,11981,11984,11987,11990,11992,11995],{"class":59,"line":102},[57,11982,11983],{"class":280},"                if",[57,11985,11986],{"class":289}," \"",[57,11988,11989],{"class":293},"weight",[57,11991,297],{"class":289},[57,11993,11994],{"class":280}," in",[57,11996,11997],{"class":269}," param:\n",[57,11999,12000],{"class":59,"line":108},[57,12001,12002],{"class":269},"                    nn.init.xavier_uniform_(m._parameters[param])\n",[57,12004,12005],{"class":59,"line":114},[57,12006,3448],{"emptyLinePlaceholder":963},[57,12008,12009],{"class":59,"line":120},[57,12010,12011],{"class":269},"    net.apply(xavier_init_weights)\n",[57,12013,12014],{"class":59,"line":126},[57,12015,12016],{"class":269},"    net.to(device)\n",[57,12018,12019,12022,12024,12027,12029,12031],{"class":59,"line":132},[57,12020,12021],{"class":269},"    optimizer ",[57,12023,281],{"class":280},[57,12025,12026],{"class":269}," torch.optim.Adam(net.parameters(), ",[57,12028,11894],{"class":3208},[57,12030,281],{"class":280},[57,12032,12033],{"class":269},"lr)\n",[57,12035,12036,12039,12041],{"class":59,"line":138},[57,12037,12038],{"class":269},"    loss ",[57,12040,281],{"class":280},[57,12042,12043],{"class":269}," MaskedSoftmaxCELoss()\n",[57,12045,12046],{"class":59,"line":144},[57,12047,12048],{"class":269},"    net.train()\n",[57,12050,12051,12054,12056,12059,12062,12064,12066,12069,12071,12073,12076,12078,12080,12083,12085],{"class":59,"line":150},[57,12052,12053],{"class":269},"    animator ",[57,12055,281],{"class":280},[57,12057,12058],{"class":269}," d2l.Animator(",[57,12060,12061],{"class":3208},"xlabel",[57,12063,281],{"class":280},[57,12065,11620],{"class":289},[57,12067,12068],{"class":293},"epoch",[57,12070,11620],{"class":289},[57,12072,3163],{"class":269},[57,12074,12075],{"class":3208},"ylabel",[57,12077,281],{"class":280},[57,12079,11620],{"class":289},[57,12081,12082],{"class":293},"loss",[57,12084,11620],{"class":289},[57,12086,369],{"class":269},[57,12088,12089,12092,12094,12097,12100],{"class":59,"line":156},[57,12090,12091],{"class":3208},"                     xlim",[57,12093,281],{"class":280},[57,12095,12096],{"class":269},"[",[57,12098,12099],{"class":273},"10",[57,12101,12102],{"class":269},", num_epochs])\n",[57,12104,12105,12108,12111,12113,12115],{"class":59,"line":162},[57,12106,12107],{"class":280},"    for",[57,12109,12110],{"class":269}," epoch ",[57,12112,10190],{"class":280},[57,12114,10194],{"class":10193},[57,12116,12117],{"class":269},"(num_epochs):\n",[57,12119,12120,12123,12125],{"class":59,"line":168},[57,12121,12122],{"class":269},"        timer ",[57,12124,281],{"class":280},[57,12126,12127],{"class":269}," d2l.Timer()\n",[57,12129,12130,12133,12135,12138,12140,12143],{"class":59,"line":174},[57,12131,12132],{"class":269},"        metric ",[57,12134,281],{"class":280},[57,12136,12137],{"class":269}," d2l.Accumulator(",[57,12139,590],{"class":273},[57,12141,12142],{"class":269},")  ",[57,12144,12145],{"class":303},"# 训练损失总和，词元数量\n",[57,12147,12148,12150,12153,12155],{"class":59,"line":5790},[57,12149,10184],{"class":280},[57,12151,12152],{"class":269}," batch ",[57,12154,10190],{"class":280},[57,12156,12157],{"class":269}," data_iter:\n",[57,12159,12160],{"class":59,"line":5811},[57,12161,12162],{"class":269},"            optimizer.zero_grad()\n",[57,12164,12165,12168,12170,12173,12175,12178,12180],{"class":59,"line":5830},[57,12166,12167],{"class":269},"            X, X_valid_len, Y, Y_valid_len ",[57,12169,281],{"class":280},[57,12171,12172],{"class":269}," [x.to(device) ",[57,12174,11259],{"class":280},[57,12176,12177],{"class":269}," x ",[57,12179,10190],{"class":280},[57,12181,12182],{"class":269}," batch]\n",[57,12184,12185,12188,12190,12193,12195,12197,12199,12202,12204,12207,12209],{"class":59,"line":5849},[57,12186,12187],{"class":269},"            bos ",[57,12189,281],{"class":280},[57,12191,12192],{"class":269}," torch.tensor([tgt_vocab[",[57,12194,11620],{"class":289},[57,12196,11858],{"class":293},[57,12198,11620],{"class":289},[57,12200,12201],{"class":269},"]] ",[57,12203,9461],{"class":280},[57,12205,12206],{"class":269}," Y.shape[",[57,12208,560],{"class":273},[57,12210,12211],{"class":269},"],\n",[57,12213,12214,12217,12219,12222,12224,12226,12228,12230],{"class":59,"line":5854},[57,12215,12216],{"class":3208},"                          device",[57,12218,281],{"class":280},[57,12220,12221],{"class":269},"device).reshape(",[57,12223,3284],{"class":280},[57,12225,574],{"class":273},[57,12227,3163],{"class":269},[57,12229,574],{"class":273},[57,12231,3217],{"class":269},[57,12233,12234,12237,12239,12242,12244,12246,12248,12250,12252,12255,12257,12259],{"class":59,"line":5870},[57,12235,12236],{"class":269},"            dec_input ",[57,12238,281],{"class":280},[57,12240,12241],{"class":269}," torch.cat([bos, Y[",[57,12243,7851],{"class":7850},[57,12245,3163],{"class":269},[57,12247,7851],{"class":7850},[57,12249,3284],{"class":280},[57,12251,574],{"class":273},[57,12253,12254],{"class":269},"]], ",[57,12256,574],{"class":273},[57,12258,12142],{"class":269},[57,12260,12261],{"class":303},"# 强制教学\n",[57,12263,12264,12267,12269],{"class":59,"line":5876},[57,12265,12266],{"class":269},"            Y_hat, _ ",[57,12268,281],{"class":280},[57,12270,12271],{"class":269}," net(X, dec_input, X_valid_len)\n",[57,12273,12274,12277,12279],{"class":59,"line":5882},[57,12275,12276],{"class":269},"            l ",[57,12278,281],{"class":280},[57,12280,12281],{"class":269}," loss(Y_hat, Y, Y_valid_len)\n",[57,12283,12284,12287],{"class":59,"line":5892},[57,12285,12286],{"class":269},"            l.sum().backward()      ",[57,12288,12289],{"class":303},"# 损失函数的标量进行“反向传播”\n",[57,12291,12292,12295,12297],{"class":59,"line":5916},[57,12293,12294],{"class":269},"            grad_clipping(net, ",[57,12296,574],{"class":273},[57,12298,3217],{"class":269},[57,12300,12301,12304,12306],{"class":59,"line":5921},[57,12302,12303],{"class":269},"            num_tokens ",[57,12305,281],{"class":280},[57,12307,12308],{"class":269}," Y_valid_len.sum()\n",[57,12310,12311],{"class":59,"line":5927},[57,12312,12313],{"class":269},"            optimizer.step()\n",[57,12315,12316,12319],{"class":59,"line":5941},[57,12317,12318],{"class":280},"            with",[57,12320,12321],{"class":269}," torch.no_grad():\n",[57,12323,12324],{"class":59,"line":5946},[57,12325,12326],{"class":269},"                metric.add(l.sum(), num_tokens)\n",[57,12328,12329,12331,12334,12336,12338,12340,12343,12346,12349,12352],{"class":59,"line":5952},[57,12330,3239],{"class":280},[57,12332,12333],{"class":269}," (epoch ",[57,12335,6487],{"class":280},[57,12337,3248],{"class":273},[57,12339,7813],{"class":269},[57,12341,12342],{"class":280},"%",[57,12344,12345],{"class":273}," 10",[57,12347,12348],{"class":280}," ==",[57,12350,12351],{"class":273}," 0",[57,12353,3197],{"class":269},[57,12355,12356,12359,12361,12363,12366,12368,12370,12372,12375,12377],{"class":59,"line":5967},[57,12357,12358],{"class":269},"            animator.add(epoch ",[57,12360,6487],{"class":280},[57,12362,3248],{"class":273},[57,12364,12365],{"class":269},", (metric[",[57,12367,560],{"class":273},[57,12369,277],{"class":269},[57,12371,3549],{"class":280},[57,12373,12374],{"class":269}," metric[",[57,12376,574],{"class":273},[57,12378,12379],{"class":269},"],))\n",[57,12381,12382,12385,12387,12390,12393,12396,12399,12401,12403,12405,12407,12409,12412,12415,12418,12420,12422,12424,12426,12428,12430,12433,12436,12438],{"class":59,"line":10273},[57,12383,12384],{"class":10193},"    print",[57,12386,1754],{"class":269},[57,12388,12389],{"class":280},"f",[57,12391,12392],{"class":293},"'loss ",[57,12394,12395],{"class":273},"{",[57,12397,12398],{"class":269},"metric[",[57,12400,560],{"class":273},[57,12402,277],{"class":269},[57,12404,3549],{"class":280},[57,12406,12374],{"class":269},[57,12408,574],{"class":273},[57,12410,12411],{"class":269},"]",[57,12413,12414],{"class":280},":.3f",[57,12416,12417],{"class":273},"}",[57,12419,3163],{"class":293},[57,12421,12395],{"class":273},[57,12423,12398],{"class":269},[57,12425,574],{"class":273},[57,12427,277],{"class":269},[57,12429,3549],{"class":280},[57,12431,12432],{"class":269}," timer.stop()",[57,12434,12435],{"class":280},":.1f",[57,12437,12417],{"class":273},[57,12439,12440],{"class":293}," '\n",[57,12442,12443,12446,12449,12451,12453,12456,12458,12460],{"class":59,"line":10279},[57,12444,12445],{"class":280},"        f",[57,12447,12448],{"class":293},"'tokens/sec on ",[57,12450,12395],{"class":273},[57,12452,10217],{"class":3412},[57,12454,12455],{"class":269},"(device)",[57,12457,12417],{"class":273},[57,12459,11620],{"class":293},[57,12461,3217],{"class":269},[14,12463,12464],{},"进行数据集的数据处理",[47,12466,12468],{"className":256,"code":12467,"language":258,"meta":52,"style":52},"def read_data_nmt():\n    \"\"\"载入“英语－法语”数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    data_dir = d2l.download_extract('fra-eng')\n    with open(os.path.join(data_dir, 'fra.txt'), 'r',\n             encoding='utf-8') as f:\n        return f.read()\n\ndef preprocess_nmt(text):\n    \"\"\"预处理“英语－法语”数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    def no_space(char, prev_char):\n        return char in set(',.!?') and prev_char != ' '\n\n    # 使用空格替换不间断空格\n    # 使用小写字母替换大写字母\n    text = text.replace('\\u202f', ' ').replace('\\xa0', ' ').lower()\n    # 在单词和标点符号之间插入空格\n    out = [' ' + char if i > 0 and no_space(char, text[i - 1]) else char\n           for i, char in enumerate(text)]\n    return ''.join(out)\n\ndef tokenize_nmt(text, num_examples=None):\n    \"\"\"词元化“英语－法语”数据数据集\n\n    Defined in :numref:`sec_machine_translation`\"\"\"\n    source, target = [], []\n    for i, line in enumerate(text.split('\\n')):\n        if num_examples and i > num_examples:\n            break\n        parts = line.split('\\t')\n        if len(parts) == 2:\n            source.append(parts[0].split(' '))\n            target.append(parts[1].split(' '))\n    return source, target\n\ndef load_data_nmt(batch_size, num_steps, num_examples=600):\n    \"\"\"返回翻译数据集的迭代器和词表\n\n    Defined in :numref:`subsec_mt_data_loading`\"\"\"\n    text = preprocess_nmt(read_data_nmt())\n    source, target = tokenize_nmt(text, num_examples)\n    src_vocab = d2l.Vocab(source, min_freq=2,\n                          reserved_tokens=['\u003Cpad>', '\u003Cbos>', '\u003Ceos>'])\n    tgt_vocab = d2l.Vocab(target, min_freq=2,\n                          reserved_tokens=['\u003Cpad>', '\u003Cbos>', '\u003Ceos>'])\n    src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)\n    tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps)\n    data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len)\n    data_iter = d2l.load_array(data_arrays, batch_size)\n    return data_iter, src_vocab, tgt_vocab\n",[54,12469,12470,12480,12485,12489,12494,12513,12542,12564,12571,12575,12588,12593,12597,12601,12620,12657,12661,12666,12671,12714,12719,12764,12779,12789,12793,12815,12820,12824,12828,12838,12862,12878,12883,12902,12917,12934,12949,12956,12960,12987,12992,12996,13001,13010,13019,13038,13072,13090,13122,13132,13142,13152,13162],{"__ignoreMap":52},[57,12471,12472,12474,12477],{"class":59,"line":60},[57,12473,3150],{"class":280},[57,12475,12476],{"class":3153}," read_data_nmt",[57,12478,12479],{"class":269},"():\n",[57,12481,12482],{"class":59,"line":66},[57,12483,12484],{"class":3174},"    \"\"\"载入“英语－法语”数据集\n",[57,12486,12487],{"class":59,"line":72},[57,12488,3448],{"emptyLinePlaceholder":963},[57,12490,12491],{"class":59,"line":78},[57,12492,12493],{"class":3174},"    Defined in :numref:`sec_machine_translation`\"\"\"\n",[57,12495,12496,12499,12501,12504,12506,12509,12511],{"class":59,"line":84},[57,12497,12498],{"class":269},"    data_dir ",[57,12500,281],{"class":280},[57,12502,12503],{"class":269}," d2l.download_extract(",[57,12505,11620],{"class":289},[57,12507,12508],{"class":293},"fra-eng",[57,12510,11620],{"class":289},[57,12512,3217],{"class":269},[57,12514,12515,12518,12521,12524,12526,12529,12531,12534,12536,12538,12540],{"class":59,"line":90},[57,12516,12517],{"class":280},"    with",[57,12519,12520],{"class":10193}," open",[57,12522,12523],{"class":269},"(os.path.join(data_dir, ",[57,12525,11620],{"class":289},[57,12527,12528],{"class":293},"fra.txt",[57,12530,11620],{"class":289},[57,12532,12533],{"class":269},"), ",[57,12535,11620],{"class":289},[57,12537,8500],{"class":293},[57,12539,11620],{"class":289},[57,12541,369],{"class":269},[57,12543,12544,12547,12549,12551,12554,12556,12558,12561],{"class":59,"line":96},[57,12545,12546],{"class":3208},"             encoding",[57,12548,281],{"class":280},[57,12550,11620],{"class":289},[57,12552,12553],{"class":293},"utf-8",[57,12555,11620],{"class":289},[57,12557,7813],{"class":269},[57,12559,12560],{"class":280},"as",[57,12562,12563],{"class":269}," f:\n",[57,12565,12566,12568],{"class":59,"line":102},[57,12567,3202],{"class":280},[57,12569,12570],{"class":269}," f.read()\n",[57,12572,12573],{"class":59,"line":108},[57,12574,3448],{"emptyLinePlaceholder":963},[57,12576,12577,12579,12582,12584,12586],{"class":59,"line":114},[57,12578,3150],{"class":280},[57,12580,12581],{"class":3153}," preprocess_nmt",[57,12583,1754],{"class":269},[57,12585,6043],{"class":3159},[57,12587,3169],{"class":269},[57,12589,12590],{"class":59,"line":120},[57,12591,12592],{"class":3174},"    \"\"\"预处理“英语－法语”数据集\n",[57,12594,12595],{"class":59,"line":126},[57,12596,3448],{"emptyLinePlaceholder":963},[57,12598,12599],{"class":59,"line":132},[57,12600,12493],{"class":3174},[57,12602,12603,12605,12608,12610,12613,12615,12618],{"class":59,"line":138},[57,12604,3384],{"class":280},[57,12606,12607],{"class":3153}," no_space",[57,12609,1754],{"class":269},[57,12611,12612],{"class":3159},"char",[57,12614,3163],{"class":269},[57,12616,12617],{"class":3159},"prev_char",[57,12619,3169],{"class":269},[57,12621,12622,12624,12627,12629,12632,12634,12636,12639,12641,12643,12646,12649,12652,12655],{"class":59,"line":144},[57,12623,3202],{"class":280},[57,12625,12626],{"class":269}," char ",[57,12628,10190],{"class":280},[57,12630,12631],{"class":3412}," set",[57,12633,1754],{"class":269},[57,12635,11620],{"class":289},[57,12637,12638],{"class":293},",.!?",[57,12640,11620],{"class":289},[57,12642,7813],{"class":269},[57,12644,12645],{"class":280},"and",[57,12647,12648],{"class":269}," prev_char ",[57,12650,12651],{"class":280},"!=",[57,12653,12654],{"class":289}," '",[57,12656,12440],{"class":289},[57,12658,12659],{"class":59,"line":150},[57,12660,3448],{"emptyLinePlaceholder":963},[57,12662,12663],{"class":59,"line":156},[57,12664,12665],{"class":303},"    # 使用空格替换不间断空格\n",[57,12667,12668],{"class":59,"line":162},[57,12669,12670],{"class":303},"    # 使用小写字母替换大写字母\n",[57,12672,12673,12676,12678,12681,12683,12687,12689,12691,12693,12695,12698,12700,12703,12705,12707,12709,12711],{"class":59,"line":168},[57,12674,12675],{"class":269},"    text ",[57,12677,281],{"class":280},[57,12679,12680],{"class":269}," text.replace(",[57,12682,11620],{"class":289},[57,12684,12686],{"class":12685},"sRfyP","\\u202f",[57,12688,11620],{"class":289},[57,12690,3163],{"class":269},[57,12692,11620],{"class":289},[57,12694,12654],{"class":289},[57,12696,12697],{"class":269},").replace(",[57,12699,11620],{"class":289},[57,12701,12702],{"class":12685},"\\xa0",[57,12704,11620],{"class":289},[57,12706,3163],{"class":269},[57,12708,11620],{"class":289},[57,12710,12654],{"class":289},[57,12712,12713],{"class":269},").lower()\n",[57,12715,12716],{"class":59,"line":174},[57,12717,12718],{"class":303},"    # 在单词和标点符号之间插入空格\n",[57,12720,12721,12724,12726,12728,12730,12732,12735,12737,12739,12741,12743,12745,12748,12751,12753,12755,12758,12761],{"class":59,"line":5790},[57,12722,12723],{"class":269},"    out ",[57,12725,281],{"class":280},[57,12727,10321],{"class":269},[57,12729,11620],{"class":289},[57,12731,12654],{"class":289},[57,12733,12734],{"class":280}," +",[57,12736,12626],{"class":269},[57,12738,11757],{"class":280},[57,12740,10187],{"class":269},[57,12742,11817],{"class":280},[57,12744,12351],{"class":273},[57,12746,12747],{"class":280}," and",[57,12749,12750],{"class":269}," no_space(char, text[i ",[57,12752,3284],{"class":280},[57,12754,3248],{"class":273},[57,12756,12757],{"class":269},"]) ",[57,12759,12760],{"class":280},"else",[57,12762,12763],{"class":269}," char\n",[57,12765,12766,12769,12772,12774,12776],{"class":59,"line":5811},[57,12767,12768],{"class":280},"           for",[57,12770,12771],{"class":269}," i, char ",[57,12773,10190],{"class":280},[57,12775,10350],{"class":10193},[57,12777,12778],{"class":269},"(text)]\n",[57,12780,12781,12783,12786],{"class":59,"line":5830},[57,12782,5407],{"class":280},[57,12784,12785],{"class":289}," ''",[57,12787,12788],{"class":269},".join(out)\n",[57,12790,12791],{"class":59,"line":5849},[57,12792,3448],{"emptyLinePlaceholder":963},[57,12794,12795,12797,12800,12802,12804,12806,12809,12811,12813],{"class":59,"line":5854},[57,12796,3150],{"class":280},[57,12798,12799],{"class":3153}," tokenize_nmt",[57,12801,1754],{"class":269},[57,12803,6043],{"class":3159},[57,12805,3163],{"class":269},[57,12807,12808],{"class":3159},"num_examples",[57,12810,281],{"class":280},[57,12812,3503],{"class":273},[57,12814,3169],{"class":269},[57,12816,12817],{"class":59,"line":5870},[57,12818,12819],{"class":3174},"    \"\"\"词元化“英语－法语”数据数据集\n",[57,12821,12822],{"class":59,"line":5876},[57,12823,3448],{"emptyLinePlaceholder":963},[57,12825,12826],{"class":59,"line":5882},[57,12827,12493],{"class":3174},[57,12829,12830,12833,12835],{"class":59,"line":5892},[57,12831,12832],{"class":269},"    source, target ",[57,12834,281],{"class":280},[57,12836,12837],{"class":269}," [], []\n",[57,12839,12840,12842,12845,12847,12849,12852,12854,12857,12859],{"class":59,"line":5916},[57,12841,12107],{"class":280},[57,12843,12844],{"class":269}," i, line ",[57,12846,10190],{"class":280},[57,12848,10350],{"class":10193},[57,12850,12851],{"class":269},"(text.split(",[57,12853,11620],{"class":289},[57,12855,12856],{"class":12685},"\\n",[57,12858,11620],{"class":289},[57,12860,12861],{"class":269},")):\n",[57,12863,12864,12866,12869,12871,12873,12875],{"class":59,"line":5921},[57,12865,3239],{"class":280},[57,12867,12868],{"class":269}," num_examples ",[57,12870,12645],{"class":280},[57,12872,10187],{"class":269},[57,12874,11817],{"class":280},[57,12876,12877],{"class":269}," num_examples:\n",[57,12879,12880],{"class":59,"line":5927},[57,12881,12882],{"class":280},"            break\n",[57,12884,12885,12888,12890,12893,12895,12898,12900],{"class":59,"line":5941},[57,12886,12887],{"class":269},"        parts ",[57,12889,281],{"class":280},[57,12891,12892],{"class":269}," line.split(",[57,12894,11620],{"class":289},[57,12896,12897],{"class":12685},"\\t",[57,12899,11620],{"class":289},[57,12901,3217],{"class":269},[57,12903,12904,12906,12908,12911,12913,12915],{"class":59,"line":5946},[57,12905,3239],{"class":280},[57,12907,10330],{"class":10193},[57,12909,12910],{"class":269},"(parts) ",[57,12912,3245],{"class":280},[57,12914,11796],{"class":273},[57,12916,3197],{"class":269},[57,12918,12919,12922,12924,12927,12929,12931],{"class":59,"line":5952},[57,12920,12921],{"class":269},"            source.append(parts[",[57,12923,560],{"class":273},[57,12925,12926],{"class":269},"].split(",[57,12928,11620],{"class":289},[57,12930,12654],{"class":289},[57,12932,12933],{"class":269},"))\n",[57,12935,12936,12939,12941,12943,12945,12947],{"class":59,"line":5967},[57,12937,12938],{"class":269},"            target.append(parts[",[57,12940,574],{"class":273},[57,12942,12926],{"class":269},[57,12944,11620],{"class":289},[57,12946,12654],{"class":289},[57,12948,12933],{"class":269},[57,12950,12951,12953],{"class":59,"line":10273},[57,12952,5407],{"class":280},[57,12954,12955],{"class":269}," source, target\n",[57,12957,12958],{"class":59,"line":10279},[57,12959,3448],{"emptyLinePlaceholder":963},[57,12961,12962,12964,12967,12969,12972,12974,12976,12978,12980,12982,12985],{"class":59,"line":10285},[57,12963,3150],{"class":280},[57,12965,12966],{"class":3153}," load_data_nmt",[57,12968,1754],{"class":269},[57,12970,12971],{"class":3159},"batch_size",[57,12973,3163],{"class":269},[57,12975,376],{"class":3159},[57,12977,3163],{"class":269},[57,12979,12808],{"class":3159},[57,12981,281],{"class":280},[57,12983,12984],{"class":273},"600",[57,12986,3169],{"class":269},[57,12988,12989],{"class":59,"line":10312},[57,12990,12991],{"class":3174},"    \"\"\"返回翻译数据集的迭代器和词表\n",[57,12993,12994],{"class":59,"line":10340},[57,12995,3448],{"emptyLinePlaceholder":963},[57,12997,12998],{"class":59,"line":10360},[57,12999,13000],{"class":3174},"    Defined in :numref:`subsec_mt_data_loading`\"\"\"\n",[57,13002,13003,13005,13007],{"class":59,"line":10371},[57,13004,12675],{"class":269},[57,13006,281],{"class":280},[57,13008,13009],{"class":269}," preprocess_nmt(read_data_nmt())\n",[57,13011,13012,13014,13016],{"class":59,"line":10384},[57,13013,12832],{"class":269},[57,13015,281],{"class":280},[57,13017,13018],{"class":269}," tokenize_nmt(text, num_examples)\n",[57,13020,13021,13024,13026,13029,13032,13034,13036],{"class":59,"line":10887},[57,13022,13023],{"class":269},"    src_vocab ",[57,13025,281],{"class":280},[57,13027,13028],{"class":269}," d2l.Vocab(source, ",[57,13030,13031],{"class":3208},"min_freq",[57,13033,281],{"class":280},[57,13035,590],{"class":273},[57,13037,369],{"class":269},[57,13039,13040,13043,13045,13047,13049,13052,13054,13056,13058,13060,13062,13064,13066,13068,13070],{"class":59,"line":10902},[57,13041,13042],{"class":3208},"                          reserved_tokens",[57,13044,281],{"class":280},[57,13046,12096],{"class":269},[57,13048,11620],{"class":289},[57,13050,13051],{"class":293},"\u003Cpad>",[57,13053,11620],{"class":289},[57,13055,3163],{"class":269},[57,13057,11620],{"class":289},[57,13059,11858],{"class":293},[57,13061,11620],{"class":289},[57,13063,3163],{"class":269},[57,13065,11620],{"class":289},[57,13067,11862],{"class":293},[57,13069,11620],{"class":289},[57,13071,3265],{"class":269},[57,13073,13074,13077,13079,13082,13084,13086,13088],{"class":59,"line":10907},[57,13075,13076],{"class":269},"    tgt_vocab ",[57,13078,281],{"class":280},[57,13080,13081],{"class":269}," d2l.Vocab(target, ",[57,13083,13031],{"class":3208},[57,13085,281],{"class":280},[57,13087,590],{"class":273},[57,13089,369],{"class":269},[57,13091,13092,13094,13096,13098,13100,13102,13104,13106,13108,13110,13112,13114,13116,13118,13120],{"class":59,"line":10922},[57,13093,13042],{"class":3208},[57,13095,281],{"class":280},[57,13097,12096],{"class":269},[57,13099,11620],{"class":289},[57,13101,13051],{"class":293},[57,13103,11620],{"class":289},[57,13105,3163],{"class":269},[57,13107,11620],{"class":289},[57,13109,11858],{"class":293},[57,13111,11620],{"class":289},[57,13113,3163],{"class":269},[57,13115,11620],{"class":289},[57,13117,11862],{"class":293},[57,13119,11620],{"class":289},[57,13121,3265],{"class":269},[57,13123,13124,13127,13129],{"class":59,"line":10951},[57,13125,13126],{"class":269},"    src_array, src_valid_len ",[57,13128,281],{"class":280},[57,13130,13131],{"class":269}," build_array_nmt(source, src_vocab, num_steps)\n",[57,13133,13134,13137,13139],{"class":59,"line":10970},[57,13135,13136],{"class":269},"    tgt_array, tgt_valid_len ",[57,13138,281],{"class":280},[57,13140,13141],{"class":269}," build_array_nmt(target, tgt_vocab, num_steps)\n",[57,13143,13144,13147,13149],{"class":59,"line":10991},[57,13145,13146],{"class":269},"    data_arrays ",[57,13148,281],{"class":280},[57,13150,13151],{"class":269}," (src_array, src_valid_len, tgt_array, tgt_valid_len)\n",[57,13153,13154,13157,13159],{"class":59,"line":11011},[57,13155,13156],{"class":269},"    data_iter ",[57,13158,281],{"class":280},[57,13160,13161],{"class":269}," d2l.load_array(data_arrays, batch_size)\n",[57,13163,13164,13166],{"class":59,"line":11022},[57,13165,5407],{"class":280},[57,13167,13168],{"class":269}," data_iter, src_vocab, tgt_vocab\n",[14,13170,13171],{},"完成数据处理后，按照 Transformer 架构搭建编码器-解码器模型并启动训练：",[47,13173,13175],{"className":256,"code":13174,"language":258,"meta":52,"style":52},"num_hiddens, num_layers, dropout, batch_size, num_steps = 32, 2, 0.1, 64, 10\nlr, num_epochs, device = 0.005, 200, torch.device('cuda:0') if torch.cuda.device_count() >= 1 else torch.device('cpu')\nffn_num_hiddens, num_heads = 64, 4\n\ntrain_iter, src_vocab, tgt_vocab = d2l.load_data_nmt(batch_size, num_steps)\n\nencoder = TransformerEncoder(\n    len(src_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n    dropout)\ndecoder = TransformerDecoder(\n    len(tgt_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n    dropout)\nnet = EncoderDecoder(encoder, decoder)\ntrain_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)\n",[54,13176,13177,13206,13258,13273,13277,13287,13291,13301,13309,13314,13324,13331,13335,13345],{"__ignoreMap":52},[57,13178,13179,13182,13184,13187,13189,13191,13193,13196,13198,13201,13203],{"class":59,"line":60},[57,13180,13181],{"class":269},"num_hiddens, num_layers, dropout, batch_size, num_steps ",[57,13183,281],{"class":280},[57,13185,13186],{"class":273}," 32",[57,13188,3163],{"class":269},[57,13190,590],{"class":273},[57,13192,3163],{"class":269},[57,13194,13195],{"class":273},"0.1",[57,13197,3163],{"class":269},[57,13199,13200],{"class":273},"64",[57,13202,3163],{"class":269},[57,13204,13205],{"class":273},"10\n",[57,13207,13208,13211,13213,13216,13218,13221,13224,13226,13229,13231,13233,13235,13238,13241,13243,13246,13249,13251,13254,13256],{"class":59,"line":66},[57,13209,13210],{"class":269},"lr, num_epochs, device ",[57,13212,281],{"class":280},[57,13214,13215],{"class":273}," 0.005",[57,13217,3163],{"class":269},[57,13219,13220],{"class":273},"200",[57,13222,13223],{"class":269},", torch.device(",[57,13225,11620],{"class":289},[57,13227,13228],{"class":293},"cuda:0",[57,13230,11620],{"class":289},[57,13232,7813],{"class":269},[57,13234,11757],{"class":280},[57,13236,13237],{"class":269}," torch.cuda.device_count() ",[57,13239,13240],{"class":280},">=",[57,13242,3248],{"class":273},[57,13244,13245],{"class":280}," else",[57,13247,13248],{"class":269}," torch.device(",[57,13250,11620],{"class":289},[57,13252,13253],{"class":293},"cpu",[57,13255,11620],{"class":289},[57,13257,3217],{"class":269},[57,13259,13260,13263,13265,13268,13270],{"class":59,"line":72},[57,13261,13262],{"class":269},"ffn_num_hiddens, num_heads ",[57,13264,281],{"class":280},[57,13266,13267],{"class":273}," 64",[57,13269,3163],{"class":269},[57,13271,13272],{"class":273},"4\n",[57,13274,13275],{"class":59,"line":78},[57,13276,3448],{"emptyLinePlaceholder":963},[57,13278,13279,13282,13284],{"class":59,"line":84},[57,13280,13281],{"class":269},"train_iter, src_vocab, tgt_vocab ",[57,13283,281],{"class":280},[57,13285,13286],{"class":269}," d2l.load_data_nmt(batch_size, num_steps)\n",[57,13288,13289],{"class":59,"line":90},[57,13290,3448],{"emptyLinePlaceholder":963},[57,13292,13293,13296,13298],{"class":59,"line":96},[57,13294,13295],{"class":269},"encoder ",[57,13297,281],{"class":280},[57,13299,13300],{"class":269}," TransformerEncoder(\n",[57,13302,13303,13306],{"class":59,"line":102},[57,13304,13305],{"class":10193},"    len",[57,13307,13308],{"class":269},"(src_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n",[57,13310,13311],{"class":59,"line":108},[57,13312,13313],{"class":269},"    dropout)\n",[57,13315,13316,13319,13321],{"class":59,"line":114},[57,13317,13318],{"class":269},"decoder ",[57,13320,281],{"class":280},[57,13322,13323],{"class":269}," TransformerDecoder(\n",[57,13325,13326,13328],{"class":59,"line":120},[57,13327,13305],{"class":10193},[57,13329,13330],{"class":269},"(tgt_vocab), num_hiddens, ffn_num_hiddens, num_heads, num_layers,\n",[57,13332,13333],{"class":59,"line":126},[57,13334,13313],{"class":269},[57,13336,13337,13340,13342],{"class":59,"line":132},[57,13338,13339],{"class":269},"net ",[57,13341,281],{"class":280},[57,13343,13344],{"class":269}," EncoderDecoder(encoder, decoder)\n",[57,13346,13347],{"class":59,"line":138},[57,13348,13349],{"class":269},"train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)\n",[10,13351,13352],{"id":13352},"参考文献",[14,13354,13355],{},[18,13356,993],{"href":991,"rel":13357},[22],[14,13359,13360],{},[18,13361,987],{"href":985,"rel":13362},[22],[936,13364,13365],{},"html pre.shiki code .szJfE, html code.shiki .szJfE{--shiki-default:#D73A49;--shiki-dark:#FF79C6}html pre.shiki code .sCLZk, html code.shiki .sCLZk{--shiki-default:#6F42C1;--shiki-dark:#50FA7B}html pre.shiki code .scbbO, html code.shiki .scbbO{--shiki-default:#24292E;--shiki-dark:#F8F8F2}html pre.shiki code .syNf4, html code.shiki .syNf4{--shiki-default:#24292E;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic}html pre.shiki code .seLWX, html code.shiki .seLWX{--shiki-default:#032F62;--shiki-dark:#6272A4}html pre.shiki code .sfgPZ, html code.shiki .sfgPZ{--shiki-default:#6A737D;--shiki-dark:#6272A4}html pre.shiki code .soDru, html code.shiki .soDru{--shiki-default:#005CC5;--shiki-dark:#BD93F9}html pre.shiki code .sQkXh, html code.shiki .sQkXh{--shiki-default:#E36209;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .skCyd, html code.shiki .skCyd{--shiki-default:#6F42C1;--shiki-dark:#8BE9FD}html pre.shiki code .sDP9b, html code.shiki .sDP9b{--shiki-default:#6F42C1;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .sD3jg, html code.shiki .sD3jg{--shiki-default:#24292E;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html pre.shiki code .sPGBF, html code.shiki .sPGBF{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .sJti5, html code.shiki .sJti5{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html pre.shiki code .sDoOe, html code.shiki .sDoOe{--shiki-default:#24292E;--shiki-dark:#FF79C6}html pre.shiki code .sDgm9, html code.shiki .sDgm9{--shiki-default:#005CC5;--shiki-dark:#8BE9FD}html pre.shiki code .sMWOi, html code.shiki .sMWOi{--shiki-default:#032F62;--shiki-dark:#E9F284}html pre.shiki code .sEzAm, html code.shiki .sEzAm{--shiki-default:#032F62;--shiki-dark:#F1FA8C}html pre.shiki code .sRfyP, html code.shiki .sRfyP{--shiki-default:#005CC5;--shiki-dark:#FF79C6}",{"title":52,"searchDepth":66,"depth":66,"links":13367},[13368,13369,13370,13374,13375,13376,13377,13378,13379,13380,13381],{"id":976,"depth":66,"text":976},{"id":996,"depth":66,"text":996},{"id":1006,"depth":66,"text":1006,"children":13371},[13372,13373],{"id":1009,"depth":72,"text":1009},{"id":3595,"depth":72,"text":3595},{"id":5977,"depth":66,"text":5977},{"id":7963,"depth":66,"text":7963},{"id":8467,"depth":66,"text":8467},{"id":8861,"depth":66,"text":8861},{"id":9776,"depth":66,"text":9777},{"id":10392,"depth":66,"text":10393},{"id":11399,"depth":66,"text":11399},{"id":13352,"depth":66,"text":13352},"距离首次学习 transformer 已经过去一年，内容忘的差不多了，决定复习一下。",{"date":13384,"image":13385,"alt":971,"tags":13386,"published":963},"30st Jan 2026","/blogs-img/blog1.jpg",[13387,961],"deep-learning","/blogs/transformer",{"title":971,"description":13382},"blogs/1. transformer复习笔记","1ycHTU2X8OkVcaQ6csyYCF8ZjnFmnQXik5F9B9P0AtI",{"id":13393,"title":13394,"body":13395,"description":46080,"extension":955,"meta":46081,"navigation":963,"ogImage":46083,"path":46087,"seo":46088,"stem":46089,"__hash__":46090},"content/blogs/2. PPO学习笔记.md","零基础速通 PPO 学习笔记",{"type":7,"value":13396,"toc":46067},[13397,13400,13403,13406,13409,13949,14251,14715,14721,14724,14799,15136,19129,19731,20250,20254,20416,21078,21444,21777,21780,22466,22469,23608,23781,23854,26485,26608,28479,28483,28486,28580,31361,31364,31472,31475,32962,33035,33039,33233,33236,33574,35238,35241,35244,35896,36087,40612,40615,41660,41981,41986,42051,43342,43346,43356,43906,44070,46054,46057,46060],[10,13398,13399],{"id":13399},"学习背景",[14,13401,13402],{},"最近套磁到了港科的一个 RA 岗位，组里在做 Agent 强化学习相关的研究，按照学长的建议先把 PPO 算法系统学习一遍，于是有了这篇笔记。",[10,13404,13405],{"id":13405},"符号定义",[14,13407,13408],{},"在强化学习中，常用符号约定如下：",[13410,13411,13412,13517,13619,13801,13903,13937,13943],"ul",{},[13413,13414,13415,727,13487,13516],"li",{},[57,13416,13418,13436],{"className":13417},[1019],[57,13419,13421],{"className":13420},[1023],[1025,13422,13423],{"xmlns":1027},[1029,13424,13425,13433],{},[1032,13426,13427],{},[1044,13428,13429,13431],{},[1047,13430,18],{},[1047,13432,1733],{},[1054,13434,13435],{"encoding":1056},"a_t",[57,13437,13439],{"className":13438,"ariaHidden":1062},[1061],[57,13440,13442,13446],{"className":13441},[1066],[57,13443],{"className":13444,"style":13445},[1070],"height:0.5806em;vertical-align:-0.15em;",[57,13447,13449,13452],{"className":13448},[1075],[57,13450,18],{"className":13451},[1075,1148],[57,13453,13455],{"className":13454},[1152],[57,13456,13458,13479],{"className":13457},[1087,1088],[57,13459,13461,13476],{"className":13460},[1092],[57,13462,13465],{"className":13463,"style":13464},[1096],"height:0.2806em;",[57,13466,13467,13470],{"style":2211},[57,13468],{"className":13469,"style":1493},[1104],[57,13471,13473],{"className":13472},[1109,1110,1111,1112],[57,13474,1733],{"className":13475},[1075,1148,1112],[57,13477,1183],{"className":13478},[1182],[57,13480,13482],{"className":13481},[1092],[57,13483,13485],{"className":13484,"style":2230},[1096],[57,13486],{},[57,13488,13490,13503],{"className":13489},[1019],[57,13491,13493],{"className":13492},[1023],[1025,13494,13495],{"xmlns":1027},[1029,13496,13497,13501],{},[1032,13498,13499],{},[1047,13500,1733],{},[1054,13502,1733],{"encoding":1056},[57,13504,13506],{"className":13505,"ariaHidden":1062},[1061],[57,13507,13509,13513],{"className":13508},[1066],[57,13510],{"className":13511,"style":13512},[1070],"height:0.6151em;",[57,13514,1733],{"className":13515},[1075,1148]," 时刻 Agent 采取的动作（action）；",[13413,13518,13519,727,13590,13618],{},[57,13520,13522,13541],{"className":13521},[1019],[57,13523,13525],{"className":13524},[1023],[1025,13526,13527],{"xmlns":1027},[1029,13528,13529,13538],{},[1032,13530,13531],{},[1044,13532,13533,13536],{},[1047,13534,13535],{},"s",[1047,13537,1733],{},[1054,13539,13540],{"encoding":1056},"s_t",[57,13542,13544],{"className":13543,"ariaHidden":1062},[1061],[57,13545,13547,13550],{"className":13546},[1066],[57,13548],{"className":13549,"style":13445},[1070],[57,13551,13553,13556],{"className":13552},[1075],[57,13554,13535],{"className":13555},[1075,1148],[57,13557,13559],{"className":13558},[1152],[57,13560,13562,13582],{"className":13561},[1087,1088],[57,13563,13565,13579],{"className":13564},[1092],[57,13566,13568],{"className":13567,"style":13464},[1096],[57,13569,13570,13573],{"style":2211},[57,13571],{"className":13572,"style":1493},[1104],[57,13574,13576],{"className":13575},[1109,1110,1111,1112],[57,13577,1733],{"className":13578},[1075,1148,1112],[57,13580,1183],{"className":13581},[1182],[57,13583,13585],{"className":13584},[1092],[57,13586,13588],{"className":13587,"style":2230},[1096],[57,13589],{},[57,13591,13593,13606],{"className":13592},[1019],[57,13594,13596],{"className":13595},[1023],[1025,13597,13598],{"xmlns":1027},[1029,13599,13600,13604],{},[1032,13601,13602],{},[1047,13603,1733],{},[1054,13605,1733],{"encoding":1056},[57,13607,13609],{"className":13608,"ariaHidden":1062},[1061],[57,13610,13612,13615],{"className":13611},[1066],[57,13613],{"className":13614,"style":13512},[1070],[57,13616,1733],{"className":13617},[1075,1148]," 时刻 Agent 所处的状态（state）；",[13413,13620,13621,13650,13651,13800],{},[57,13622,13624,13638],{"className":13623},[1019],[57,13625,13627],{"className":13626},[1023],[1025,13628,13629],{"xmlns":1027},[1029,13630,13631,13635],{},[1032,13632,13633],{},[1047,13634,7412],{},[1054,13636,13637],{"encoding":1056},"\\pi",[57,13639,13641],{"className":13640,"ariaHidden":1062},[1061],[57,13642,13644,13647],{"className":13643},[1066],[57,13645],{"className":13646,"style":1300},[1070],[57,13648,7412],{"className":13649,"style":1394},[1075,1148],"：Agent 的策略函数（policy），输入状态，输出每个动作的概率分布 ",[57,13652,13654,13687],{"className":13653},[1019],[57,13655,13657],{"className":13656},[1023],[1025,13658,13659],{"xmlns":1027},[1029,13660,13661,13684],{},[1032,13662,13663,13665,13667,13673,13676,13682],{},[1047,13664,7412],{},[1414,13666,1754],{"stretchy":1753},[1044,13668,13669,13671],{},[1047,13670,18],{},[1047,13672,1733],{},[1414,13674,13675],{},"∣",[1044,13677,13678,13680],{},[1047,13679,13535],{},[1047,13681,1733],{},[1414,13683,1768],{"stretchy":1753},[1054,13685,13686],{"encoding":1056},"\\pi(a_t \\mid s_t)",[57,13688,13690,13751],{"className":13689,"ariaHidden":1062},[1061],[57,13691,13693,13696,13699,13702,13742,13745,13748],{"className":13692},[1066],[57,13694],{"className":13695,"style":1831},[1070],[57,13697,7412],{"className":13698,"style":1394},[1075,1148],[57,13700,1754],{"className":13701},[1079],[57,13703,13705,13708],{"className":13704},[1075],[57,13706,18],{"className":13707},[1075,1148],[57,13709,13711],{"className":13710},[1152],[57,13712,13714,13734],{"className":13713},[1087,1088],[57,13715,13717,13731],{"className":13716},[1092],[57,13718,13720],{"className":13719,"style":13464},[1096],[57,13721,13722,13725],{"style":2211},[57,13723],{"className":13724,"style":1493},[1104],[57,13726,13728],{"className":13727},[1109,1110,1111,1112],[57,13729,1733],{"className":13730},[1075,1148,1112],[57,13732,1183],{"className":13733},[1182],[57,13735,13737],{"className":13736},[1092],[57,13738,13740],{"className":13739,"style":2230},[1096],[57,13741],{},[57,13743],{"className":13744,"style":1453},[1452],[57,13746,13675],{"className":13747},[1457],[57,13749],{"className":13750,"style":1453},[1452],[57,13752,13754,13757,13797],{"className":13753},[1066],[57,13755],{"className":13756,"style":1831},[1070],[57,13758,13760,13763],{"className":13759},[1075],[57,13761,13535],{"className":13762},[1075,1148],[57,13764,13766],{"className":13765},[1152],[57,13767,13769,13789],{"className":13768},[1087,1088],[57,13770,13772,13786],{"className":13771},[1092],[57,13773,13775],{"className":13774,"style":13464},[1096],[57,13776,13777,13780],{"style":2211},[57,13778],{"className":13779,"style":1493},[1104],[57,13781,13783],{"className":13782},[1109,1110,1111,1112],[57,13784,1733],{"className":13785},[1075,1148,1112],[57,13787,1183],{"className":13788},[1182],[57,13790,13792],{"className":13791},[1092],[57,13793,13795],{"className":13794,"style":2230},[1096],[57,13796],{},[57,13798,1768],{"className":13799},[1269],"；",[13413,13802,13803,727,13874,13902],{},[57,13804,13806,13824],{"className":13805},[1019],[57,13807,13809],{"className":13808},[1023],[1025,13810,13811],{"xmlns":1027},[1029,13812,13813,13821],{},[1032,13814,13815],{},[1044,13816,13817,13819],{},[1047,13818,8500],{},[1047,13820,1733],{},[1054,13822,13823],{"encoding":1056},"r_t",[57,13825,13827],{"className":13826,"ariaHidden":1062},[1061],[57,13828,13830,13833],{"className":13829},[1066],[57,13831],{"className":13832,"style":13445},[1070],[57,13834,13836,13839],{"className":13835},[1075],[57,13837,8500],{"className":13838,"style":3932},[1075,1148],[57,13840,13842],{"className":13841},[1152],[57,13843,13845,13866],{"className":13844},[1087,1088],[57,13846,13848,13863],{"className":13847},[1092],[57,13849,13851],{"className":13850,"style":13464},[1096],[57,13852,13854,13857],{"style":13853},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[57,13855],{"className":13856,"style":1493},[1104],[57,13858,13860],{"className":13859},[1109,1110,1111,1112],[57,13861,1733],{"className":13862},[1075,1148,1112],[57,13864,1183],{"className":13865},[1182],[57,13867,13869],{"className":13868},[1092],[57,13870,13872],{"className":13871,"style":2230},[1096],[57,13873],{},[57,13875,13877,13890],{"className":13876},[1019],[57,13878,13880],{"className":13879},[1023],[1025,13881,13882],{"xmlns":1027},[1029,13883,13884,13888],{},[1032,13885,13886],{},[1047,13887,1733],{},[1054,13889,1733],{"encoding":1056},[57,13891,13893],{"className":13892,"ariaHidden":1062},[1061],[57,13894,13896,13899],{"className":13895},[1066],[57,13897],{"className":13898,"style":13512},[1070],[57,13900,1733],{"className":13901},[1075,1148]," 时刻 Agent 采取动作后获得的奖励（reward）；",[13413,13904,13905,13936],{},[57,13906,13908,13923],{"className":13907},[1019],[57,13909,13911],{"className":13910},[1023],[1025,13912,13913],{"xmlns":1027},[1029,13914,13915,13920],{},[1032,13916,13917],{},[1047,13918,13919],{},"τ",[1054,13921,13922],{"encoding":1056},"\\tau",[57,13924,13926],{"className":13925,"ariaHidden":1062},[1061],[57,13927,13929,13932],{"className":13928},[1066],[57,13930],{"className":13931,"style":1300},[1070],[57,13933,13919],{"className":13934,"style":13935},[1075,1148],"margin-right:0.1132em;","：一条轨迹（trajectory），即一段状态-动作序列；",[13413,13938,13939,13942],{},[11865,13940,13941],{},"Episode","：一次完整的交互过程，从环境初始化开始，到达到终止状态为止；",[13413,13944,13945,13948],{},[11865,13946,13947],{},"Rollout","：按当前策略实际\"跑出来\"的一段轨迹数据，不一定是完整的一局，也可以只是其中的一段。",[14,13950,13951],{},[57,13952,13954,14008],{"className":13953},[1019],[57,13955,13957],{"className":13956},[1023],[1025,13958,13959],{"xmlns":1027},[1029,13960,13961,14005],{},[1032,13962,13963,13965,13971,13973,13979,13981,13987,13989,13995,13997,14003],{},[1414,13964,1754],{"stretchy":1753},[1044,13966,13967,13969],{},[1047,13968,13535],{},[1038,13970,560],{},[1414,13972,1759],{"separator":1062},[1044,13974,13975,13977],{},[1047,13976,18],{},[1038,13978,560],{},[1414,13980,1759],{"separator":1062},[1044,13982,13983,13985],{},[1047,13984,13535],{},[1038,13986,574],{},[1414,13988,1759],{"separator":1062},[1044,13990,13991,13993],{},[1047,13992,18],{},[1038,13994,574],{},[1414,13996,3693],{},[1044,13998,13999,14001],{},[1047,14000,13535],{},[1047,14002,738],{},[1414,14004,1768],{"stretchy":1753},[1054,14006,14007],{"encoding":1056},"(s_0,a_0,s_1,a_1 \\dots s_T)",[57,14009,14011],{"className":14010,"ariaHidden":1062},[1061],[57,14012,14014,14017,14020,14060,14063,14066,14106,14109,14112,14152,14155,14158,14198,14201,14204,14207,14248],{"className":14013},[1066],[57,14015],{"className":14016,"style":1831},[1070],[57,14018,1754],{"className":14019},[1079],[57,14021,14023,14026],{"className":14022},[1075],[57,14024,13535],{"className":14025},[1075,1148],[57,14027,14029],{"className":14028},[1152],[57,14030,14032,14052],{"className":14031},[1087,1088],[57,14033,14035,14049],{"className":14034},[1092],[57,14036,14038],{"className":14037,"style":3813},[1096],[57,14039,14040,14043],{"style":2211},[57,14041],{"className":14042,"style":1493},[1104],[57,14044,14046],{"className":14045},[1109,1110,1111,1112],[57,14047,560],{"className":14048},[1075,1112],[57,14050,1183],{"className":14051},[1182],[57,14053,14055],{"className":14054},[1092],[57,14056,14058],{"className":14057,"style":2230},[1096],[57,14059],{},[57,14061,1759],{"className":14062},[1849],[57,14064],{"className":14065,"style":1853},[1452],[57,14067,14069,14072],{"className":14068},[1075],[57,14070,18],{"className":14071},[1075,1148],[57,14073,14075],{"className":14074},[1152],[57,14076,14078,14098],{"className":14077},[1087,1088],[57,14079,14081,14095],{"className":14080},[1092],[57,14082,14084],{"className":14083,"style":3813},[1096],[57,14085,14086,14089],{"style":2211},[57,14087],{"className":14088,"style":1493},[1104],[57,14090,14092],{"className":14091},[1109,1110,1111,1112],[57,14093,560],{"className":14094},[1075,1112],[57,14096,1183],{"className":14097},[1182],[57,14099,14101],{"className":14100},[1092],[57,14102,14104],{"className":14103,"style":2230},[1096],[57,14105],{},[57,14107,1759],{"className":14108},[1849],[57,14110],{"className":14111,"style":1853},[1452],[57,14113,14115,14118],{"className":14114},[1075],[57,14116,13535],{"className":14117},[1075,1148],[57,14119,14121],{"className":14120},[1152],[57,14122,14124,14144],{"className":14123},[1087,1088],[57,14125,14127,14141],{"className":14126},[1092],[57,14128,14130],{"className":14129,"style":3813},[1096],[57,14131,14132,14135],{"style":2211},[57,14133],{"className":14134,"style":1493},[1104],[57,14136,14138],{"className":14137},[1109,1110,1111,1112],[57,14139,574],{"className":14140},[1075,1112],[57,14142,1183],{"className":14143},[1182],[57,14145,14147],{"className":14146},[1092],[57,14148,14150],{"className":14149,"style":2230},[1096],[57,14151],{},[57,14153,1759],{"className":14154},[1849],[57,14156],{"className":14157,"style":1853},[1452],[57,14159,14161,14164],{"className":14160},[1075],[57,14162,18],{"className":14163},[1075,1148],[57,14165,14167],{"className":14166},[1152],[57,14168,14170,14190],{"className":14169},[1087,1088],[57,14171,14173,14187],{"className":14172},[1092],[57,14174,14176],{"className":14175,"style":3813},[1096],[57,14177,14178,14181],{"style":2211},[57,14179],{"className":14180,"style":1493},[1104],[57,14182,14184],{"className":14183},[1109,1110,1111,1112],[57,14185,574],{"className":14186},[1075,1112],[57,14188,1183],{"className":14189},[1182],[57,14191,14193],{"className":14192},[1092],[57,14194,14196],{"className":14195,"style":2230},[1096],[57,14197],{},[57,14199],{"className":14200,"style":1853},[1452],[57,14202,3693],{"className":14203},[1898],[57,14205],{"className":14206,"style":1853},[1452],[57,14208,14210,14213],{"className":14209},[1075],[57,14211,13535],{"className":14212},[1075,1148],[57,14214,14216],{"className":14215},[1152],[57,14217,14219,14240],{"className":14218},[1087,1088],[57,14220,14222,14237],{"className":14221},[1092],[57,14223,14226],{"className":14224,"style":14225},[1096],"height:0.3283em;",[57,14227,14228,14231],{"style":2211},[57,14229],{"className":14230,"style":1493},[1104],[57,14232,14234],{"className":14233},[1109,1110,1111,1112],[57,14235,738],{"className":14236,"style":3907},[1075,1148,1112],[57,14238,1183],{"className":14239},[1182],[57,14241,14243],{"className":14242},[1092],[57,14244,14246],{"className":14245,"style":2230},[1096],[57,14247],{},[57,14249,1768],{"className":14250},[1269],[14,14252,14253,14254,14473,14474,14714],{},"环境的状态转移满足 ",[57,14255,14257,14303],{"className":14256},[1019],[57,14258,14260],{"className":14259},[1023],[1025,14261,14262],{"xmlns":1027},[1029,14263,14264,14300],{},[1032,14265,14266,14278,14280,14282,14284,14290,14292,14298],{},[1044,14267,14268,14270],{},[1047,14269,13535],{},[1032,14271,14272,14274,14276],{},[1047,14273,1733],{},[1414,14275,6487],{},[1038,14277,574],{},[1414,14279,281],{},[1047,14281,12389],{},[1414,14283,1754],{"stretchy":1753},[1044,14285,14286,14288],{},[1047,14287,13535],{},[1047,14289,1733],{},[1414,14291,1759],{"separator":1062},[1044,14293,14294,14296],{},[1047,14295,18],{},[1047,14297,1733],{},[1414,14299,1768],{"stretchy":1753},[1054,14301,14302],{"encoding":1056},"s_{t+1} = f(s_t,a_t)",[57,14304,14306,14371],{"className":14305,"ariaHidden":1062},[1061],[57,14307,14309,14313,14362,14365,14368],{"className":14308},[1066],[57,14310],{"className":14311,"style":14312},[1070],"height:0.6389em;vertical-align:-0.2083em;",[57,14314,14316,14319],{"className":14315},[1075],[57,14317,13535],{"className":14318},[1075,1148],[57,14320,14322],{"className":14321},[1152],[57,14323,14325,14354],{"className":14324},[1087,1088],[57,14326,14328,14351],{"className":14327},[1092],[57,14329,14331],{"className":14330,"style":3813},[1096],[57,14332,14333,14336],{"style":2211},[57,14334],{"className":14335,"style":1493},[1104],[57,14337,14339],{"className":14338},[1109,1110,1111,1112],[57,14340,14342,14345,14348],{"className":14341},[1075,1112],[57,14343,1733],{"className":14344},[1075,1148,1112],[57,14346,6487],{"className":14347},[1506,1112],[57,14349,574],{"className":14350},[1075,1112],[57,14352,1183],{"className":14353},[1182],[57,14355,14357],{"className":14356},[1092],[57,14358,14360],{"className":14359,"style":7608},[1096],[57,14361],{},[57,14363],{"className":14364,"style":1453},[1452],[57,14366,281],{"className":14367},[1457],[57,14369],{"className":14370,"style":1453},[1452],[57,14372,14374,14377,14381,14384,14424,14427,14430,14470],{"className":14373},[1066],[57,14375],{"className":14376,"style":1831},[1070],[57,14378,12389],{"className":14379,"style":14380},[1075,1148],"margin-right:0.10764em;",[57,14382,1754],{"className":14383},[1079],[57,14385,14387,14390],{"className":14386},[1075],[57,14388,13535],{"className":14389},[1075,1148],[57,14391,14393],{"className":14392},[1152],[57,14394,14396,14416],{"className":14395},[1087,1088],[57,14397,14399,14413],{"className":14398},[1092],[57,14400,14402],{"className":14401,"style":13464},[1096],[57,14403,14404,14407],{"style":2211},[57,14405],{"className":14406,"style":1493},[1104],[57,14408,14410],{"className":14409},[1109,1110,1111,1112],[57,14411,1733],{"className":14412},[1075,1148,1112],[57,14414,1183],{"className":14415},[1182],[57,14417,14419],{"className":14418},[1092],[57,14420,14422],{"className":14421,"style":2230},[1096],[57,14423],{},[57,14425,1759],{"className":14426},[1849],[57,14428],{"className":14429,"style":1853},[1452],[57,14431,14433,14436],{"className":14432},[1075],[57,14434,18],{"className":14435},[1075,1148],[57,14437,14439],{"className":14438},[1152],[57,14440,14442,14462],{"className":14441},[1087,1088],[57,14443,14445,14459],{"className":14444},[1092],[57,14446,14448],{"className":14447,"style":13464},[1096],[57,14449,14450,14453],{"style":2211},[57,14451],{"className":14452,"style":1493},[1104],[57,14454,14456],{"className":14455},[1109,1110,1111,1112],[57,14457,1733],{"className":14458},[1075,1148,1112],[57,14460,1183],{"className":14461},[1182],[57,14463,14465],{"className":14464},[1092],[57,14466,14468],{"className":14467,"style":2230},[1096],[57,14469],{},[57,14471,1768],{"className":14472},[1269],"（确定性环境）或 ",[57,14475,14477,14528],{"className":14476},[1019],[57,14478,14480],{"className":14479},[1023],[1025,14481,14482],{"xmlns":1027},[1029,14483,14484,14525],{},[1032,14485,14486,14498,14501,14503,14505,14507,14509,14515,14517,14523],{},[1044,14487,14488,14490],{},[1047,14489,13535],{},[1032,14491,14492,14494,14496],{},[1047,14493,1733],{},[1414,14495,6487],{},[1038,14497,574],{},[1414,14499,14500],{},"∼",[1047,14502,6333],{},[1414,14504,1754],{"stretchy":1753},[1414,14506,2539],{},[1414,14508,13675],{},[1044,14510,14511,14513],{},[1047,14512,13535],{},[1047,14514,1733],{},[1414,14516,1759],{"separator":1062},[1044,14518,14519,14521],{},[1047,14520,18],{},[1047,14522,1733],{},[1414,14524,1768],{"stretchy":1753},[1054,14526,14527],{"encoding":1056},"s_{t+1} \\sim P(\\cdot \\mid s_t,a_t)",[57,14529,14531,14595,14619],{"className":14530,"ariaHidden":1062},[1061],[57,14532,14534,14537,14586,14589,14592],{"className":14533},[1066],[57,14535],{"className":14536,"style":14312},[1070],[57,14538,14540,14543],{"className":14539},[1075],[57,14541,13535],{"className":14542},[1075,1148],[57,14544,14546],{"className":14545},[1152],[57,14547,14549,14578],{"className":14548},[1087,1088],[57,14550,14552,14575],{"className":14551},[1092],[57,14553,14555],{"className":14554,"style":3813},[1096],[57,14556,14557,14560],{"style":2211},[57,14558],{"className":14559,"style":1493},[1104],[57,14561,14563],{"className":14562},[1109,1110,1111,1112],[57,14564,14566,14569,14572],{"className":14565},[1075,1112],[57,14567,1733],{"className":14568},[1075,1148,1112],[57,14570,6487],{"className":14571},[1506,1112],[57,14573,574],{"className":14574},[1075,1112],[57,14576,1183],{"className":14577},[1182],[57,14579,14581],{"className":14580},[1092],[57,14582,14584],{"className":14583,"style":7608},[1096],[57,14585],{},[57,14587],{"className":14588,"style":1453},[1452],[57,14590,14500],{"className":14591},[1457],[57,14593],{"className":14594,"style":1453},[1452],[57,14596,14598,14601,14604,14607,14610,14613,14616],{"className":14597},[1066],[57,14599],{"className":14600,"style":1831},[1070],[57,14602,6333],{"className":14603,"style":3907},[1075,1148],[57,14605,1754],{"className":14606},[1079],[57,14608,2539],{"className":14609},[1075],[57,14611],{"className":14612,"style":1453},[1452],[57,14614,13675],{"className":14615},[1457],[57,14617],{"className":14618,"style":1453},[1452],[57,14620,14622,14625,14665,14668,14671,14711],{"className":14621},[1066],[57,14623],{"className":14624,"style":1831},[1070],[57,14626,14628,14631],{"className":14627},[1075],[57,14629,13535],{"className":14630},[1075,1148],[57,14632,14634],{"className":14633},[1152],[57,14635,14637,14657],{"className":14636},[1087,1088],[57,14638,14640,14654],{"className":14639},[1092],[57,14641,14643],{"className":14642,"style":13464},[1096],[57,14644,14645,14648],{"style":2211},[57,14646],{"className":14647,"style":1493},[1104],[57,14649,14651],{"className":14650},[1109,1110,1111,1112],[57,14652,1733],{"className":14653},[1075,1148,1112],[57,14655,1183],{"className":14656},[1182],[57,14658,14660],{"className":14659},[1092],[57,14661,14663],{"className":14662,"style":2230},[1096],[57,14664],{},[57,14666,1759],{"className":14667},[1849],[57,14669],{"className":14670,"style":1853},[1452],[57,14672,14674,14677],{"className":14673},[1075],[57,14675,18],{"className":14676},[1075,1148],[57,14678,14680],{"className":14679},[1152],[57,14681,14683,14703],{"className":14682},[1087,1088],[57,14684,14686,14700],{"className":14685},[1092],[57,14687,14689],{"className":14688,"style":13464},[1096],[57,14690,14691,14694],{"style":2211},[57,14692],{"className":14693,"style":1493},[1104],[57,14695,14697],{"className":14696},[1109,1110,1111,1112],[57,14698,1733],{"className":14699},[1075,1148,1112],[57,14701,1183],{"className":14702},[1182],[57,14704,14706],{"className":14705},[1092],[57,14707,14709],{"className":14708,"style":2230},[1096],[57,14710],{},[57,14712,1768],{"className":14713},[1269],"（随机性环境）。",[14,14716,14717,14720],{},[11865,14718,14719],{},"Return","（回报）指从当前时间步到 episode 结束所获得奖励的累积和（或带折扣的累积和）。",[10,14722,14723],{"id":14723},"训练目标",[14,14725,14726,14727,14798],{},"我们要训练一个策略网络 ",[57,14728,14730,14749],{"className":14729},[1019],[57,14731,14733],{"className":14732},[1023],[1025,14734,14735],{"xmlns":1027},[1029,14736,14737,14746],{},[1032,14738,14739],{},[1044,14740,14741,14743],{},[1047,14742,7412],{},[1047,14744,14745],{},"θ",[1054,14747,14748],{"encoding":1056},"\\pi_\\theta",[57,14750,14752],{"className":14751,"ariaHidden":1062},[1061],[57,14753,14755,14758],{"className":14754},[1066],[57,14756],{"className":14757,"style":13445},[1070],[57,14759,14761,14764],{"className":14760},[1075],[57,14762,7412],{"className":14763,"style":1394},[1075,1148],[57,14765,14767],{"className":14766},[1152],[57,14768,14770,14790],{"className":14769},[1087,1088],[57,14771,14773,14787],{"className":14772},[1092],[57,14774,14776],{"className":14775,"style":2208},[1096],[57,14777,14778,14781],{"style":2758},[57,14779],{"className":14780,"style":1493},[1104],[57,14782,14784],{"className":14783},[1109,1110,1111,1112],[57,14785,14745],{"className":14786,"style":3932},[1075,1148,1112],[57,14788,1183],{"className":14789},[1182],[57,14791,14793],{"className":14792},[1092],[57,14794,14796],{"className":14795,"style":2230},[1096],[57,14797],{},"，使其在所有可能的状态下做出动作后，期望回报最大化：",[14,14800,14801],{},[57,14802,14804,14879],{"className":14803},[1019],[57,14805,14807],{"className":14806},[1023],[1025,14808,14809],{"xmlns":1027},[1029,14810,14811,14876],{},[1032,14812,14813,14815,14817,14819,14821,14823,14825,14847,14849,14856,14858,14860,14862,14864,14870,14872,14874],{},[1047,14814,6336],{},[1414,14816,1754],{"stretchy":1753},[1047,14818,1423],{},[1414,14820,1754],{"stretchy":1753},[1047,14822,13919],{},[1414,14824,1768],{"stretchy":1753},[1044,14826,14827,14829],{},[1414,14828,1768],{"stretchy":1753},[1032,14830,14831,14833,14835,14841,14843,14845],{},[1047,14832,13919],{},[1414,14834,14500],{},[1044,14836,14837,14839],{},[1047,14838,6333],{},[1047,14840,14745],{},[1414,14842,1754],{"stretchy":1753},[1047,14844,13919],{},[1414,14846,1768],{"stretchy":1753},[1414,14848,281],{},[14850,14851,14852,14854],"munder",{},[1414,14853,2549],{},[1047,14855,13919],{},[1047,14857,1423],{},[1414,14859,1754],{"stretchy":1753},[1047,14861,13919],{},[1414,14863,1768],{"stretchy":1753},[1044,14865,14866,14868],{},[1047,14867,6333],{},[1047,14869,14745],{},[1414,14871,1754],{"stretchy":1753},[1047,14873,13919],{},[1414,14875,1768],{"stretchy":1753},[1054,14877,14878],{"encoding":1056}," E (R(\\tau))_{\\tau \\sim P_{\\theta}(\\tau)} = \\sum\\limits_\\tau R(\\tau)P_\\theta(\\tau)",[57,14880,14882,15017],{"className":14881,"ariaHidden":1062},[1061],[57,14883,14885,14889,14892,14895,14899,14902,14905,14908,15008,15011,15014],{"className":14884},[1066],[57,14886],{"className":14887,"style":14888},[1070],"height:1.1052em;vertical-align:-0.3552em;",[57,14890,6336],{"className":14891,"style":6749},[1075,1148],[57,14893,1754],{"className":14894},[1079],[57,14896,1423],{"className":14897,"style":14898},[1075,1148],"margin-right:0.00773em;",[57,14900,1754],{"className":14901},[1079],[57,14903,13919],{"className":14904,"style":13935},[1075,1148],[57,14906,1768],{"className":14907},[1269],[57,14909,14911,14914],{"className":14910},[1269],[57,14912,1768],{"className":14913},[1269],[57,14915,14917],{"className":14916},[1152],[57,14918,14920,15000],{"className":14919},[1087,1088],[57,14921,14923,14997],{"className":14922},[1092],[57,14924,14926],{"className":14925,"style":1162},[1096],[57,14927,14929,14932],{"style":14928},"top:-2.5198em;margin-left:0em;margin-right:0.05em;",[57,14930],{"className":14931,"style":1493},[1104],[57,14933,14935],{"className":14934},[1109,1110,1111,1112],[57,14936,14938,14941,14944,14988,14991,14994],{"className":14937},[1075,1112],[57,14939,13919],{"className":14940,"style":13935},[1075,1148,1112],[57,14942,14500],{"className":14943},[1457,1112],[57,14945,14947,14950],{"className":14946},[1075,1112],[57,14948,6333],{"className":14949,"style":3907},[1075,1148,1112],[57,14951,14953],{"className":14952},[1152],[57,14954,14956,14980],{"className":14955},[1087,1088],[57,14957,14959,14977],{"className":14958},[1092],[57,14960,14962],{"className":14961,"style":1162},[1096],[57,14963,14965,14968],{"style":14964},"top:-2.3488em;margin-left:-0.1389em;margin-right:0.0714em;",[57,14966],{"className":14967,"style":1169},[1104],[57,14969,14971],{"className":14970},[1109,1173,1174,1112],[57,14972,14974],{"className":14973},[1075,1112],[57,14975,14745],{"className":14976,"style":3932},[1075,1148,1112],[57,14978,1183],{"className":14979},[1182],[57,14981,14983],{"className":14982},[1092],[57,14984,14986],{"className":14985,"style":1190},[1096],[57,14987],{},[57,14989,1754],{"className":14990},[1079,1112],[57,14992,13919],{"className":14993,"style":13935},[1075,1148,1112],[57,14995,1768],{"className":14996},[1269,1112],[57,14998,1183],{"className":14999},[1182],[57,15001,15003],{"className":15002},[1092],[57,15004,15006],{"className":15005,"style":6801},[1096],[57,15007],{},[57,15009],{"className":15010,"style":1453},[1452],[57,15012,281],{"className":15013},[1457],[57,15015],{"className":15016,"style":1453},[1452],[57,15018,15020,15024,15072,15075,15078,15081,15084,15087,15127,15130,15133],{"className":15019},[1066],[57,15021],{"className":15022,"style":15023},[1070],"height:1.7em;vertical-align:-0.95em;",[57,15025,15028],{"className":15026},[1888,15027],"op-limits",[57,15029,15031,15063],{"className":15030},[1087,1088],[57,15032,15034,15060],{"className":15033},[1092],[57,15035,15038,15050],{"className":15036,"style":15037},[1096],"height:0.75em;",[57,15039,15041,15044],{"style":15040},"top:-2.15em;margin-left:0em;",[57,15042],{"className":15043,"style":1105},[1104],[57,15045,15047],{"className":15046},[1109,1110,1111,1112],[57,15048,13919],{"className":15049,"style":13935},[1075,1148,1112],[57,15051,15052,15055],{"style":1134},[57,15053],{"className":15054,"style":1105},[1104],[57,15056,15057],{},[57,15058,2549],{"className":15059},[1888,2633,2634],[57,15061,1183],{"className":15062},[1182],[57,15064,15066],{"className":15065},[1092],[57,15067,15070],{"className":15068,"style":15069},[1096],"height:0.95em;",[57,15071],{},[57,15073],{"className":15074,"style":1853},[1452],[57,15076,1423],{"className":15077,"style":14898},[1075,1148],[57,15079,1754],{"className":15080},[1079],[57,15082,13919],{"className":15083,"style":13935},[1075,1148],[57,15085,1768],{"className":15086},[1269],[57,15088,15090,15093],{"className":15089},[1075],[57,15091,6333],{"className":15092,"style":3907},[1075,1148],[57,15094,15096],{"className":15095},[1152],[57,15097,15099,15119],{"className":15098},[1087,1088],[57,15100,15102,15116],{"className":15101},[1092],[57,15103,15105],{"className":15104,"style":2208},[1096],[57,15106,15107,15110],{"style":8131},[57,15108],{"className":15109,"style":1493},[1104],[57,15111,15113],{"className":15112},[1109,1110,1111,1112],[57,15114,14745],{"className":15115,"style":3932},[1075,1148,1112],[57,15117,1183],{"className":15118},[1182],[57,15120,15122],{"className":15121},[1092],[57,15123,15125],{"className":15124,"style":2230},[1096],[57,15126],{},[57,15128,1754],{"className":15129},[1079],[57,15131,13919],{"className":15132,"style":13935},[1075,1148],[57,15134,1768],{"className":15135},[1269],[57,15137,15140],{"className":15138},[15139],"katex-display",[57,15141,15143,15923],{"className":15142},[1019],[57,15144,15146],{"className":15145},[1023],[1025,15147,15148],{"xmlns":1027,"display":10210},[1029,15149,15150,15920],{},[15151,15152,15156,15242,15288,15368,15446,15540,15613,15716,15818],"mtable",{"rowspacing":15153,"columnalign":15154,"columnspacing":15155},"0.25em","right left","0em",[15157,15158,15159,15204],"mtr",{},[15160,15161,15162],"mtd",{},[15163,15164,15165],"mstyle",{"scriptlevel":560,"displaystyle":1062},[1032,15166,15167,15170,15172,15174,15176,15178,15180,15182],{},[1047,15168,15169],{"mathvariant":1729},"∇",[1047,15171,6336],{},[1414,15173,1754],{"stretchy":1753},[1047,15175,1423],{},[1414,15177,1754],{"stretchy":1753},[1047,15179,13919],{},[1414,15181,1768],{"stretchy":1753},[1044,15183,15184,15186],{},[1414,15185,1768],{"stretchy":1753},[1032,15187,15188,15190,15192,15198,15200,15202],{},[1047,15189,13919],{},[1414,15191,14500],{},[1044,15193,15194,15196],{},[1047,15195,6333],{},[1047,15197,14745],{},[1414,15199,1754],{"stretchy":1753},[1047,15201,13919],{},[1414,15203,1768],{"stretchy":1753},[15160,15205,15206],{},[15163,15207,15208],{"scriptlevel":560,"displaystyle":1062},[1032,15209,15210,15212,15214,15216,15222,15224,15226,15228,15230,15236,15238,15240],{},[1032,15211],{},[1414,15213,281],{},[1047,15215,15169],{"mathvariant":1729},[14850,15217,15218,15220],{},[1414,15219,2549],{},[1047,15221,13919],{},[1047,15223,1423],{},[1414,15225,1754],{"stretchy":1753},[1047,15227,13919],{},[1414,15229,1768],{"stretchy":1753},[1044,15231,15232,15234],{},[1047,15233,6333],{},[1047,15235,14745],{},[1414,15237,1754],{"stretchy":1753},[1047,15239,13919],{},[1414,15241,1768],{"stretchy":1753},[15157,15243,15244,15250],{},[15160,15245,15246],{},[15163,15247,15248],{"scriptlevel":560,"displaystyle":1062},[1032,15249],{},[15160,15251,15252],{},[15163,15253,15254],{"scriptlevel":560,"displaystyle":1062},[1032,15255,15256,15258,15260,15266,15268,15270,15272,15274,15276,15282,15284,15286],{},[1032,15257],{},[1414,15259,281],{},[14850,15261,15262,15264],{},[1414,15263,2549],{},[1047,15265,13919],{},[1047,15267,1423],{},[1414,15269,1754],{"stretchy":1753},[1047,15271,13919],{},[1414,15273,1768],{"stretchy":1753},[1047,15275,15169],{"mathvariant":1729},[1044,15277,15278,15280],{},[1047,15279,6333],{},[1047,15281,14745],{},[1414,15283,1754],{"stretchy":1753},[1047,15285,13919],{},[1414,15287,1768],{"stretchy":1753},[15157,15289,15290,15296],{},[15160,15291,15292],{},[15163,15293,15294],{"scriptlevel":560,"displaystyle":1062},[1032,15295],{},[15160,15297,15298],{},[15163,15299,15300],{"scriptlevel":560,"displaystyle":1062},[1032,15301,15302,15304,15306,15312,15314,15316,15318,15320,15322,15328,15330,15332,15334,15336],{},[1032,15303],{},[1414,15305,281],{},[14850,15307,15308,15310],{},[1414,15309,2549],{},[1047,15311,13919],{},[1047,15313,1423],{},[1414,15315,1754],{"stretchy":1753},[1047,15317,13919],{},[1414,15319,1768],{"stretchy":1753},[1047,15321,15169],{"mathvariant":1729},[1044,15323,15324,15326],{},[1047,15325,6333],{},[1047,15327,14745],{},[1414,15329,1754],{"stretchy":1753},[1047,15331,13919],{},[1414,15333,1768],{"stretchy":1753},[1414,15335,2539],{},[15163,15337,15338],{"scriptlevel":560,"displaystyle":1062},[1035,15339,15340,15354],{},[1032,15341,15342,15348,15350,15352],{},[1044,15343,15344,15346],{},[1047,15345,6333],{},[1047,15347,14745],{},[1414,15349,1754],{"stretchy":1753},[1047,15351,13919],{},[1414,15353,1768],{"stretchy":1753},[1032,15355,15356,15362,15364,15366],{},[1044,15357,15358,15360],{},[1047,15359,6333],{},[1047,15361,14745],{},[1414,15363,1754],{"stretchy":1753},[1047,15365,13919],{},[1414,15367,1768],{"stretchy":1753},[15157,15369,15370,15376],{},[15160,15371,15372],{},[15163,15373,15374],{"scriptlevel":560,"displaystyle":1062},[1032,15375],{},[15160,15377,15378],{},[15163,15379,15380],{"scriptlevel":560,"displaystyle":1062},[1032,15381,15382,15384,15386,15392,15398,15400,15402,15404,15406,15408,15410,15412],{},[1032,15383],{},[1414,15385,281],{},[14850,15387,15388,15390],{},[1414,15389,2549],{},[1047,15391,13919],{},[1044,15393,15394,15396],{},[1047,15395,6333],{},[1047,15397,14745],{},[1414,15399,1754],{"stretchy":1753},[1047,15401,13919],{},[1414,15403,1768],{"stretchy":1753},[1047,15405,1423],{},[1414,15407,1754],{"stretchy":1753},[1047,15409,13919],{},[1414,15411,1768],{"stretchy":1753},[15163,15413,15414],{"scriptlevel":560,"displaystyle":1062},[1035,15415,15416,15432],{},[1032,15417,15418,15420,15426,15428,15430],{},[1047,15419,15169],{"mathvariant":1729},[1044,15421,15422,15424],{},[1047,15423,6333],{},[1047,15425,14745],{},[1414,15427,1754],{"stretchy":1753},[1047,15429,13919],{},[1414,15431,1768],{"stretchy":1753},[1032,15433,15434,15440,15442,15444],{},[1044,15435,15436,15438],{},[1047,15437,6333],{},[1047,15439,14745],{},[1414,15441,1754],{"stretchy":1753},[1047,15443,13919],{},[1414,15445,1768],{"stretchy":1753},[15157,15447,15448,15454],{},[15160,15449,15450],{},[15163,15451,15452],{"scriptlevel":560,"displaystyle":1062},[1032,15453],{},[15160,15455,15456],{},[15163,15457,15458],{"scriptlevel":560,"displaystyle":1062},[1032,15459,15460,15462,15465,15471,15486,15488,15490,15496,15498],{},[1032,15461],{},[1414,15463,15464],{},"≈",[1035,15466,15467,15469],{},[1038,15468,574],{},[1047,15470,7990],{},[15472,15473,15474,15476,15484],"munderover",{},[1414,15475,2549],{},[1032,15477,15478,15480,15482],{},[1047,15479,1288],{},[1414,15481,281],{},[1038,15483,574],{},[1047,15485,7990],{},[1047,15487,1423],{},[1414,15489,1754],{"stretchy":1753},[1418,15491,15492,15494],{},[1047,15493,13919],{},[1047,15495,1288],{},[1414,15497,1768],{"stretchy":1753},[15163,15499,15500],{"scriptlevel":560,"displaystyle":1062},[1035,15501,15502,15522],{},[1032,15503,15504,15506,15512,15514,15520],{},[1047,15505,15169],{"mathvariant":1729},[1044,15507,15508,15510],{},[1047,15509,14],{},[1047,15511,14745],{},[1414,15513,1754],{"stretchy":1753},[1418,15515,15516,15518],{},[1047,15517,13919],{},[1047,15519,1288],{},[1414,15521,1768],{"stretchy":1753},[1032,15523,15524,15530,15532,15538],{},[1044,15525,15526,15528],{},[1047,15527,6333],{},[1047,15529,14745],{},[1414,15531,1754],{"stretchy":1753},[1418,15533,15534,15536],{},[1047,15535,13919],{},[1047,15537,1288],{},[1414,15539,1768],{"stretchy":1753},[15157,15541,15542,15548],{},[15160,15543,15544],{},[15163,15545,15546],{"scriptlevel":560,"displaystyle":1062},[1032,15547],{},[15160,15549,15550],{},[15163,15551,15552],{"scriptlevel":560,"displaystyle":1062},[1032,15553,15554,15556,15558,15564,15578,15580,15582,15588,15590,15592,15595,15597,15603,15605,15611],{},[1032,15555],{},[1414,15557,281],{},[1035,15559,15560,15562],{},[1038,15561,574],{},[1047,15563,7990],{},[15472,15565,15566,15568,15576],{},[1414,15567,2549],{},[1032,15569,15570,15572,15574],{},[1047,15571,1288],{},[1414,15573,281],{},[1038,15575,574],{},[1047,15577,7990],{},[1047,15579,1423],{},[1414,15581,1754],{"stretchy":1753},[1418,15583,15584,15586],{},[1047,15585,13919],{},[1047,15587,1288],{},[1414,15589,1768],{"stretchy":1753},[1047,15591,15169],{"mathvariant":1729},[1047,15593,15594],{},"log",[1414,15596,1776],{},[1044,15598,15599,15601],{},[1047,15600,6333],{},[1047,15602,14745],{},[1414,15604,1754],{"stretchy":1753},[1418,15606,15607,15609],{},[1047,15608,13919],{},[1047,15610,1288],{},[1414,15612,1768],{"stretchy":1753},[15157,15614,15615,15621],{},[15160,15616,15617],{},[15163,15618,15619],{"scriptlevel":560,"displaystyle":1062},[1032,15620],{},[15160,15622,15623],{},[15163,15624,15625],{"scriptlevel":560,"displaystyle":1062},[1032,15626,15627,15629,15631,15637,15651,15653,15655,15661,15663,15665,15667,15669,15688,15694,15696,15704,15706,15714],{},[1032,15628],{},[1414,15630,281],{},[1035,15632,15633,15635],{},[1038,15634,574],{},[1047,15636,7990],{},[15472,15638,15639,15641,15649],{},[1414,15640,2549],{},[1032,15642,15643,15645,15647],{},[1047,15644,1288],{},[1414,15646,281],{},[1038,15648,574],{},[1047,15650,7990],{},[1047,15652,1423],{},[1414,15654,1754],{"stretchy":1753},[1418,15656,15657,15659],{},[1047,15658,13919],{},[1047,15660,1288],{},[1414,15662,1768],{"stretchy":1753},[1047,15664,15169],{"mathvariant":1729},[1047,15666,15594],{},[1414,15668,1776],{},[15472,15670,15671,15674,15682],{},[1414,15672,15673],{},"∏",[1032,15675,15676,15678,15680],{},[1047,15677,1733],{},[1414,15679,281],{},[1038,15681,574],{},[1044,15683,15684,15686],{},[1047,15685,738],{},[1047,15687,1288],{},[1044,15689,15690,15692],{},[1047,15691,6333],{},[1047,15693,14745],{},[1414,15695,1754],{"stretchy":1753},[2545,15697,15698,15700,15702],{},[1047,15699,18],{},[1047,15701,1288],{},[1047,15703,1733],{},[1047,15705,13675],{"mathvariant":1729},[2545,15707,15708,15710,15712],{},[1047,15709,13535],{},[1047,15711,1288],{},[1047,15713,1733],{},[1414,15715,1768],{"stretchy":1753},[15157,15717,15718,15724],{},[15160,15719,15720],{},[15163,15721,15722],{"scriptlevel":560,"displaystyle":1062},[1032,15723],{},[15160,15725,15726],{},[15163,15727,15728],{"scriptlevel":560,"displaystyle":1062},[1032,15729,15730,15732,15734,15740,15754,15756,15758,15764,15766,15784,15786,15788,15790,15796,15798,15806,15808,15816],{},[1032,15731],{},[1414,15733,281],{},[1035,15735,15736,15738],{},[1038,15737,574],{},[1047,15739,7990],{},[15472,15741,15742,15744,15752],{},[1414,15743,2549],{},[1032,15745,15746,15748,15750],{},[1047,15747,1288],{},[1414,15749,281],{},[1038,15751,574],{},[1047,15753,7990],{},[1047,15755,1423],{},[1414,15757,1754],{"stretchy":1753},[1418,15759,15760,15762],{},[1047,15761,13919],{},[1047,15763,1288],{},[1414,15765,1768],{"stretchy":1753},[15472,15767,15768,15770,15778],{},[1414,15769,2549],{},[1032,15771,15772,15774,15776],{},[1047,15773,1733],{},[1414,15775,281],{},[1038,15777,574],{},[1044,15779,15780,15782],{},[1047,15781,738],{},[1047,15783,1288],{},[1047,15785,15169],{"mathvariant":1729},[1047,15787,15594],{},[1414,15789,1776],{},[1044,15791,15792,15794],{},[1047,15793,6333],{},[1047,15795,14745],{},[1414,15797,1754],{"stretchy":1753},[2545,15799,15800,15802,15804],{},[1047,15801,18],{},[1047,15803,1288],{},[1047,15805,1733],{},[1047,15807,13675],{"mathvariant":1729},[2545,15809,15810,15812,15814],{},[1047,15811,13535],{},[1047,15813,1288],{},[1047,15815,1733],{},[1414,15817,1768],{"stretchy":1753},[15157,15819,15820,15826],{},[15160,15821,15822],{},[15163,15823,15824],{"scriptlevel":560,"displaystyle":1062},[1032,15825],{},[15160,15827,15828],{},[15163,15829,15830],{"scriptlevel":560,"displaystyle":1062},[1032,15831,15832,15834,15836,15842,15856,15874,15876,15878,15884,15886,15888,15890,15892,15898,15900,15908,15910,15918],{},[1032,15833],{},[1414,15835,281],{},[1035,15837,15838,15840],{},[1038,15839,574],{},[1047,15841,7990],{},[15472,15843,15844,15846,15854],{},[1414,15845,2549],{},[1032,15847,15848,15850,15852],{},[1047,15849,1288],{},[1414,15851,281],{},[1038,15853,574],{},[1047,15855,7990],{},[15472,15857,15858,15860,15868],{},[1414,15859,2549],{},[1032,15861,15862,15864,15866],{},[1047,15863,1733],{},[1414,15865,281],{},[1038,15867,574],{},[1044,15869,15870,15872],{},[1047,15871,738],{},[1047,15873,1288],{},[1047,15875,1423],{},[1414,15877,1754],{"stretchy":1753},[1418,15879,15880,15882],{},[1047,15881,13919],{},[1047,15883,1288],{},[1414,15885,1768],{"stretchy":1753},[1047,15887,15169],{"mathvariant":1729},[1047,15889,15594],{},[1414,15891,1776],{},[1044,15893,15894,15896],{},[1047,15895,6333],{},[1047,15897,14745],{},[1414,15899,1754],{"stretchy":1753},[2545,15901,15902,15904,15906],{},[1047,15903,18],{},[1047,15905,1288],{},[1047,15907,1733],{},[1047,15909,13675],{"mathvariant":1729},[2545,15911,15912,15914,15916],{},[1047,15913,13535],{},[1047,15915,1288],{},[1047,15917,1733],{},[1414,15919,1768],{"stretchy":1753},[1054,15921,15922],{"encoding":1056},"\\begin{aligned}\n\\nabla E(R(\\tau))_{\\tau \\sim P_\\theta(\\tau)} &= \\nabla \\sum_\\tau R(\\tau)P_\\theta(\\tau)\\\\\n&= \\sum_\\tau R(\\tau)\\nabla P_\\theta(\\tau)\\\\\n&= \\sum_\\tau R(\\tau)\\nabla P_\\theta(\\tau) \\cdot \\dfrac{P_\\theta(\\tau)}{P_\\theta(\\tau)}\\\\\n&= \\sum_\\tau P_\\theta(\\tau)R(\\tau) \\dfrac{\\nabla P_\\theta(\\tau)}{P_\\theta(\\tau)}\\\\\n&\\approx \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\dfrac{\\nabla p_\\theta(\\tau^n)}{P_\\theta(\\tau^n)}\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\nabla \\log P_\\theta(\\tau^n)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\nabla \\log \\prod\\limits_{t=1}^{T_n}P_\\theta(a_n^t|s_n^t)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N R(\\tau^n)\\sum\\limits_{t=1}^{T_n}\\nabla \\log P_\\theta(a_n^t|s_n^t)\\\\\n&= \\frac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} R(\\tau^n)\\nabla \\log P_\\theta(a_n^t|s_n^t)\\\\\n\\end{aligned}",[57,15924,15926],{"className":15925,"ariaHidden":1062},[1061],[57,15927,15929,15933],{"className":15928},[1066],[57,15930],{"className":15931,"style":15932},[1070],"height:28.1646em;vertical-align:-13.8323em;",[57,15934,15936],{"className":15935},[1075],[57,15937,15939,16163],{"className":15938},[15151],[57,15940,15943],{"className":15941},[15942],"col-align-r",[57,15944,15946,16154],{"className":15945},[1087,1088],[57,15947,15949,16151],{"className":15948},[1092],[57,15950,15953,16079,16088,16097,16106,16115,16124,16133,16142],{"className":15951,"style":15952},[1096],"height:14.3323em;",[57,15954,15956,15960],{"style":15955},"top:-17.1217em;",[57,15957],{"className":15958,"style":15959},[1104],"height:3.8394em;",[57,15961,15963,15966,15969,15972,15975,15978,15981,15984],{"className":15962},[1075],[57,15964,15169],{"className":15965},[1075],[57,15967,6336],{"className":15968,"style":6749},[1075,1148],[57,15970,1754],{"className":15971},[1079],[57,15973,1423],{"className":15974,"style":14898},[1075,1148],[57,15976,1754],{"className":15977},[1079],[57,15979,13919],{"className":15980,"style":13935},[1075,1148],[57,15982,1768],{"className":15983},[1269],[57,15985,15987,15990],{"className":15986},[1269],[57,15988,1768],{"className":15989},[1269],[57,15991,15993],{"className":15992},[1152],[57,15994,15996,16071],{"className":15995},[1087,1088],[57,15997,15999,16068],{"className":15998},[1092],[57,16000,16002],{"className":16001,"style":1162},[1096],[57,16003,16004,16007],{"style":14928},[57,16005],{"className":16006,"style":1493},[1104],[57,16008,16010],{"className":16009},[1109,1110,1111,1112],[57,16011,16013,16016,16019,16059,16062,16065],{"className":16012},[1075,1112],[57,16014,13919],{"className":16015,"style":13935},[1075,1148,1112],[57,16017,14500],{"className":16018},[1457,1112],[57,16020,16022,16025],{"className":16021},[1075,1112],[57,16023,6333],{"className":16024,"style":3907},[1075,1148,1112],[57,16026,16028],{"className":16027},[1152],[57,16029,16031,16051],{"className":16030},[1087,1088],[57,16032,16034,16048],{"className":16033},[1092],[57,16035,16037],{"className":16036,"style":1162},[1096],[57,16038,16039,16042],{"style":14964},[57,16040],{"className":16041,"style":1169},[1104],[57,16043,16045],{"className":16044},[1109,1173,1174,1112],[57,16046,14745],{"className":16047,"style":3932},[1075,1148,1112],[57,16049,1183],{"className":16050},[1182],[57,16052,16054],{"className":16053},[1092],[57,16055,16057],{"className":16056,"style":1190},[1096],[57,16058],{},[57,16060,1754],{"className":16061},[1079,1112],[57,16063,13919],{"className":16064,"style":13935},[1075,1148,1112],[57,16066,1768],{"className":16067},[1269,1112],[57,16069,1183],{"className":16070},[1182],[57,16072,16074],{"className":16073},[1092],[57,16075,16077],{"className":16076,"style":6801},[1096],[57,16078],{},[57,16080,16082,16085],{"style":16081},"top:-14.5217em;",[57,16083],{"className":16084,"style":15959},[1104],[57,16086],{"className":16087},[1075],[57,16089,16091,16094],{"style":16090},"top:-11.5447em;",[57,16092],{"className":16093,"style":15959},[1104],[57,16095],{"className":16096},[1075],[57,16098,16100,16103],{"style":16099},"top:-8.5677em;",[57,16101],{"className":16102,"style":15959},[1104],[57,16104],{"className":16105},[1075],[57,16107,16109,16112],{"style":16108},"top:-5.1894em;",[57,16110],{"className":16111,"style":15959},[1104],[57,16113],{"className":16114},[1075],[57,16116,16118,16121],{"style":16117},"top:-1.7939em;",[57,16119],{"className":16120,"style":15959},[1104],[57,16122],{"className":16123},[1075],[57,16125,16127,16130],{"style":16126},"top:1.6126em;",[57,16128],{"className":16129,"style":15959},[1104],[57,16131],{"className":16132},[1075],[57,16134,16136,16139],{"style":16135},"top:5.0192em;",[57,16137],{"className":16138,"style":15959},[1104],[57,16140],{"className":16141},[1075],[57,16143,16145,16148],{"style":16144},"top:8.4257em;",[57,16146],{"className":16147,"style":15959},[1104],[57,16149],{"className":16150},[1075],[57,16152,1183],{"className":16153},[1182],[57,16155,16157],{"className":16156},[1092],[57,16158,16161],{"className":16159,"style":16160},[1096],"height:13.8323em;",[57,16162],{},[57,16164,16167],{"className":16165},[16166],"col-align-l",[57,16168,16170,19121],{"className":16169},[1087,1088],[57,16171,16173,19118],{"className":16172},[1092],[57,16174,16176,16316,16447,16745,17030,17441,17723,18191,18656],{"className":16175,"style":15952},[1096],[57,16177,16178,16181],{"style":15955},[57,16179],{"className":16180,"style":15959},[1104],[57,16182,16184,16187,16190,16193,16196,16199,16202,16252,16255,16258,16261,16264,16267,16307,16310,16313],{"className":16183},[1075],[57,16185],{"className":16186},[1075],[57,16188],{"className":16189,"style":1453},[1452],[57,16191,281],{"className":16192},[1457],[57,16194],{"className":16195,"style":1453},[1452],[57,16197,15169],{"className":16198},[1075],[57,16200],{"className":16201,"style":1853},[1452],[57,16203,16205],{"className":16204},[1888,15027],[57,16206,16208,16243],{"className":16207},[1087,1088],[57,16209,16211,16240],{"className":16210},[1092],[57,16212,16215,16228],{"className":16213,"style":16214},[1096],"height:1.05em;",[57,16216,16218,16222],{"style":16217},"top:-1.9em;margin-left:0em;",[57,16219],{"className":16220,"style":16221},[1104],"height:3.05em;",[57,16223,16225],{"className":16224},[1109,1110,1111,1112],[57,16226,13919],{"className":16227,"style":13935},[1075,1148,1112],[57,16229,16231,16234],{"style":16230},"top:-3.05em;",[57,16232],{"className":16233,"style":16221},[1104],[57,16235,16236],{},[57,16237,2549],{"className":16238},[1888,2633,16239],"large-op",[57,16241,1183],{"className":16242},[1182],[57,16244,16246],{"className":16245},[1092],[57,16247,16250],{"className":16248,"style":16249},[1096],"height:1.25em;",[57,16251],{},[57,16253],{"className":16254,"style":1853},[1452],[57,16256,1423],{"className":16257,"style":14898},[1075,1148],[57,16259,1754],{"className":16260},[1079],[57,16262,13919],{"className":16263,"style":13935},[1075,1148],[57,16265,1768],{"className":16266},[1269],[57,16268,16270,16273],{"className":16269},[1075],[57,16271,6333],{"className":16272,"style":3907},[1075,1148],[57,16274,16276],{"className":16275},[1152],[57,16277,16279,16299],{"className":16278},[1087,1088],[57,16280,16282,16296],{"className":16281},[1092],[57,16283,16285],{"className":16284,"style":2208},[1096],[57,16286,16287,16290],{"style":8131},[57,16288],{"className":16289,"style":1493},[1104],[57,16291,16293],{"className":16292},[1109,1110,1111,1112],[57,16294,14745],{"className":16295,"style":3932},[1075,1148,1112],[57,16297,1183],{"className":16298},[1182],[57,16300,16302],{"className":16301},[1092],[57,16303,16305],{"className":16304,"style":2230},[1096],[57,16306],{},[57,16308,1754],{"className":16309},[1079],[57,16311,13919],{"className":16312,"style":13935},[1075,1148],[57,16314,1768],{"className":16315},[1269],[57,16317,16318,16321],{"style":16081},[57,16319],{"className":16320,"style":15959},[1104],[57,16322,16324,16327,16330,16333,16336,16380,16383,16386,16389,16392,16395,16398,16438,16441,16444],{"className":16323},[1075],[57,16325],{"className":16326},[1075],[57,16328],{"className":16329,"style":1453},[1452],[57,16331,281],{"className":16332},[1457],[57,16334],{"className":16335,"style":1453},[1452],[57,16337,16339],{"className":16338},[1888,15027],[57,16340,16342,16372],{"className":16341},[1087,1088],[57,16343,16345,16369],{"className":16344},[1092],[57,16346,16348,16359],{"className":16347,"style":16214},[1096],[57,16349,16350,16353],{"style":16217},[57,16351],{"className":16352,"style":16221},[1104],[57,16354,16356],{"className":16355},[1109,1110,1111,1112],[57,16357,13919],{"className":16358,"style":13935},[1075,1148,1112],[57,16360,16361,16364],{"style":16230},[57,16362],{"className":16363,"style":16221},[1104],[57,16365,16366],{},[57,16367,2549],{"className":16368},[1888,2633,16239],[57,16370,1183],{"className":16371},[1182],[57,16373,16375],{"className":16374},[1092],[57,16376,16378],{"className":16377,"style":16249},[1096],[57,16379],{},[57,16381],{"className":16382,"style":1853},[1452],[57,16384,1423],{"className":16385,"style":14898},[1075,1148],[57,16387,1754],{"className":16388},[1079],[57,16390,13919],{"className":16391,"style":13935},[1075,1148],[57,16393,1768],{"className":16394},[1269],[57,16396,15169],{"className":16397},[1075],[57,16399,16401,16404],{"className":16400},[1075],[57,16402,6333],{"className":16403,"style":3907},[1075,1148],[57,16405,16407],{"className":16406},[1152],[57,16408,16410,16430],{"className":16409},[1087,1088],[57,16411,16413,16427],{"className":16412},[1092],[57,16414,16416],{"className":16415,"style":2208},[1096],[57,16417,16418,16421],{"style":8131},[57,16419],{"className":16420,"style":1493},[1104],[57,16422,16424],{"className":16423},[1109,1110,1111,1112],[57,16425,14745],{"className":16426,"style":3932},[1075,1148,1112],[57,16428,1183],{"className":16429},[1182],[57,16431,16433],{"className":16432},[1092],[57,16434,16436],{"className":16435,"style":2230},[1096],[57,16437],{},[57,16439,1754],{"className":16440},[1079],[57,16442,13919],{"className":16443,"style":13935},[1075,1148],[57,16445,1768],{"className":16446},[1269],[57,16448,16449,16452],{"style":16090},[57,16450],{"className":16451,"style":15959},[1104],[57,16453,16455,16458,16461,16464,16467,16511,16514,16517,16520,16523,16526,16529,16569,16572,16575,16578,16581,16584,16587],{"className":16454},[1075],[57,16456],{"className":16457},[1075],[57,16459],{"className":16460,"style":1453},[1452],[57,16462,281],{"className":16463},[1457],[57,16465],{"className":16466,"style":1453},[1452],[57,16468,16470],{"className":16469},[1888,15027],[57,16471,16473,16503],{"className":16472},[1087,1088],[57,16474,16476,16500],{"className":16475},[1092],[57,16477,16479,16490],{"className":16478,"style":16214},[1096],[57,16480,16481,16484],{"style":16217},[57,16482],{"className":16483,"style":16221},[1104],[57,16485,16487],{"className":16486},[1109,1110,1111,1112],[57,16488,13919],{"className":16489,"style":13935},[1075,1148,1112],[57,16491,16492,16495],{"style":16230},[57,16493],{"className":16494,"style":16221},[1104],[57,16496,16497],{},[57,16498,2549],{"className":16499},[1888,2633,16239],[57,16501,1183],{"className":16502},[1182],[57,16504,16506],{"className":16505},[1092],[57,16507,16509],{"className":16508,"style":16249},[1096],[57,16510],{},[57,16512],{"className":16513,"style":1853},[1452],[57,16515,1423],{"className":16516,"style":14898},[1075,1148],[57,16518,1754],{"className":16519},[1079],[57,16521,13919],{"className":16522,"style":13935},[1075,1148],[57,16524,1768],{"className":16525},[1269],[57,16527,15169],{"className":16528},[1075],[57,16530,16532,16535],{"className":16531},[1075],[57,16533,6333],{"className":16534,"style":3907},[1075,1148],[57,16536,16538],{"className":16537},[1152],[57,16539,16541,16561],{"className":16540},[1087,1088],[57,16542,16544,16558],{"className":16543},[1092],[57,16545,16547],{"className":16546,"style":2208},[1096],[57,16548,16549,16552],{"style":8131},[57,16550],{"className":16551,"style":1493},[1104],[57,16553,16555],{"className":16554},[1109,1110,1111,1112],[57,16556,14745],{"className":16557,"style":3932},[1075,1148,1112],[57,16559,1183],{"className":16560},[1182],[57,16562,16564],{"className":16563},[1092],[57,16565,16567],{"className":16566,"style":2230},[1096],[57,16568],{},[57,16570,1754],{"className":16571},[1079],[57,16573,13919],{"className":16574,"style":13935},[1075,1148],[57,16576,1768],{"className":16577},[1269],[57,16579],{"className":16580,"style":2595},[1452],[57,16582,2539],{"className":16583},[1506],[57,16585],{"className":16586,"style":2595},[1452],[57,16588,16590,16593,16742],{"className":16589},[1075],[57,16591],{"className":16592},[1079,1080],[57,16594,16596],{"className":16595},[1035],[57,16597,16599,16733],{"className":16598},[1087,1088],[57,16600,16602,16730],{"className":16601},[1092],[57,16603,16606,16664,16672],{"className":16604,"style":16605},[1096],"height:1.427em;",[57,16607,16609,16612],{"style":16608},"top:-2.314em;",[57,16610],{"className":16611,"style":1105},[1104],[57,16613,16615,16655,16658,16661],{"className":16614},[1075],[57,16616,16618,16621],{"className":16617},[1075],[57,16619,6333],{"className":16620,"style":3907},[1075,1148],[57,16622,16624],{"className":16623},[1152],[57,16625,16627,16647],{"className":16626},[1087,1088],[57,16628,16630,16644],{"className":16629},[1092],[57,16631,16633],{"className":16632,"style":2208},[1096],[57,16634,16635,16638],{"style":8131},[57,16636],{"className":16637,"style":1493},[1104],[57,16639,16641],{"className":16640},[1109,1110,1111,1112],[57,16642,14745],{"className":16643,"style":3932},[1075,1148,1112],[57,16645,1183],{"className":16646},[1182],[57,16648,16650],{"className":16649},[1092],[57,16651,16653],{"className":16652,"style":2230},[1096],[57,16654],{},[57,16656,1754],{"className":16657},[1079],[57,16659,13919],{"className":16660,"style":13935},[1075,1148],[57,16662,1768],{"className":16663},[1269],[57,16665,16666,16669],{"style":1230},[57,16667],{"className":16668,"style":1105},[1104],[57,16670],{"className":16671,"style":1238},[1237],[57,16673,16675,16678],{"style":16674},"top:-3.677em;",[57,16676],{"className":16677,"style":1105},[1104],[57,16679,16681,16721,16724,16727],{"className":16680},[1075],[57,16682,16684,16687],{"className":16683},[1075],[57,16685,6333],{"className":16686,"style":3907},[1075,1148],[57,16688,16690],{"className":16689},[1152],[57,16691,16693,16713],{"className":16692},[1087,1088],[57,16694,16696,16710],{"className":16695},[1092],[57,16697,16699],{"className":16698,"style":2208},[1096],[57,16700,16701,16704],{"style":8131},[57,16702],{"className":16703,"style":1493},[1104],[57,16705,16707],{"className":16706},[1109,1110,1111,1112],[57,16708,14745],{"className":16709,"style":3932},[1075,1148,1112],[57,16711,1183],{"className":16712},[1182],[57,16714,16716],{"className":16715},[1092],[57,16717,16719],{"className":16718,"style":2230},[1096],[57,16720],{},[57,16722,1754],{"className":16723},[1079],[57,16725,13919],{"className":16726,"style":13935},[1075,1148],[57,16728,1768],{"className":16729},[1269],[57,16731,1183],{"className":16732},[1182],[57,16734,16736],{"className":16735},[1092],[57,16737,16740],{"className":16738,"style":16739},[1096],"height:0.936em;",[57,16741],{},[57,16743],{"className":16744},[1269,1080],[57,16746,16747,16750],{"style":16099},[57,16748],{"className":16749,"style":15959},[1104],[57,16751,16753,16756,16759,16762,16765,16809,16812,16852,16855,16858,16861,16864,16867,16870,16873],{"className":16752},[1075],[57,16754],{"className":16755},[1075],[57,16757],{"className":16758,"style":1453},[1452],[57,16760,281],{"className":16761},[1457],[57,16763],{"className":16764,"style":1453},[1452],[57,16766,16768],{"className":16767},[1888,15027],[57,16769,16771,16801],{"className":16770},[1087,1088],[57,16772,16774,16798],{"className":16773},[1092],[57,16775,16777,16788],{"className":16776,"style":16214},[1096],[57,16778,16779,16782],{"style":16217},[57,16780],{"className":16781,"style":16221},[1104],[57,16783,16785],{"className":16784},[1109,1110,1111,1112],[57,16786,13919],{"className":16787,"style":13935},[1075,1148,1112],[57,16789,16790,16793],{"style":16230},[57,16791],{"className":16792,"style":16221},[1104],[57,16794,16795],{},[57,16796,2549],{"className":16797},[1888,2633,16239],[57,16799,1183],{"className":16800},[1182],[57,16802,16804],{"className":16803},[1092],[57,16805,16807],{"className":16806,"style":16249},[1096],[57,16808],{},[57,16810],{"className":16811,"style":1853},[1452],[57,16813,16815,16818],{"className":16814},[1075],[57,16816,6333],{"className":16817,"style":3907},[1075,1148],[57,16819,16821],{"className":16820},[1152],[57,16822,16824,16844],{"className":16823},[1087,1088],[57,16825,16827,16841],{"className":16826},[1092],[57,16828,16830],{"className":16829,"style":2208},[1096],[57,16831,16832,16835],{"style":8131},[57,16833],{"className":16834,"style":1493},[1104],[57,16836,16838],{"className":16837},[1109,1110,1111,1112],[57,16839,14745],{"className":16840,"style":3932},[1075,1148,1112],[57,16842,1183],{"className":16843},[1182],[57,16845,16847],{"className":16846},[1092],[57,16848,16850],{"className":16849,"style":2230},[1096],[57,16851],{},[57,16853,1754],{"className":16854},[1079],[57,16856,13919],{"className":16857,"style":13935},[1075,1148],[57,16859,1768],{"className":16860},[1269],[57,16862,1423],{"className":16863,"style":14898},[1075,1148],[57,16865,1754],{"className":16866},[1079],[57,16868,13919],{"className":16869,"style":13935},[1075,1148],[57,16871,1768],{"className":16872},[1269],[57,16874,16876,16879,17027],{"className":16875},[1075],[57,16877],{"className":16878},[1079,1080],[57,16880,16882],{"className":16881},[1035],[57,16883,16885,17019],{"className":16884},[1087,1088],[57,16886,16888,17016],{"className":16887},[1092],[57,16889,16891,16948,16956],{"className":16890,"style":16605},[1096],[57,16892,16893,16896],{"style":16608},[57,16894],{"className":16895,"style":1105},[1104],[57,16897,16899,16939,16942,16945],{"className":16898},[1075],[57,16900,16902,16905],{"className":16901},[1075],[57,16903,6333],{"className":16904,"style":3907},[1075,1148],[57,16906,16908],{"className":16907},[1152],[57,16909,16911,16931],{"className":16910},[1087,1088],[57,16912,16914,16928],{"className":16913},[1092],[57,16915,16917],{"className":16916,"style":2208},[1096],[57,16918,16919,16922],{"style":8131},[57,16920],{"className":16921,"style":1493},[1104],[57,16923,16925],{"className":16924},[1109,1110,1111,1112],[57,16926,14745],{"className":16927,"style":3932},[1075,1148,1112],[57,16929,1183],{"className":16930},[1182],[57,16932,16934],{"className":16933},[1092],[57,16935,16937],{"className":16936,"style":2230},[1096],[57,16938],{},[57,16940,1754],{"className":16941},[1079],[57,16943,13919],{"className":16944,"style":13935},[1075,1148],[57,16946,1768],{"className":16947},[1269],[57,16949,16950,16953],{"style":1230},[57,16951],{"className":16952,"style":1105},[1104],[57,16954],{"className":16955,"style":1238},[1237],[57,16957,16958,16961],{"style":16674},[57,16959],{"className":16960,"style":1105},[1104],[57,16962,16964,16967,17007,17010,17013],{"className":16963},[1075],[57,16965,15169],{"className":16966},[1075],[57,16968,16970,16973],{"className":16969},[1075],[57,16971,6333],{"className":16972,"style":3907},[1075,1148],[57,16974,16976],{"className":16975},[1152],[57,16977,16979,16999],{"className":16978},[1087,1088],[57,16980,16982,16996],{"className":16981},[1092],[57,16983,16985],{"className":16984,"style":2208},[1096],[57,16986,16987,16990],{"style":8131},[57,16988],{"className":16989,"style":1493},[1104],[57,16991,16993],{"className":16992},[1109,1110,1111,1112],[57,16994,14745],{"className":16995,"style":3932},[1075,1148,1112],[57,16997,1183],{"className":16998},[1182],[57,17000,17002],{"className":17001},[1092],[57,17003,17005],{"className":17004,"style":2230},[1096],[57,17006],{},[57,17008,1754],{"className":17009},[1079],[57,17011,13919],{"className":17012,"style":13935},[1075,1148],[57,17014,1768],{"className":17015},[1269],[57,17017,1183],{"className":17018},[1182],[57,17020,17022],{"className":17021},[1092],[57,17023,17025],{"className":17024,"style":16739},[1096],[57,17026],{},[57,17028],{"className":17029},[1269,1080],[57,17031,17032,17035],{"style":16108},[57,17033],{"className":17034,"style":15959},[1104],[57,17036,17038,17041,17044,17047,17050,17115,17118,17186,17189,17192,17195,17226,17229],{"className":17037},[1075],[57,17039],{"className":17040},[1075],[57,17042],{"className":17043,"style":1453},[1452],[57,17045,15464],{"className":17046},[1457],[57,17048],{"className":17049,"style":1453},[1452],[57,17051,17053,17056,17112],{"className":17052},[1075],[57,17054],{"className":17055},[1079,1080],[57,17057,17059],{"className":17058},[1035],[57,17060,17062,17103],{"className":17061},[1087,1088],[57,17063,17065,17100],{"className":17064},[1092],[57,17066,17069,17081,17089],{"className":17067,"style":17068},[1096],"height:1.3214em;",[57,17070,17071,17074],{"style":16608},[57,17072],{"className":17073,"style":1105},[1104],[57,17075,17077],{"className":17076},[1075],[57,17078,7990],{"className":17079,"style":17080},[1075,1148],"margin-right:0.10903em;",[57,17082,17083,17086],{"style":1230},[57,17084],{"className":17085,"style":1105},[1104],[57,17087],{"className":17088,"style":1238},[1237],[57,17090,17091,17094],{"style":16674},[57,17092],{"className":17093,"style":1105},[1104],[57,17095,17097],{"className":17096},[1075],[57,17098,574],{"className":17099},[1075],[57,17101,1183],{"className":17102},[1182],[57,17104,17106],{"className":17105},[1092],[57,17107,17110],{"className":17108,"style":17109},[1096],"height:0.686em;",[57,17111],{},[57,17113],{"className":17114},[1269,1080],[57,17116],{"className":17117,"style":1853},[1452],[57,17119,17121],{"className":17120},[1888,15027],[57,17122,17124,17177],{"className":17123},[1087,1088],[57,17125,17127,17174],{"className":17126},[1092],[57,17128,17131,17152,17162],{"className":17129,"style":17130},[1096],"height:1.8283em;",[57,17132,17134,17137],{"style":17133},"top:-1.8829em;margin-left:0em;",[57,17135],{"className":17136,"style":16221},[1104],[57,17138,17140],{"className":17139},[1109,1110,1111,1112],[57,17141,17143,17146,17149],{"className":17142},[1075,1112],[57,17144,1288],{"className":17145},[1075,1148,1112],[57,17147,281],{"className":17148},[1457,1112],[57,17150,574],{"className":17151},[1075,1112],[57,17153,17154,17157],{"style":16230},[57,17155],{"className":17156,"style":16221},[1104],[57,17158,17159],{},[57,17160,2549],{"className":17161},[1888,2633,16239],[57,17163,17165,17168],{"style":17164},"top:-4.3em;margin-left:0em;",[57,17166],{"className":17167,"style":16221},[1104],[57,17169,17171],{"className":17170},[1109,1110,1111,1112],[57,17172,7990],{"className":17173,"style":17080},[1075,1148,1112],[57,17175,1183],{"className":17176},[1182],[57,17178,17180],{"className":17179},[1092],[57,17181,17184],{"className":17182,"style":17183},[1096],"height:1.2671em;",[57,17185],{},[57,17187],{"className":17188,"style":1853},[1452],[57,17190,1423],{"className":17191,"style":14898},[1075,1148],[57,17193,1754],{"className":17194},[1079],[57,17196,17198,17201],{"className":17197},[1075],[57,17199,13919],{"className":17200,"style":13935},[1075,1148],[57,17202,17204],{"className":17203},[1152],[57,17205,17207],{"className":17206},[1087],[57,17208,17210],{"className":17209},[1092],[57,17211,17214],{"className":17212,"style":17213},[1096],"height:0.7144em;",[57,17215,17217,17220],{"style":17216},"top:-3.113em;margin-right:0.05em;",[57,17218],{"className":17219,"style":1493},[1104],[57,17221,17223],{"className":17222},[1109,1110,1111,1112],[57,17224,1288],{"className":17225},[1075,1148,1112],[57,17227,1768],{"className":17228},[1269],[57,17230,17232,17235,17438],{"className":17231},[1075],[57,17233],{"className":17234},[1079,1080],[57,17236,17238],{"className":17237},[1035],[57,17239,17241,17430],{"className":17240},[1087,1088],[57,17242,17244,17427],{"className":17243},[1092],[57,17245,17247,17332,17340],{"className":17246,"style":16605},[1096],[57,17248,17249,17252],{"style":16608},[57,17250],{"className":17251,"style":1105},[1104],[57,17253,17255,17295,17298,17329],{"className":17254},[1075],[57,17256,17258,17261],{"className":17257},[1075],[57,17259,6333],{"className":17260,"style":3907},[1075,1148],[57,17262,17264],{"className":17263},[1152],[57,17265,17267,17287],{"className":17266},[1087,1088],[57,17268,17270,17284],{"className":17269},[1092],[57,17271,17273],{"className":17272,"style":2208},[1096],[57,17274,17275,17278],{"style":8131},[57,17276],{"className":17277,"style":1493},[1104],[57,17279,17281],{"className":17280},[1109,1110,1111,1112],[57,17282,14745],{"className":17283,"style":3932},[1075,1148,1112],[57,17285,1183],{"className":17286},[1182],[57,17288,17290],{"className":17289},[1092],[57,17291,17293],{"className":17292,"style":2230},[1096],[57,17294],{},[57,17296,1754],{"className":17297},[1079],[57,17299,17301,17304],{"className":17300},[1075],[57,17302,13919],{"className":17303,"style":13935},[1075,1148],[57,17305,17307],{"className":17306},[1152],[57,17308,17310],{"className":17309},[1087],[57,17311,17313],{"className":17312},[1092],[57,17314,17317],{"className":17315,"style":17316},[1096],"height:0.5904em;",[57,17318,17320,17323],{"style":17319},"top:-2.989em;margin-right:0.05em;",[57,17321],{"className":17322,"style":1493},[1104],[57,17324,17326],{"className":17325},[1109,1110,1111,1112],[57,17327,1288],{"className":17328},[1075,1148,1112],[57,17330,1768],{"className":17331},[1269],[57,17333,17334,17337],{"style":1230},[57,17335],{"className":17336,"style":1105},[1104],[57,17338],{"className":17339,"style":1238},[1237],[57,17341,17342,17345],{"style":16674},[57,17343],{"className":17344,"style":1105},[1104],[57,17346,17348,17351,17391,17394,17424],{"className":17347},[1075],[57,17349,15169],{"className":17350},[1075],[57,17352,17354,17357],{"className":17353},[1075],[57,17355,14],{"className":17356},[1075,1148],[57,17358,17360],{"className":17359},[1152],[57,17361,17363,17383],{"className":17362},[1087,1088],[57,17364,17366,17380],{"className":17365},[1092],[57,17367,17369],{"className":17368,"style":2208},[1096],[57,17370,17371,17374],{"style":2211},[57,17372],{"className":17373,"style":1493},[1104],[57,17375,17377],{"className":17376},[1109,1110,1111,1112],[57,17378,14745],{"className":17379,"style":3932},[1075,1148,1112],[57,17381,1183],{"className":17382},[1182],[57,17384,17386],{"className":17385},[1092],[57,17387,17389],{"className":17388,"style":2230},[1096],[57,17390],{},[57,17392,1754],{"className":17393},[1079],[57,17395,17397,17400],{"className":17396},[1075],[57,17398,13919],{"className":17399,"style":13935},[1075,1148],[57,17401,17403],{"className":17402},[1152],[57,17404,17406],{"className":17405},[1087],[57,17407,17409],{"className":17408},[1092],[57,17410,17413],{"className":17411,"style":17412},[1096],"height:0.6644em;",[57,17414,17415,17418],{"style":1489},[57,17416],{"className":17417,"style":1493},[1104],[57,17419,17421],{"className":17420},[1109,1110,1111,1112],[57,17422,1288],{"className":17423},[1075,1148,1112],[57,17425,1768],{"className":17426},[1269],[57,17428,1183],{"className":17429},[1182],[57,17431,17433],{"className":17432},[1092],[57,17434,17436],{"className":17435,"style":16739},[1096],[57,17437],{},[57,17439],{"className":17440},[1269,1080],[57,17442,17443,17446],{"style":16117},[57,17444],{"className":17445,"style":15959},[1104],[57,17447,17449,17452,17455,17458,17461,17523,17526,17590,17593,17596,17599,17628,17631,17634,17637,17645,17648,17688,17691,17720],{"className":17448},[1075],[57,17450],{"className":17451},[1075],[57,17453],{"className":17454,"style":1453},[1452],[57,17456,281],{"className":17457},[1457],[57,17459],{"className":17460,"style":1453},[1452],[57,17462,17464,17467,17520],{"className":17463},[1075],[57,17465],{"className":17466},[1079,1080],[57,17468,17470],{"className":17469},[1035],[57,17471,17473,17512],{"className":17472},[1087,1088],[57,17474,17476,17509],{"className":17475},[1092],[57,17477,17479,17490,17498],{"className":17478,"style":17068},[1096],[57,17480,17481,17484],{"style":16608},[57,17482],{"className":17483,"style":1105},[1104],[57,17485,17487],{"className":17486},[1075],[57,17488,7990],{"className":17489,"style":17080},[1075,1148],[57,17491,17492,17495],{"style":1230},[57,17493],{"className":17494,"style":1105},[1104],[57,17496],{"className":17497,"style":1238},[1237],[57,17499,17500,17503],{"style":16674},[57,17501],{"className":17502,"style":1105},[1104],[57,17504,17506],{"className":17505},[1075],[57,17507,574],{"className":17508},[1075],[57,17510,1183],{"className":17511},[1182],[57,17513,17515],{"className":17514},[1092],[57,17516,17518],{"className":17517,"style":17109},[1096],[57,17519],{},[57,17521],{"className":17522},[1269,1080],[57,17524],{"className":17525,"style":1853},[1452],[57,17527,17529],{"className":17528},[1888,15027],[57,17530,17532,17582],{"className":17531},[1087,1088],[57,17533,17535,17579],{"className":17534},[1092],[57,17536,17538,17558,17568],{"className":17537,"style":17130},[1096],[57,17539,17540,17543],{"style":17133},[57,17541],{"className":17542,"style":16221},[1104],[57,17544,17546],{"className":17545},[1109,1110,1111,1112],[57,17547,17549,17552,17555],{"className":17548},[1075,1112],[57,17550,1288],{"className":17551},[1075,1148,1112],[57,17553,281],{"className":17554},[1457,1112],[57,17556,574],{"className":17557},[1075,1112],[57,17559,17560,17563],{"style":16230},[57,17561],{"className":17562,"style":16221},[1104],[57,17564,17565],{},[57,17566,2549],{"className":17567},[1888,2633,16239],[57,17569,17570,17573],{"style":17164},[57,17571],{"className":17572,"style":16221},[1104],[57,17574,17576],{"className":17575},[1109,1110,1111,1112],[57,17577,7990],{"className":17578,"style":17080},[1075,1148,1112],[57,17580,1183],{"className":17581},[1182],[57,17583,17585],{"className":17584},[1092],[57,17586,17588],{"className":17587,"style":17183},[1096],[57,17589],{},[57,17591],{"className":17592,"style":1853},[1452],[57,17594,1423],{"className":17595,"style":14898},[1075,1148],[57,17597,1754],{"className":17598},[1079],[57,17600,17602,17605],{"className":17601},[1075],[57,17603,13919],{"className":17604,"style":13935},[1075,1148],[57,17606,17608],{"className":17607},[1152],[57,17609,17611],{"className":17610},[1087],[57,17612,17614],{"className":17613},[1092],[57,17615,17617],{"className":17616,"style":17213},[1096],[57,17618,17619,17622],{"style":17216},[57,17620],{"className":17621,"style":1493},[1104],[57,17623,17625],{"className":17624},[1109,1110,1111,1112],[57,17626,1288],{"className":17627},[1075,1148,1112],[57,17629,1768],{"className":17630},[1269],[57,17632,15169],{"className":17633},[1075],[57,17635],{"className":17636,"style":1853},[1452],[57,17638,17640,17641],{"className":17639},[1888],"lo",[57,17642,17644],{"style":17643},"margin-right:0.01389em;","g",[57,17646],{"className":17647,"style":1853},[1452],[57,17649,17651,17654],{"className":17650},[1075],[57,17652,6333],{"className":17653,"style":3907},[1075,1148],[57,17655,17657],{"className":17656},[1152],[57,17658,17660,17680],{"className":17659},[1087,1088],[57,17661,17663,17677],{"className":17662},[1092],[57,17664,17666],{"className":17665,"style":2208},[1096],[57,17667,17668,17671],{"style":8131},[57,17669],{"className":17670,"style":1493},[1104],[57,17672,17674],{"className":17673},[1109,1110,1111,1112],[57,17675,14745],{"className":17676,"style":3932},[1075,1148,1112],[57,17678,1183],{"className":17679},[1182],[57,17681,17683],{"className":17682},[1092],[57,17684,17686],{"className":17685,"style":2230},[1096],[57,17687],{},[57,17689,1754],{"className":17690},[1079],[57,17692,17694,17697],{"className":17693},[1075],[57,17695,13919],{"className":17696,"style":13935},[1075,1148],[57,17698,17700],{"className":17699},[1152],[57,17701,17703],{"className":17702},[1087],[57,17704,17706],{"className":17705},[1092],[57,17707,17709],{"className":17708,"style":17213},[1096],[57,17710,17711,17714],{"style":17216},[57,17712],{"className":17713,"style":1493},[1104],[57,17715,17717],{"className":17716},[1109,1110,1111,1112],[57,17718,1288],{"className":17719},[1075,1148,1112],[57,17721,1768],{"className":17722},[1269],[57,17724,17725,17728],{"style":16126},[57,17726],{"className":17727,"style":15959},[1104],[57,17729,17731,17734,17737,17740,17743,17805,17808,17872,17875,17878,17881,17910,17913,17916,17919,17924,17927,18034,18037,18077,18080,18134,18137,18188],{"className":17730},[1075],[57,17732],{"className":17733},[1075],[57,17735],{"className":17736,"style":1453},[1452],[57,17738,281],{"className":17739},[1457],[57,17741],{"className":17742,"style":1453},[1452],[57,17744,17746,17749,17802],{"className":17745},[1075],[57,17747],{"className":17748},[1079,1080],[57,17750,17752],{"className":17751},[1035],[57,17753,17755,17794],{"className":17754},[1087,1088],[57,17756,17758,17791],{"className":17757},[1092],[57,17759,17761,17772,17780],{"className":17760,"style":17068},[1096],[57,17762,17763,17766],{"style":16608},[57,17764],{"className":17765,"style":1105},[1104],[57,17767,17769],{"className":17768},[1075],[57,17770,7990],{"className":17771,"style":17080},[1075,1148],[57,17773,17774,17777],{"style":1230},[57,17775],{"className":17776,"style":1105},[1104],[57,17778],{"className":17779,"style":1238},[1237],[57,17781,17782,17785],{"style":16674},[57,17783],{"className":17784,"style":1105},[1104],[57,17786,17788],{"className":17787},[1075],[57,17789,574],{"className":17790},[1075],[57,17792,1183],{"className":17793},[1182],[57,17795,17797],{"className":17796},[1092],[57,17798,17800],{"className":17799,"style":17109},[1096],[57,17801],{},[57,17803],{"className":17804},[1269,1080],[57,17806],{"className":17807,"style":1853},[1452],[57,17809,17811],{"className":17810},[1888,15027],[57,17812,17814,17864],{"className":17813},[1087,1088],[57,17815,17817,17861],{"className":17816},[1092],[57,17818,17820,17840,17850],{"className":17819,"style":17130},[1096],[57,17821,17822,17825],{"style":17133},[57,17823],{"className":17824,"style":16221},[1104],[57,17826,17828],{"className":17827},[1109,1110,1111,1112],[57,17829,17831,17834,17837],{"className":17830},[1075,1112],[57,17832,1288],{"className":17833},[1075,1148,1112],[57,17835,281],{"className":17836},[1457,1112],[57,17838,574],{"className":17839},[1075,1112],[57,17841,17842,17845],{"style":16230},[57,17843],{"className":17844,"style":16221},[1104],[57,17846,17847],{},[57,17848,2549],{"className":17849},[1888,2633,16239],[57,17851,17852,17855],{"style":17164},[57,17853],{"className":17854,"style":16221},[1104],[57,17856,17858],{"className":17857},[1109,1110,1111,1112],[57,17859,7990],{"className":17860,"style":17080},[1075,1148,1112],[57,17862,1183],{"className":17863},[1182],[57,17865,17867],{"className":17866},[1092],[57,17868,17870],{"className":17869,"style":17183},[1096],[57,17871],{},[57,17873],{"className":17874,"style":1853},[1452],[57,17876,1423],{"className":17877,"style":14898},[1075,1148],[57,17879,1754],{"className":17880},[1079],[57,17882,17884,17887],{"className":17883},[1075],[57,17885,13919],{"className":17886,"style":13935},[1075,1148],[57,17888,17890],{"className":17889},[1152],[57,17891,17893],{"className":17892},[1087],[57,17894,17896],{"className":17895},[1092],[57,17897,17899],{"className":17898,"style":17213},[1096],[57,17900,17901,17904],{"style":17216},[57,17902],{"className":17903,"style":1493},[1104],[57,17905,17907],{"className":17906},[1109,1110,1111,1112],[57,17908,1288],{"className":17909},[1075,1148,1112],[57,17911,1768],{"className":17912},[1269],[57,17914,15169],{"className":17915},[1075],[57,17917],{"className":17918,"style":1853},[1452],[57,17920,17640,17922],{"className":17921},[1888],[57,17923,17644],{"style":17643},[57,17925],{"className":17926,"style":1853},[1452],[57,17928,17930],{"className":17929},[1888,15027],[57,17931,17933,18026],{"className":17932},[1087,1088],[57,17934,17936,18023],{"className":17935},[1092],[57,17937,17940,17960,17970],{"className":17938,"style":17939},[1096],"height:1.8394em;",[57,17941,17942,17945],{"style":17133},[57,17943],{"className":17944,"style":16221},[1104],[57,17946,17948],{"className":17947},[1109,1110,1111,1112],[57,17949,17951,17954,17957],{"className":17950},[1075,1112],[57,17952,1733],{"className":17953},[1075,1148,1112],[57,17955,281],{"className":17956},[1457,1112],[57,17958,574],{"className":17959},[1075,1112],[57,17961,17962,17965],{"style":16230},[57,17963],{"className":17964,"style":16221},[1104],[57,17966,17967],{},[57,17968,15673],{"className":17969},[1888,2633,16239],[57,17971,17973,17976],{"style":17972},"top:-4.3111em;margin-left:0em;",[57,17974],{"className":17975,"style":16221},[1104],[57,17977,17979],{"className":17978},[1109,1110,1111,1112],[57,17980,17982],{"className":17981},[1075,1112],[57,17983,17985,17988],{"className":17984},[1075,1112],[57,17986,738],{"className":17987,"style":3907},[1075,1148,1112],[57,17989,17991],{"className":17990},[1152],[57,17992,17994,18015],{"className":17993},[1087,1088],[57,17995,17997,18012],{"className":17996},[1092],[57,17998,18000],{"className":17999,"style":5022},[1096],[57,18001,18003,18006],{"style":18002},"top:-2.357em;margin-left:-0.1389em;margin-right:0.0714em;",[57,18004],{"className":18005,"style":1169},[1104],[57,18007,18009],{"className":18008},[1109,1173,1174,1112],[57,18010,1288],{"className":18011},[1075,1148,1112],[57,18013,1183],{"className":18014},[1182],[57,18016,18018],{"className":18017},[1092],[57,18019,18021],{"className":18020,"style":5044},[1096],[57,18022],{},[57,18024,1183],{"className":18025},[1182],[57,18027,18029],{"className":18028},[1092],[57,18030,18032],{"className":18031,"style":17183},[1096],[57,18033],{},[57,18035],{"className":18036,"style":1853},[1452],[57,18038,18040,18043],{"className":18039},[1075],[57,18041,6333],{"className":18042,"style":3907},[1075,1148],[57,18044,18046],{"className":18045},[1152],[57,18047,18049,18069],{"className":18048},[1087,1088],[57,18050,18052,18066],{"className":18051},[1092],[57,18053,18055],{"className":18054,"style":2208},[1096],[57,18056,18057,18060],{"style":8131},[57,18058],{"className":18059,"style":1493},[1104],[57,18061,18063],{"className":18062},[1109,1110,1111,1112],[57,18064,14745],{"className":18065,"style":3932},[1075,1148,1112],[57,18067,1183],{"className":18068},[1182],[57,18070,18072],{"className":18071},[1092],[57,18073,18075],{"className":18074,"style":2230},[1096],[57,18076],{},[57,18078,1754],{"className":18079},[1079],[57,18081,18083,18086],{"className":18082},[1075],[57,18084,18],{"className":18085},[1075,1148],[57,18087,18089],{"className":18088},[1152],[57,18090,18092,18125],{"className":18091},[1087,1088],[57,18093,18095,18122],{"className":18094},[1092],[57,18096,18099,18111],{"className":18097,"style":18098},[1096],"height:0.8436em;",[57,18100,18102,18105],{"style":18101},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[57,18103],{"className":18104,"style":1493},[1104],[57,18106,18108],{"className":18107},[1109,1110,1111,1112],[57,18109,1288],{"className":18110},[1075,1148,1112],[57,18112,18113,18116],{"style":17216},[57,18114],{"className":18115,"style":1493},[1104],[57,18117,18119],{"className":18118},[1109,1110,1111,1112],[57,18120,1733],{"className":18121},[1075,1148,1112],[57,18123,1183],{"className":18124},[1182],[57,18126,18128],{"className":18127},[1092],[57,18129,18132],{"className":18130,"style":18131},[1096],"height:0.247em;",[57,18133],{},[57,18135,13675],{"className":18136},[1075],[57,18138,18140,18143],{"className":18139},[1075],[57,18141,13535],{"className":18142},[1075,1148],[57,18144,18146],{"className":18145},[1152],[57,18147,18149,18180],{"className":18148},[1087,1088],[57,18150,18152,18177],{"className":18151},[1092],[57,18153,18155,18166],{"className":18154,"style":18098},[1096],[57,18156,18157,18160],{"style":18101},[57,18158],{"className":18159,"style":1493},[1104],[57,18161,18163],{"className":18162},[1109,1110,1111,1112],[57,18164,1288],{"className":18165},[1075,1148,1112],[57,18167,18168,18171],{"style":17216},[57,18169],{"className":18170,"style":1493},[1104],[57,18172,18174],{"className":18173},[1109,1110,1111,1112],[57,18175,1733],{"className":18176},[1075,1148,1112],[57,18178,1183],{"className":18179},[1182],[57,18181,18183],{"className":18182},[1092],[57,18184,18186],{"className":18185,"style":18131},[1096],[57,18187],{},[57,18189,1768],{"className":18190},[1269],[57,18192,18193,18196],{"style":16135},[57,18194],{"className":18195,"style":15959},[1104],[57,18197,18199,18202,18205,18208,18211,18273,18276,18340,18343,18346,18349,18378,18381,18384,18488,18491,18494,18497,18502,18505,18545,18548,18599,18602,18653],{"className":18198},[1075],[57,18200],{"className":18201},[1075],[57,18203],{"className":18204,"style":1453},[1452],[57,18206,281],{"className":18207},[1457],[57,18209],{"className":18210,"style":1453},[1452],[57,18212,18214,18217,18270],{"className":18213},[1075],[57,18215],{"className":18216},[1079,1080],[57,18218,18220],{"className":18219},[1035],[57,18221,18223,18262],{"className":18222},[1087,1088],[57,18224,18226,18259],{"className":18225},[1092],[57,18227,18229,18240,18248],{"className":18228,"style":17068},[1096],[57,18230,18231,18234],{"style":16608},[57,18232],{"className":18233,"style":1105},[1104],[57,18235,18237],{"className":18236},[1075],[57,18238,7990],{"className":18239,"style":17080},[1075,1148],[57,18241,18242,18245],{"style":1230},[57,18243],{"className":18244,"style":1105},[1104],[57,18246],{"className":18247,"style":1238},[1237],[57,18249,18250,18253],{"style":16674},[57,18251],{"className":18252,"style":1105},[1104],[57,18254,18256],{"className":18255},[1075],[57,18257,574],{"className":18258},[1075],[57,18260,1183],{"className":18261},[1182],[57,18263,18265],{"className":18264},[1092],[57,18266,18268],{"className":18267,"style":17109},[1096],[57,18269],{},[57,18271],{"className":18272},[1269,1080],[57,18274],{"className":18275,"style":1853},[1452],[57,18277,18279],{"className":18278},[1888,15027],[57,18280,18282,18332],{"className":18281},[1087,1088],[57,18283,18285,18329],{"className":18284},[1092],[57,18286,18288,18308,18318],{"className":18287,"style":17130},[1096],[57,18289,18290,18293],{"style":17133},[57,18291],{"className":18292,"style":16221},[1104],[57,18294,18296],{"className":18295},[1109,1110,1111,1112],[57,18297,18299,18302,18305],{"className":18298},[1075,1112],[57,18300,1288],{"className":18301},[1075,1148,1112],[57,18303,281],{"className":18304},[1457,1112],[57,18306,574],{"className":18307},[1075,1112],[57,18309,18310,18313],{"style":16230},[57,18311],{"className":18312,"style":16221},[1104],[57,18314,18315],{},[57,18316,2549],{"className":18317},[1888,2633,16239],[57,18319,18320,18323],{"style":17164},[57,18321],{"className":18322,"style":16221},[1104],[57,18324,18326],{"className":18325},[1109,1110,1111,1112],[57,18327,7990],{"className":18328,"style":17080},[1075,1148,1112],[57,18330,1183],{"className":18331},[1182],[57,18333,18335],{"className":18334},[1092],[57,18336,18338],{"className":18337,"style":17183},[1096],[57,18339],{},[57,18341],{"className":18342,"style":1853},[1452],[57,18344,1423],{"className":18345,"style":14898},[1075,1148],[57,18347,1754],{"className":18348},[1079],[57,18350,18352,18355],{"className":18351},[1075],[57,18353,13919],{"className":18354,"style":13935},[1075,1148],[57,18356,18358],{"className":18357},[1152],[57,18359,18361],{"className":18360},[1087],[57,18362,18364],{"className":18363},[1092],[57,18365,18367],{"className":18366,"style":17213},[1096],[57,18368,18369,18372],{"style":17216},[57,18370],{"className":18371,"style":1493},[1104],[57,18373,18375],{"className":18374},[1109,1110,1111,1112],[57,18376,1288],{"className":18377},[1075,1148,1112],[57,18379,1768],{"className":18380},[1269],[57,18382],{"className":18383,"style":1853},[1452],[57,18385,18387],{"className":18386},[1888,15027],[57,18388,18390,18480],{"className":18389},[1087,1088],[57,18391,18393,18477],{"className":18392},[1092],[57,18394,18396,18416,18426],{"className":18395,"style":17939},[1096],[57,18397,18398,18401],{"style":17133},[57,18399],{"className":18400,"style":16221},[1104],[57,18402,18404],{"className":18403},[1109,1110,1111,1112],[57,18405,18407,18410,18413],{"className":18406},[1075,1112],[57,18408,1733],{"className":18409},[1075,1148,1112],[57,18411,281],{"className":18412},[1457,1112],[57,18414,574],{"className":18415},[1075,1112],[57,18417,18418,18421],{"style":16230},[57,18419],{"className":18420,"style":16221},[1104],[57,18422,18423],{},[57,18424,2549],{"className":18425},[1888,2633,16239],[57,18427,18428,18431],{"style":17972},[57,18429],{"className":18430,"style":16221},[1104],[57,18432,18434],{"className":18433},[1109,1110,1111,1112],[57,18435,18437],{"className":18436},[1075,1112],[57,18438,18440,18443],{"className":18439},[1075,1112],[57,18441,738],{"className":18442,"style":3907},[1075,1148,1112],[57,18444,18446],{"className":18445},[1152],[57,18447,18449,18469],{"className":18448},[1087,1088],[57,18450,18452,18466],{"className":18451},[1092],[57,18453,18455],{"className":18454,"style":5022},[1096],[57,18456,18457,18460],{"style":18002},[57,18458],{"className":18459,"style":1169},[1104],[57,18461,18463],{"className":18462},[1109,1173,1174,1112],[57,18464,1288],{"className":18465},[1075,1148,1112],[57,18467,1183],{"className":18468},[1182],[57,18470,18472],{"className":18471},[1092],[57,18473,18475],{"className":18474,"style":5044},[1096],[57,18476],{},[57,18478,1183],{"className":18479},[1182],[57,18481,18483],{"className":18482},[1092],[57,18484,18486],{"className":18485,"style":17183},[1096],[57,18487],{},[57,18489],{"className":18490,"style":1853},[1452],[57,18492,15169],{"className":18493},[1075],[57,18495],{"className":18496,"style":1853},[1452],[57,18498,17640,18500],{"className":18499},[1888],[57,18501,17644],{"style":17643},[57,18503],{"className":18504,"style":1853},[1452],[57,18506,18508,18511],{"className":18507},[1075],[57,18509,6333],{"className":18510,"style":3907},[1075,1148],[57,18512,18514],{"className":18513},[1152],[57,18515,18517,18537],{"className":18516},[1087,1088],[57,18518,18520,18534],{"className":18519},[1092],[57,18521,18523],{"className":18522,"style":2208},[1096],[57,18524,18525,18528],{"style":8131},[57,18526],{"className":18527,"style":1493},[1104],[57,18529,18531],{"className":18530},[1109,1110,1111,1112],[57,18532,14745],{"className":18533,"style":3932},[1075,1148,1112],[57,18535,1183],{"className":18536},[1182],[57,18538,18540],{"className":18539},[1092],[57,18541,18543],{"className":18542,"style":2230},[1096],[57,18544],{},[57,18546,1754],{"className":18547},[1079],[57,18549,18551,18554],{"className":18550},[1075],[57,18552,18],{"className":18553},[1075,1148],[57,18555,18557],{"className":18556},[1152],[57,18558,18560,18591],{"className":18559},[1087,1088],[57,18561,18563,18588],{"className":18562},[1092],[57,18564,18566,18577],{"className":18565,"style":18098},[1096],[57,18567,18568,18571],{"style":18101},[57,18569],{"className":18570,"style":1493},[1104],[57,18572,18574],{"className":18573},[1109,1110,1111,1112],[57,18575,1288],{"className":18576},[1075,1148,1112],[57,18578,18579,18582],{"style":17216},[57,18580],{"className":18581,"style":1493},[1104],[57,18583,18585],{"className":18584},[1109,1110,1111,1112],[57,18586,1733],{"className":18587},[1075,1148,1112],[57,18589,1183],{"className":18590},[1182],[57,18592,18594],{"className":18593},[1092],[57,18595,18597],{"className":18596,"style":18131},[1096],[57,18598],{},[57,18600,13675],{"className":18601},[1075],[57,18603,18605,18608],{"className":18604},[1075],[57,18606,13535],{"className":18607},[1075,1148],[57,18609,18611],{"className":18610},[1152],[57,18612,18614,18645],{"className":18613},[1087,1088],[57,18615,18617,18642],{"className":18616},[1092],[57,18618,18620,18631],{"className":18619,"style":18098},[1096],[57,18621,18622,18625],{"style":18101},[57,18623],{"className":18624,"style":1493},[1104],[57,18626,18628],{"className":18627},[1109,1110,1111,1112],[57,18629,1288],{"className":18630},[1075,1148,1112],[57,18632,18633,18636],{"style":17216},[57,18634],{"className":18635,"style":1493},[1104],[57,18637,18639],{"className":18638},[1109,1110,1111,1112],[57,18640,1733],{"className":18641},[1075,1148,1112],[57,18643,1183],{"className":18644},[1182],[57,18646,18648],{"className":18647},[1092],[57,18649,18651],{"className":18650,"style":18131},[1096],[57,18652],{},[57,18654,1768],{"className":18655},[1269],[57,18657,18658,18661],{"style":16144},[57,18659],{"className":18660,"style":15959},[1104],[57,18662,18664,18667,18670,18673,18676,18738,18741,18805,18808,18912,18915,18918,18921,18950,18953,18956,18959,18964,18967,19007,19010,19061,19064,19115],{"className":18663},[1075],[57,18665],{"className":18666},[1075],[57,18668],{"className":18669,"style":1453},[1452],[57,18671,281],{"className":18672},[1457],[57,18674],{"className":18675,"style":1453},[1452],[57,18677,18679,18682,18735],{"className":18678},[1075],[57,18680],{"className":18681},[1079,1080],[57,18683,18685],{"className":18684},[1035],[57,18686,18688,18727],{"className":18687},[1087,1088],[57,18689,18691,18724],{"className":18690},[1092],[57,18692,18694,18705,18713],{"className":18693,"style":17068},[1096],[57,18695,18696,18699],{"style":16608},[57,18697],{"className":18698,"style":1105},[1104],[57,18700,18702],{"className":18701},[1075],[57,18703,7990],{"className":18704,"style":17080},[1075,1148],[57,18706,18707,18710],{"style":1230},[57,18708],{"className":18709,"style":1105},[1104],[57,18711],{"className":18712,"style":1238},[1237],[57,18714,18715,18718],{"style":16674},[57,18716],{"className":18717,"style":1105},[1104],[57,18719,18721],{"className":18720},[1075],[57,18722,574],{"className":18723},[1075],[57,18725,1183],{"className":18726},[1182],[57,18728,18730],{"className":18729},[1092],[57,18731,18733],{"className":18732,"style":17109},[1096],[57,18734],{},[57,18736],{"className":18737},[1269,1080],[57,18739],{"className":18740,"style":1853},[1452],[57,18742,18744],{"className":18743},[1888,15027],[57,18745,18747,18797],{"className":18746},[1087,1088],[57,18748,18750,18794],{"className":18749},[1092],[57,18751,18753,18773,18783],{"className":18752,"style":17130},[1096],[57,18754,18755,18758],{"style":17133},[57,18756],{"className":18757,"style":16221},[1104],[57,18759,18761],{"className":18760},[1109,1110,1111,1112],[57,18762,18764,18767,18770],{"className":18763},[1075,1112],[57,18765,1288],{"className":18766},[1075,1148,1112],[57,18768,281],{"className":18769},[1457,1112],[57,18771,574],{"className":18772},[1075,1112],[57,18774,18775,18778],{"style":16230},[57,18776],{"className":18777,"style":16221},[1104],[57,18779,18780],{},[57,18781,2549],{"className":18782},[1888,2633,16239],[57,18784,18785,18788],{"style":17164},[57,18786],{"className":18787,"style":16221},[1104],[57,18789,18791],{"className":18790},[1109,1110,1111,1112],[57,18792,7990],{"className":18793,"style":17080},[1075,1148,1112],[57,18795,1183],{"className":18796},[1182],[57,18798,18800],{"className":18799},[1092],[57,18801,18803],{"className":18802,"style":17183},[1096],[57,18804],{},[57,18806],{"className":18807,"style":1853},[1452],[57,18809,18811],{"className":18810},[1888,15027],[57,18812,18814,18904],{"className":18813},[1087,1088],[57,18815,18817,18901],{"className":18816},[1092],[57,18818,18820,18840,18850],{"className":18819,"style":17939},[1096],[57,18821,18822,18825],{"style":17133},[57,18823],{"className":18824,"style":16221},[1104],[57,18826,18828],{"className":18827},[1109,1110,1111,1112],[57,18829,18831,18834,18837],{"className":18830},[1075,1112],[57,18832,1733],{"className":18833},[1075,1148,1112],[57,18835,281],{"className":18836},[1457,1112],[57,18838,574],{"className":18839},[1075,1112],[57,18841,18842,18845],{"style":16230},[57,18843],{"className":18844,"style":16221},[1104],[57,18846,18847],{},[57,18848,2549],{"className":18849},[1888,2633,16239],[57,18851,18852,18855],{"style":17972},[57,18853],{"className":18854,"style":16221},[1104],[57,18856,18858],{"className":18857},[1109,1110,1111,1112],[57,18859,18861],{"className":18860},[1075,1112],[57,18862,18864,18867],{"className":18863},[1075,1112],[57,18865,738],{"className":18866,"style":3907},[1075,1148,1112],[57,18868,18870],{"className":18869},[1152],[57,18871,18873,18893],{"className":18872},[1087,1088],[57,18874,18876,18890],{"className":18875},[1092],[57,18877,18879],{"className":18878,"style":5022},[1096],[57,18880,18881,18884],{"style":18002},[57,18882],{"className":18883,"style":1169},[1104],[57,18885,18887],{"className":18886},[1109,1173,1174,1112],[57,18888,1288],{"className":18889},[1075,1148,1112],[57,18891,1183],{"className":18892},[1182],[57,18894,18896],{"className":18895},[1092],[57,18897,18899],{"className":18898,"style":5044},[1096],[57,18900],{},[57,18902,1183],{"className":18903},[1182],[57,18905,18907],{"className":18906},[1092],[57,18908,18910],{"className":18909,"style":17183},[1096],[57,18911],{},[57,18913],{"className":18914,"style":1853},[1452],[57,18916,1423],{"className":18917,"style":14898},[1075,1148],[57,18919,1754],{"className":18920},[1079],[57,18922,18924,18927],{"className":18923},[1075],[57,18925,13919],{"className":18926,"style":13935},[1075,1148],[57,18928,18930],{"className":18929},[1152],[57,18931,18933],{"className":18932},[1087],[57,18934,18936],{"className":18935},[1092],[57,18937,18939],{"className":18938,"style":17213},[1096],[57,18940,18941,18944],{"style":17216},[57,18942],{"className":18943,"style":1493},[1104],[57,18945,18947],{"className":18946},[1109,1110,1111,1112],[57,18948,1288],{"className":18949},[1075,1148,1112],[57,18951,1768],{"className":18952},[1269],[57,18954,15169],{"className":18955},[1075],[57,18957],{"className":18958,"style":1853},[1452],[57,18960,17640,18962],{"className":18961},[1888],[57,18963,17644],{"style":17643},[57,18965],{"className":18966,"style":1853},[1452],[57,18968,18970,18973],{"className":18969},[1075],[57,18971,6333],{"className":18972,"style":3907},[1075,1148],[57,18974,18976],{"className":18975},[1152],[57,18977,18979,18999],{"className":18978},[1087,1088],[57,18980,18982,18996],{"className":18981},[1092],[57,18983,18985],{"className":18984,"style":2208},[1096],[57,18986,18987,18990],{"style":8131},[57,18988],{"className":18989,"style":1493},[1104],[57,18991,18993],{"className":18992},[1109,1110,1111,1112],[57,18994,14745],{"className":18995,"style":3932},[1075,1148,1112],[57,18997,1183],{"className":18998},[1182],[57,19000,19002],{"className":19001},[1092],[57,19003,19005],{"className":19004,"style":2230},[1096],[57,19006],{},[57,19008,1754],{"className":19009},[1079],[57,19011,19013,19016],{"className":19012},[1075],[57,19014,18],{"className":19015},[1075,1148],[57,19017,19019],{"className":19018},[1152],[57,19020,19022,19053],{"className":19021},[1087,1088],[57,19023,19025,19050],{"className":19024},[1092],[57,19026,19028,19039],{"className":19027,"style":18098},[1096],[57,19029,19030,19033],{"style":18101},[57,19031],{"className":19032,"style":1493},[1104],[57,19034,19036],{"className":19035},[1109,1110,1111,1112],[57,19037,1288],{"className":19038},[1075,1148,1112],[57,19040,19041,19044],{"style":17216},[57,19042],{"className":19043,"style":1493},[1104],[57,19045,19047],{"className":19046},[1109,1110,1111,1112],[57,19048,1733],{"className":19049},[1075,1148,1112],[57,19051,1183],{"className":19052},[1182],[57,19054,19056],{"className":19055},[1092],[57,19057,19059],{"className":19058,"style":18131},[1096],[57,19060],{},[57,19062,13675],{"className":19063},[1075],[57,19065,19067,19070],{"className":19066},[1075],[57,19068,13535],{"className":19069},[1075,1148],[57,19071,19073],{"className":19072},[1152],[57,19074,19076,19107],{"className":19075},[1087,1088],[57,19077,19079,19104],{"className":19078},[1092],[57,19080,19082,19093],{"className":19081,"style":18098},[1096],[57,19083,19084,19087],{"style":18101},[57,19085],{"className":19086,"style":1493},[1104],[57,19088,19090],{"className":19089},[1109,1110,1111,1112],[57,19091,1288],{"className":19092},[1075,1148,1112],[57,19094,19095,19098],{"style":17216},[57,19096],{"className":19097,"style":1493},[1104],[57,19099,19101],{"className":19100},[1109,1110,1111,1112],[57,19102,1733],{"className":19103},[1075,1148,1112],[57,19105,1183],{"className":19106},[1182],[57,19108,19110],{"className":19109},[1092],[57,19111,19113],{"className":19112,"style":18131},[1096],[57,19114],{},[57,19116,1768],{"className":19117},[1269],[57,19119,1183],{"className":19120},[1182],[57,19122,19124],{"className":19123},[1092],[57,19125,19127],{"className":19126,"style":16160},[1096],[57,19128],{},[14,19130,19131],{},[57,19132,19134,19241],{"className":19133},[1019],[57,19135,19137],{"className":19136},[1023],[1025,19138,19139],{"xmlns":1027},[1029,19140,19141,19238],{},[1032,19142,19143,19145,19147,19149,19151,19153,19156,19162,19176,19194,19196,19198,19204,19206,19208,19210,19216,19218,19226,19228,19236],{},[1047,19144,8490],{},[1047,19146,1748],{},[1047,19148,13535],{},[1047,19150,13535],{},[1414,19152,281],{},[1414,19154,19155],{},"−",[1035,19157,19158,19160],{},[1038,19159,574],{},[1047,19161,7990],{},[2545,19163,19164,19166,19174],{},[1414,19165,2549],{},[1032,19167,19168,19170,19172],{},[1047,19169,1288],{},[1414,19171,281],{},[1038,19173,574],{},[1047,19175,7990],{},[2545,19177,19178,19180,19188],{},[1414,19179,2549],{},[1032,19181,19182,19184,19186],{},[1047,19183,1733],{},[1414,19185,281],{},[1038,19187,574],{},[1044,19189,19190,19192],{},[1047,19191,738],{},[1047,19193,1288],{},[1047,19195,1423],{},[1414,19197,1754],{"stretchy":1753},[1418,19199,19200,19202],{},[1047,19201,13919],{},[1047,19203,1288],{},[1414,19205,1768],{"stretchy":1753},[1047,19207,15594],{},[1414,19209,1776],{},[1044,19211,19212,19214],{},[1047,19213,6333],{},[1047,19215,14745],{},[1414,19217,1754],{"stretchy":1753},[2545,19219,19220,19222,19224],{},[1047,19221,18],{},[1047,19223,1288],{},[1047,19225,1733],{},[1047,19227,13675],{"mathvariant":1729},[2545,19229,19230,19232,19234],{},[1047,19231,13535],{},[1047,19233,1288],{},[1047,19235,1733],{},[1414,19237,1768],{"stretchy":1753},[1054,19239,19240],{"encoding":1056},"Loss = -\\frac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} R(\\tau^n) \\log P_\\theta(a_n^t|s_n^t)",[57,19242,19244,19267],{"className":19243,"ariaHidden":1062},[1061],[57,19245,19247,19251,19254,19258,19261,19264],{"className":19246},[1066],[57,19248],{"className":19249,"style":19250},[1070],"height:0.6833em;",[57,19252,8490],{"className":19253},[1075,1148],[57,19255,19257],{"className":19256},[1075,1148],"oss",[57,19259],{"className":19260,"style":1453},[1452],[57,19262,281],{"className":19263},[1457],[57,19265],{"className":19266,"style":1453},[1452],[57,19268,19270,19274,19277,19347,19350,19418,19421,19527,19530,19533,19536,19565,19568,19571,19576,19579,19619,19622,19674,19677,19728],{"className":19269},[1066],[57,19271],{"className":19272,"style":19273},[1070],"height:2.5065em;vertical-align:-0.9671em;",[57,19275,19155],{"className":19276},[1075],[57,19278,19280,19283,19344],{"className":19279},[1075],[57,19281],{"className":19282},[1079,1080],[57,19284,19286],{"className":19285},[1035],[57,19287,19289,19335],{"className":19288},[1087,1088],[57,19290,19292,19332],{"className":19291},[1092],[57,19293,19295,19310,19318],{"className":19294,"style":1097},[1096],[57,19296,19298,19301],{"style":19297},"top:-2.655em;",[57,19299],{"className":19300,"style":1105},[1104],[57,19302,19304],{"className":19303},[1109,1110,1111,1112],[57,19305,19307],{"className":19306},[1075,1112],[57,19308,7990],{"className":19309,"style":17080},[1075,1148,1112],[57,19311,19312,19315],{"style":1230},[57,19313],{"className":19314,"style":1105},[1104],[57,19316],{"className":19317,"style":1238},[1237],[57,19319,19320,19323],{"style":1241},[57,19321],{"className":19322,"style":1105},[1104],[57,19324,19326],{"className":19325},[1109,1110,1111,1112],[57,19327,19329],{"className":19328},[1075,1112],[57,19330,574],{"className":19331},[1075,1112],[57,19333,1183],{"className":19334},[1182],[57,19336,19338],{"className":19337},[1092],[57,19339,19342],{"className":19340,"style":19341},[1096],"height:0.345em;",[57,19343],{},[57,19345],{"className":19346},[1269,1080],[57,19348],{"className":19349,"style":1853},[1452],[57,19351,19353],{"className":19352},[1888,15027],[57,19354,19356,19409],{"className":19355},[1087,1088],[57,19357,19359,19406],{"className":19358},[1092],[57,19360,19363,19384,19394],{"className":19361,"style":19362},[1096],"height:1.5283em;",[57,19364,19366,19369],{"style":19365},"top:-2.1329em;margin-left:0em;",[57,19367],{"className":19368,"style":1105},[1104],[57,19370,19372],{"className":19371},[1109,1110,1111,1112],[57,19373,19375,19378,19381],{"className":19374},[1075,1112],[57,19376,1288],{"className":19377},[1075,1148,1112],[57,19379,281],{"className":19380},[1457,1112],[57,19382,574],{"className":19383},[1075,1112],[57,19385,19386,19389],{"style":1134},[57,19387],{"className":19388,"style":1105},[1104],[57,19390,19391],{},[57,19392,2549],{"className":19393},[1888,2633,2634],[57,19395,19397,19400],{"style":19396},"top:-3.95em;margin-left:0em;",[57,19398],{"className":19399,"style":1105},[1104],[57,19401,19403],{"className":19402},[1109,1110,1111,1112],[57,19404,7990],{"className":19405,"style":17080},[1075,1148,1112],[57,19407,1183],{"className":19408},[1182],[57,19410,19412],{"className":19411},[1092],[57,19413,19416],{"className":19414,"style":19415},[1096],"height:0.9671em;",[57,19417],{},[57,19419],{"className":19420,"style":1853},[1452],[57,19422,19424],{"className":19423},[1888,15027],[57,19425,19427,19519],{"className":19426},[1087,1088],[57,19428,19430,19516],{"className":19429},[1092],[57,19431,19434,19454,19464],{"className":19432,"style":19433},[1096],"height:1.5394em;",[57,19435,19436,19439],{"style":19365},[57,19437],{"className":19438,"style":1105},[1104],[57,19440,19442],{"className":19441},[1109,1110,1111,1112],[57,19443,19445,19448,19451],{"className":19444},[1075,1112],[57,19446,1733],{"className":19447},[1075,1148,1112],[57,19449,281],{"className":19450},[1457,1112],[57,19452,574],{"className":19453},[1075,1112],[57,19455,19456,19459],{"style":1134},[57,19457],{"className":19458,"style":1105},[1104],[57,19460,19461],{},[57,19462,2549],{"className":19463},[1888,2633,2634],[57,19465,19467,19470],{"style":19466},"top:-3.9611em;margin-left:0em;",[57,19468],{"className":19469,"style":1105},[1104],[57,19471,19473],{"className":19472},[1109,1110,1111,1112],[57,19474,19476],{"className":19475},[1075,1112],[57,19477,19479,19482],{"className":19478},[1075,1112],[57,19480,738],{"className":19481,"style":3907},[1075,1148,1112],[57,19483,19485],{"className":19484},[1152],[57,19486,19488,19508],{"className":19487},[1087,1088],[57,19489,19491,19505],{"className":19490},[1092],[57,19492,19494],{"className":19493,"style":5022},[1096],[57,19495,19496,19499],{"style":18002},[57,19497],{"className":19498,"style":1169},[1104],[57,19500,19502],{"className":19501},[1109,1173,1174,1112],[57,19503,1288],{"className":19504},[1075,1148,1112],[57,19506,1183],{"className":19507},[1182],[57,19509,19511],{"className":19510},[1092],[57,19512,19514],{"className":19513,"style":5044},[1096],[57,19515],{},[57,19517,1183],{"className":19518},[1182],[57,19520,19522],{"className":19521},[1092],[57,19523,19525],{"className":19524,"style":19415},[1096],[57,19526],{},[57,19528],{"className":19529,"style":1853},[1452],[57,19531,1423],{"className":19532,"style":14898},[1075,1148],[57,19534,1754],{"className":19535},[1079],[57,19537,19539,19542],{"className":19538},[1075],[57,19540,13919],{"className":19541,"style":13935},[1075,1148],[57,19543,19545],{"className":19544},[1152],[57,19546,19548],{"className":19547},[1087],[57,19549,19551],{"className":19550},[1092],[57,19552,19554],{"className":19553,"style":17412},[1096],[57,19555,19556,19559],{"style":1489},[57,19557],{"className":19558,"style":1493},[1104],[57,19560,19562],{"className":19561},[1109,1110,1111,1112],[57,19563,1288],{"className":19564},[1075,1148,1112],[57,19566,1768],{"className":19567},[1269],[57,19569],{"className":19570,"style":1853},[1452],[57,19572,17640,19574],{"className":19573},[1888],[57,19575,17644],{"style":17643},[57,19577],{"className":19578,"style":1853},[1452],[57,19580,19582,19585],{"className":19581},[1075],[57,19583,6333],{"className":19584,"style":3907},[1075,1148],[57,19586,19588],{"className":19587},[1152],[57,19589,19591,19611],{"className":19590},[1087,1088],[57,19592,19594,19608],{"className":19593},[1092],[57,19595,19597],{"className":19596,"style":2208},[1096],[57,19598,19599,19602],{"style":8131},[57,19600],{"className":19601,"style":1493},[1104],[57,19603,19605],{"className":19604},[1109,1110,1111,1112],[57,19606,14745],{"className":19607,"style":3932},[1075,1148,1112],[57,19609,1183],{"className":19610},[1182],[57,19612,19614],{"className":19613},[1092],[57,19615,19617],{"className":19616,"style":2230},[1096],[57,19618],{},[57,19620,1754],{"className":19621},[1079],[57,19623,19625,19628],{"className":19624},[1075],[57,19626,18],{"className":19627},[1075,1148],[57,19629,19631],{"className":19630},[1152],[57,19632,19634,19666],{"className":19633},[1087,1088],[57,19635,19637,19663],{"className":19636},[1092],[57,19638,19641,19652],{"className":19639,"style":19640},[1096],"height:0.7936em;",[57,19642,19643,19646],{"style":18101},[57,19644],{"className":19645,"style":1493},[1104],[57,19647,19649],{"className":19648},[1109,1110,1111,1112],[57,19650,1288],{"className":19651},[1075,1148,1112],[57,19653,19654,19657],{"style":1489},[57,19655],{"className":19656,"style":1493},[1104],[57,19658,19660],{"className":19659},[1109,1110,1111,1112],[57,19661,1733],{"className":19662},[1075,1148,1112],[57,19664,1183],{"className":19665},[1182],[57,19667,19669],{"className":19668},[1092],[57,19670,19672],{"className":19671,"style":18131},[1096],[57,19673],{},[57,19675,13675],{"className":19676},[1075],[57,19678,19680,19683],{"className":19679},[1075],[57,19681,13535],{"className":19682},[1075,1148],[57,19684,19686],{"className":19685},[1152],[57,19687,19689,19720],{"className":19688},[1087,1088],[57,19690,19692,19717],{"className":19691},[1092],[57,19693,19695,19706],{"className":19694,"style":19640},[1096],[57,19696,19697,19700],{"style":18101},[57,19698],{"className":19699,"style":1493},[1104],[57,19701,19703],{"className":19702},[1109,1110,1111,1112],[57,19704,1288],{"className":19705},[1075,1148,1112],[57,19707,19708,19711],{"style":1489},[57,19709],{"className":19710,"style":1493},[1104],[57,19712,19714],{"className":19713},[1109,1110,1111,1112],[57,19715,1733],{"className":19716},[1075,1148,1112],[57,19718,1183],{"className":19719},[1182],[57,19721,19723],{"className":19722},[1092],[57,19724,19726],{"className":19725,"style":18131},[1096],[57,19727],{},[57,19729,1768],{"className":19730},[1269],[14,19732,19733],{},[57,19734,19736,19824],{"className":19735},[1019],[57,19737,19739],{"className":19738},[1023],[1025,19740,19741],{"xmlns":1027},[1029,19742,19743,19821],{},[1032,19744,19745,19747,19749,19755,19757,19759,19782,19799,19811,19813],{},[1047,19746,1423],{},[1414,19748,1754],{"stretchy":1753},[1418,19750,19751,19753],{},[1047,19752,13919],{},[1047,19754,1288],{},[1414,19756,1768],{"stretchy":1753},[1414,19758,281],{},[2545,19760,19761,19763,19776],{},[1414,19762,2549],{},[1032,19764,19765,19767,19769],{},[1047,19766,1733],{},[1414,19768,281],{},[1418,19770,19771,19773],{},[1047,19772,1733],{},[1414,19774,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},"′",[1044,19777,19778,19780],{},[1047,19779,738],{},[1047,19781,1288],{},[1418,19783,19784,19787],{},[1047,19785,19786],{},"γ",[1032,19788,19789,19791,19793],{},[1047,19790,1733],{},[1414,19792,19155],{},[1418,19794,19795,19797],{},[1047,19796,1733],{},[1414,19798,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[2545,19800,19801,19803,19809],{},[1047,19802,8500],{},[1418,19804,19805,19807],{},[1047,19806,1733],{},[1414,19808,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1047,19810,1288],{},[1414,19812,281],{},[2545,19814,19815,19817,19819],{},[1047,19816,1423],{},[1047,19818,1733],{},[1047,19820,1288],{},[1054,19822,19823],{"encoding":1056},"R(\\tau^n) = \\sum\\limits_{t=t'}^{T_n} \\gamma^{t-t'} r_{t'}^n = R_t^n",[57,19825,19827,19880,20191],{"className":19826,"ariaHidden":1062},[1061],[57,19828,19830,19833,19836,19839,19868,19871,19874,19877],{"className":19829},[1066],[57,19831],{"className":19832,"style":1831},[1070],[57,19834,1423],{"className":19835,"style":14898},[1075,1148],[57,19837,1754],{"className":19838},[1079],[57,19840,19842,19845],{"className":19841},[1075],[57,19843,13919],{"className":19844,"style":13935},[1075,1148],[57,19846,19848],{"className":19847},[1152],[57,19849,19851],{"className":19850},[1087],[57,19852,19854],{"className":19853},[1092],[57,19855,19857],{"className":19856,"style":17412},[1096],[57,19858,19859,19862],{"style":1489},[57,19860],{"className":19861,"style":1493},[1104],[57,19863,19865],{"className":19864},[1109,1110,1111,1112],[57,19866,1288],{"className":19867},[1075,1148,1112],[57,19869,1768],{"className":19870},[1269],[57,19872],{"className":19873,"style":1453},[1452],[57,19875,281],{"className":19876},[1457],[57,19878],{"className":19879,"style":1453},[1452],[57,19881,19883,19887,20024,20027,20097,20182,20185,20188],{"className":19882},[1066],[57,19884],{"className":19885,"style":19886},[1070],"height:2.5334em;vertical-align:-0.994em;",[57,19888,19890],{"className":19889},[1888,15027],[57,19891,19893,20015],{"className":19892},[1087,1088],[57,19894,19896,20012],{"className":19895},[1092],[57,19897,19899,19951,19961],{"className":19898,"style":19433},[1096],[57,19900,19902,19905],{"style":19901},"top:-2.106em;margin-left:0em;",[57,19903],{"className":19904,"style":1105},[1104],[57,19906,19908],{"className":19907},[1109,1110,1111,1112],[57,19909,19911,19914,19917],{"className":19910},[1075,1112],[57,19912,1733],{"className":19913},[1075,1148,1112],[57,19915,281],{"className":19916},[1457,1112],[57,19918,19920,19923],{"className":19919},[1075,1112],[57,19921,1733],{"className":19922},[1075,1148,1112],[57,19924,19926],{"className":19925},[1152],[57,19927,19929],{"className":19928},[1087],[57,19930,19932],{"className":19931},[1092],[57,19933,19936],{"className":19934,"style":19935},[1096],"height:0.6828em;",[57,19937,19939,19942],{"style":19938},"top:-2.786em;margin-right:0.0714em;",[57,19940],{"className":19941,"style":1169},[1104],[57,19943,19945],{"className":19944},[1109,1173,1174,1112],[57,19946,19948],{"className":19947},[1075,1112],[57,19949,19775],{"className":19950},[1075,1112],[57,19952,19953,19956],{"style":1134},[57,19954],{"className":19955,"style":1105},[1104],[57,19957,19958],{},[57,19959,2549],{"className":19960},[1888,2633,2634],[57,19962,19963,19966],{"style":19466},[57,19964],{"className":19965,"style":1105},[1104],[57,19967,19969],{"className":19968},[1109,1110,1111,1112],[57,19970,19972],{"className":19971},[1075,1112],[57,19973,19975,19978],{"className":19974},[1075,1112],[57,19976,738],{"className":19977,"style":3907},[1075,1148,1112],[57,19979,19981],{"className":19980},[1152],[57,19982,19984,20004],{"className":19983},[1087,1088],[57,19985,19987,20001],{"className":19986},[1092],[57,19988,19990],{"className":19989,"style":5022},[1096],[57,19991,19992,19995],{"style":18002},[57,19993],{"className":19994,"style":1169},[1104],[57,19996,19998],{"className":19997},[1109,1173,1174,1112],[57,19999,1288],{"className":20000},[1075,1148,1112],[57,20002,1183],{"className":20003},[1182],[57,20005,20007],{"className":20006},[1092],[57,20008,20010],{"className":20009,"style":5044},[1096],[57,20011],{},[57,20013,1183],{"className":20014},[1182],[57,20016,20018],{"className":20017},[1092],[57,20019,20022],{"className":20020,"style":20021},[1096],"height:0.994em;",[57,20023],{},[57,20025],{"className":20026,"style":1853},[1452],[57,20028,20030,20034],{"className":20029},[1075],[57,20031,19786],{"className":20032,"style":20033},[1075,1148],"margin-right:0.05556em;",[57,20035,20037],{"className":20036},[1152],[57,20038,20040],{"className":20039},[1087],[57,20041,20043],{"className":20042},[1092],[57,20044,20047],{"className":20045,"style":20046},[1096],"height:0.9425em;",[57,20048,20049,20052],{"style":1489},[57,20050],{"className":20051,"style":1493},[1104],[57,20053,20055],{"className":20054},[1109,1110,1111,1112],[57,20056,20058,20061,20064],{"className":20057},[1075,1112],[57,20059,1733],{"className":20060},[1075,1148,1112],[57,20062,19155],{"className":20063},[1506,1112],[57,20065,20067,20070],{"className":20066},[1075,1112],[57,20068,1733],{"className":20069},[1075,1148,1112],[57,20071,20073],{"className":20072},[1152],[57,20074,20076],{"className":20075},[1087],[57,20077,20079],{"className":20078},[1092],[57,20080,20083],{"className":20081,"style":20082},[1096],"height:0.8278em;",[57,20084,20085,20088],{"style":2034},[57,20086],{"className":20087,"style":1169},[1104],[57,20089,20091],{"className":20090},[1109,1173,1174,1112],[57,20092,20094],{"className":20093},[1075,1112],[57,20095,19775],{"className":20096},[1075,1112],[57,20098,20100,20103],{"className":20099},[1075],[57,20101,8500],{"className":20102,"style":3932},[1075,1148],[57,20104,20106],{"className":20105},[1152],[57,20107,20109,20173],{"className":20108},[1087,1088],[57,20110,20112,20170],{"className":20111},[1092],[57,20113,20115,20159],{"className":20114,"style":17412},[1096],[57,20116,20118,20121],{"style":20117},"top:-2.425em;margin-left:-0.0278em;margin-right:0.05em;",[57,20119],{"className":20120,"style":1493},[1104],[57,20122,20124],{"className":20123},[1109,1110,1111,1112],[57,20125,20127],{"className":20126},[1075,1112],[57,20128,20130,20133],{"className":20129},[1075,1112],[57,20131,1733],{"className":20132},[1075,1148,1112],[57,20134,20136],{"className":20135},[1152],[57,20137,20139],{"className":20138},[1087],[57,20140,20142],{"className":20141},[1092],[57,20143,20145],{"className":20144,"style":19935},[1096],[57,20146,20147,20150],{"style":19938},[57,20148],{"className":20149,"style":1169},[1104],[57,20151,20153],{"className":20152},[1109,1173,1174,1112],[57,20154,20156],{"className":20155},[1075,1112],[57,20157,19775],{"className":20158},[1075,1112],[57,20160,20161,20164],{"style":1489},[57,20162],{"className":20163,"style":1493},[1104],[57,20165,20167],{"className":20166},[1109,1110,1111,1112],[57,20168,1288],{"className":20169},[1075,1148,1112],[57,20171,1183],{"className":20172},[1182],[57,20174,20176],{"className":20175},[1092],[57,20177,20180],{"className":20178,"style":20179},[1096],"height:0.275em;",[57,20181],{},[57,20183],{"className":20184,"style":1453},[1452],[57,20186,281],{"className":20187},[1457],[57,20189],{"className":20190,"style":1453},[1452],[57,20192,20194,20198],{"className":20193},[1066],[57,20195],{"className":20196,"style":20197},[1070],"height:0.9303em;vertical-align:-0.247em;",[57,20199,20201,20204],{"className":20200},[1075],[57,20202,1423],{"className":20203,"style":14898},[1075,1148],[57,20205,20207],{"className":20206},[1152],[57,20208,20210,20242],{"className":20209},[1087,1088],[57,20211,20213,20239],{"className":20212},[1092],[57,20214,20216,20228],{"className":20215,"style":17412},[1096],[57,20217,20219,20222],{"style":20218},"top:-2.453em;margin-left:-0.0077em;margin-right:0.05em;",[57,20220],{"className":20221,"style":1493},[1104],[57,20223,20225],{"className":20224},[1109,1110,1111,1112],[57,20226,1733],{"className":20227},[1075,1148,1112],[57,20229,20230,20233],{"style":1489},[57,20231],{"className":20232,"style":1493},[1104],[57,20234,20236],{"className":20235},[1109,1110,1111,1112],[57,20237,1288],{"className":20238},[1075,1148,1112],[57,20240,1183],{"className":20241},[1182],[57,20243,20245],{"className":20244},[1092],[57,20246,20248],{"className":20247,"style":18131},[1096],[57,20249],{},[10,20251,20253],{"id":20252},"actor-critic-演员-评论家算法","Actor-Critic 演员-评论家算法",[14,20255,20256,20257,20328,20329,20415],{},"为了衡量某个动作\"相对好坏\"，我们让奖励 ",[57,20258,20260,20278],{"className":20259},[1019],[57,20261,20263],{"className":20262},[1023],[1025,20264,20265],{"xmlns":1027},[1029,20266,20267,20275],{},[1032,20268,20269],{},[1044,20270,20271,20273],{},[1047,20272,1423],{},[1047,20274,1733],{},[1054,20276,20277],{"encoding":1056},"R_t",[57,20279,20281],{"className":20280,"ariaHidden":1062},[1061],[57,20282,20284,20287],{"className":20283},[1066],[57,20285],{"className":20286,"style":7644},[1070],[57,20288,20290,20293],{"className":20289},[1075],[57,20291,1423],{"className":20292,"style":14898},[1075,1148],[57,20294,20296],{"className":20295},[1152],[57,20297,20299,20320],{"className":20298},[1087,1088],[57,20300,20302,20317],{"className":20301},[1092],[57,20303,20305],{"className":20304,"style":13464},[1096],[57,20306,20308,20311],{"style":20307},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[57,20309],{"className":20310,"style":1493},[1104],[57,20312,20314],{"className":20313},[1109,1110,1111,1112],[57,20315,1733],{"className":20316},[1075,1148,1112],[57,20318,1183],{"className":20319},[1182],[57,20321,20323],{"className":20322},[1092],[57,20324,20326],{"className":20325,"style":2230},[1096],[57,20327],{}," 减去一个基准值 ",[57,20330,20332,20356],{"className":20331},[1019],[57,20333,20335],{"className":20334},[1023],[1025,20336,20337],{"xmlns":1027},[1029,20338,20339,20353],{},[1032,20340,20341,20343,20345,20351],{},[1047,20342,730],{},[1414,20344,1754],{"stretchy":1753},[1044,20346,20347,20349],{},[1047,20348,13535],{},[1047,20350,1733],{},[1414,20352,1768],{"stretchy":1753},[1054,20354,20355],{"encoding":1056},"B(s_t)",[57,20357,20359],{"className":20358,"ariaHidden":1062},[1061],[57,20360,20362,20365,20369,20372,20412],{"className":20361},[1066],[57,20363],{"className":20364,"style":1831},[1070],[57,20366,730],{"className":20367,"style":20368},[1075,1148],"margin-right:0.05017em;",[57,20370,1754],{"className":20371},[1079],[57,20373,20375,20378],{"className":20374},[1075],[57,20376,13535],{"className":20377},[1075,1148],[57,20379,20381],{"className":20380},[1152],[57,20382,20384,20404],{"className":20383},[1087,1088],[57,20385,20387,20401],{"className":20386},[1092],[57,20388,20390],{"className":20389,"style":13464},[1096],[57,20391,20392,20395],{"style":2211},[57,20393],{"className":20394,"style":1493},[1104],[57,20396,20398],{"className":20397},[1109,1110,1111,1112],[57,20399,1733],{"className":20400},[1075,1148,1112],[57,20402,1183],{"className":20403},[1182],[57,20405,20407],{"className":20406},[1092],[57,20408,20410],{"className":20409,"style":2230},[1096],[57,20411],{},[57,20413,1768],{"className":20414},[1269],"，从而降低梯度估计的方差：",[14,20417,20418],{},[57,20419,20421,20533],{"className":20420},[1019],[57,20422,20424],{"className":20423},[1023],[1025,20425,20426],{"xmlns":1027},[1029,20427,20428,20530],{},[1032,20429,20430,20436,20450,20468,20470,20478,20480,20482,20484,20492,20494,20496,20498,20500,20502,20508,20510,20518,20520,20528],{},[1035,20431,20432,20434],{},[1038,20433,574],{},[1047,20435,7990],{},[2545,20437,20438,20440,20448],{},[1414,20439,2549],{},[1032,20441,20442,20444,20446],{},[1047,20443,1288],{},[1414,20445,281],{},[1038,20447,574],{},[1047,20449,7990],{},[2545,20451,20452,20454,20462],{},[1414,20453,2549],{},[1032,20455,20456,20458,20460],{},[1047,20457,1733],{},[1414,20459,281],{},[1038,20461,574],{},[1044,20463,20464,20466],{},[1047,20465,738],{},[1047,20467,1288],{},[1414,20469,1754],{"stretchy":1753},[2545,20471,20472,20474,20476],{},[1047,20473,1423],{},[1047,20475,1733],{},[1047,20477,1288],{},[1414,20479,19155],{},[1047,20481,730],{},[1414,20483,1754],{"stretchy":1753},[2545,20485,20486,20488,20490],{},[1047,20487,13535],{},[1047,20489,1733],{},[1047,20491,1288],{},[1414,20493,1768],{"stretchy":1753},[1414,20495,1768],{"stretchy":1753},[1047,20497,15169],{"mathvariant":1729},[1047,20499,15594],{},[1414,20501,1776],{},[1044,20503,20504,20506],{},[1047,20505,6333],{},[1047,20507,14745],{},[1414,20509,1754],{"stretchy":1753},[2545,20511,20512,20514,20516],{},[1047,20513,18],{},[1047,20515,1733],{},[1047,20517,1288],{},[1047,20519,13675],{"mathvariant":1729},[2545,20521,20522,20524,20526],{},[1047,20523,13535],{},[1047,20525,1733],{},[1047,20527,1288],{},[1414,20529,1768],{"stretchy":1753},[1054,20531,20532],{"encoding":1056}," \\frac{1}{N} \\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} (R_t^n - B(s_t^n))\\nabla \\log P_\\theta(a_t^n|s_t^n)",[57,20534,20536,20847],{"className":20535,"ariaHidden":1062},[1061],[57,20537,20539,20542,20610,20613,20677,20680,20784,20787,20838,20841,20844],{"className":20538},[1066],[57,20540],{"className":20541,"style":19273},[1070],[57,20543,20545,20548,20607],{"className":20544},[1075],[57,20546],{"className":20547},[1079,1080],[57,20549,20551],{"className":20550},[1035],[57,20552,20554,20599],{"className":20553},[1087,1088],[57,20555,20557,20596],{"className":20556},[1092],[57,20558,20560,20574,20582],{"className":20559,"style":1097},[1096],[57,20561,20562,20565],{"style":19297},[57,20563],{"className":20564,"style":1105},[1104],[57,20566,20568],{"className":20567},[1109,1110,1111,1112],[57,20569,20571],{"className":20570},[1075,1112],[57,20572,7990],{"className":20573,"style":17080},[1075,1148,1112],[57,20575,20576,20579],{"style":1230},[57,20577],{"className":20578,"style":1105},[1104],[57,20580],{"className":20581,"style":1238},[1237],[57,20583,20584,20587],{"style":1241},[57,20585],{"className":20586,"style":1105},[1104],[57,20588,20590],{"className":20589},[1109,1110,1111,1112],[57,20591,20593],{"className":20592},[1075,1112],[57,20594,574],{"className":20595},[1075,1112],[57,20597,1183],{"className":20598},[1182],[57,20600,20602],{"className":20601},[1092],[57,20603,20605],{"className":20604,"style":19341},[1096],[57,20606],{},[57,20608],{"className":20609},[1269,1080],[57,20611],{"className":20612,"style":1853},[1452],[57,20614,20616],{"className":20615},[1888,15027],[57,20617,20619,20669],{"className":20618},[1087,1088],[57,20620,20622,20666],{"className":20621},[1092],[57,20623,20625,20645,20655],{"className":20624,"style":19362},[1096],[57,20626,20627,20630],{"style":19365},[57,20628],{"className":20629,"style":1105},[1104],[57,20631,20633],{"className":20632},[1109,1110,1111,1112],[57,20634,20636,20639,20642],{"className":20635},[1075,1112],[57,20637,1288],{"className":20638},[1075,1148,1112],[57,20640,281],{"className":20641},[1457,1112],[57,20643,574],{"className":20644},[1075,1112],[57,20646,20647,20650],{"style":1134},[57,20648],{"className":20649,"style":1105},[1104],[57,20651,20652],{},[57,20653,2549],{"className":20654},[1888,2633,2634],[57,20656,20657,20660],{"style":19396},[57,20658],{"className":20659,"style":1105},[1104],[57,20661,20663],{"className":20662},[1109,1110,1111,1112],[57,20664,7990],{"className":20665,"style":17080},[1075,1148,1112],[57,20667,1183],{"className":20668},[1182],[57,20670,20672],{"className":20671},[1092],[57,20673,20675],{"className":20674,"style":19415},[1096],[57,20676],{},[57,20678],{"className":20679,"style":1853},[1452],[57,20681,20683],{"className":20682},[1888,15027],[57,20684,20686,20776],{"className":20685},[1087,1088],[57,20687,20689,20773],{"className":20688},[1092],[57,20690,20692,20712,20722],{"className":20691,"style":19433},[1096],[57,20693,20694,20697],{"style":19365},[57,20695],{"className":20696,"style":1105},[1104],[57,20698,20700],{"className":20699},[1109,1110,1111,1112],[57,20701,20703,20706,20709],{"className":20702},[1075,1112],[57,20704,1733],{"className":20705},[1075,1148,1112],[57,20707,281],{"className":20708},[1457,1112],[57,20710,574],{"className":20711},[1075,1112],[57,20713,20714,20717],{"style":1134},[57,20715],{"className":20716,"style":1105},[1104],[57,20718,20719],{},[57,20720,2549],{"className":20721},[1888,2633,2634],[57,20723,20724,20727],{"style":19466},[57,20725],{"className":20726,"style":1105},[1104],[57,20728,20730],{"className":20729},[1109,1110,1111,1112],[57,20731,20733],{"className":20732},[1075,1112],[57,20734,20736,20739],{"className":20735},[1075,1112],[57,20737,738],{"className":20738,"style":3907},[1075,1148,1112],[57,20740,20742],{"className":20741},[1152],[57,20743,20745,20765],{"className":20744},[1087,1088],[57,20746,20748,20762],{"className":20747},[1092],[57,20749,20751],{"className":20750,"style":5022},[1096],[57,20752,20753,20756],{"style":18002},[57,20754],{"className":20755,"style":1169},[1104],[57,20757,20759],{"className":20758},[1109,1173,1174,1112],[57,20760,1288],{"className":20761},[1075,1148,1112],[57,20763,1183],{"className":20764},[1182],[57,20766,20768],{"className":20767},[1092],[57,20769,20771],{"className":20770,"style":5044},[1096],[57,20772],{},[57,20774,1183],{"className":20775},[1182],[57,20777,20779],{"className":20778},[1092],[57,20780,20782],{"className":20781,"style":19415},[1096],[57,20783],{},[57,20785,1754],{"className":20786},[1079],[57,20788,20790,20793],{"className":20789},[1075],[57,20791,1423],{"className":20792,"style":14898},[1075,1148],[57,20794,20796],{"className":20795},[1152],[57,20797,20799,20830],{"className":20798},[1087,1088],[57,20800,20802,20827],{"className":20801},[1092],[57,20803,20805,20816],{"className":20804,"style":17412},[1096],[57,20806,20807,20810],{"style":20218},[57,20808],{"className":20809,"style":1493},[1104],[57,20811,20813],{"className":20812},[1109,1110,1111,1112],[57,20814,1733],{"className":20815},[1075,1148,1112],[57,20817,20818,20821],{"style":1489},[57,20819],{"className":20820,"style":1493},[1104],[57,20822,20824],{"className":20823},[1109,1110,1111,1112],[57,20825,1288],{"className":20826},[1075,1148,1112],[57,20828,1183],{"className":20829},[1182],[57,20831,20833],{"className":20832},[1092],[57,20834,20836],{"className":20835,"style":18131},[1096],[57,20837],{},[57,20839],{"className":20840,"style":2595},[1452],[57,20842,19155],{"className":20843},[1506],[57,20845],{"className":20846,"style":2595},[1452],[57,20848,20850,20853,20856,20859,20910,20913,20916,20919,20924,20927,20967,20970,21021,21024,21075],{"className":20849},[1066],[57,20851],{"className":20852,"style":1831},[1070],[57,20854,730],{"className":20855,"style":20368},[1075,1148],[57,20857,1754],{"className":20858},[1079],[57,20860,20862,20865],{"className":20861},[1075],[57,20863,13535],{"className":20864},[1075,1148],[57,20866,20868],{"className":20867},[1152],[57,20869,20871,20902],{"className":20870},[1087,1088],[57,20872,20874,20899],{"className":20873},[1092],[57,20875,20877,20888],{"className":20876,"style":17412},[1096],[57,20878,20879,20882],{"style":18101},[57,20880],{"className":20881,"style":1493},[1104],[57,20883,20885],{"className":20884},[1109,1110,1111,1112],[57,20886,1733],{"className":20887},[1075,1148,1112],[57,20889,20890,20893],{"style":1489},[57,20891],{"className":20892,"style":1493},[1104],[57,20894,20896],{"className":20895},[1109,1110,1111,1112],[57,20897,1288],{"className":20898},[1075,1148,1112],[57,20900,1183],{"className":20901},[1182],[57,20903,20905],{"className":20904},[1092],[57,20906,20908],{"className":20907,"style":18131},[1096],[57,20909],{},[57,20911,8598],{"className":20912},[1269],[57,20914,15169],{"className":20915},[1075],[57,20917],{"className":20918,"style":1853},[1452],[57,20920,17640,20922],{"className":20921},[1888],[57,20923,17644],{"style":17643},[57,20925],{"className":20926,"style":1853},[1452],[57,20928,20930,20933],{"className":20929},[1075],[57,20931,6333],{"className":20932,"style":3907},[1075,1148],[57,20934,20936],{"className":20935},[1152],[57,20937,20939,20959],{"className":20938},[1087,1088],[57,20940,20942,20956],{"className":20941},[1092],[57,20943,20945],{"className":20944,"style":2208},[1096],[57,20946,20947,20950],{"style":8131},[57,20948],{"className":20949,"style":1493},[1104],[57,20951,20953],{"className":20952},[1109,1110,1111,1112],[57,20954,14745],{"className":20955,"style":3932},[1075,1148,1112],[57,20957,1183],{"className":20958},[1182],[57,20960,20962],{"className":20961},[1092],[57,20963,20965],{"className":20964,"style":2230},[1096],[57,20966],{},[57,20968,1754],{"className":20969},[1079],[57,20971,20973,20976],{"className":20972},[1075],[57,20974,18],{"className":20975},[1075,1148],[57,20977,20979],{"className":20978},[1152],[57,20980,20982,21013],{"className":20981},[1087,1088],[57,20983,20985,21010],{"className":20984},[1092],[57,20986,20988,20999],{"className":20987,"style":17412},[1096],[57,20989,20990,20993],{"style":18101},[57,20991],{"className":20992,"style":1493},[1104],[57,20994,20996],{"className":20995},[1109,1110,1111,1112],[57,20997,1733],{"className":20998},[1075,1148,1112],[57,21000,21001,21004],{"style":1489},[57,21002],{"className":21003,"style":1493},[1104],[57,21005,21007],{"className":21006},[1109,1110,1111,1112],[57,21008,1288],{"className":21009},[1075,1148,1112],[57,21011,1183],{"className":21012},[1182],[57,21014,21016],{"className":21015},[1092],[57,21017,21019],{"className":21018,"style":18131},[1096],[57,21020],{},[57,21022,13675],{"className":21023},[1075],[57,21025,21027,21030],{"className":21026},[1075],[57,21028,13535],{"className":21029},[1075,1148],[57,21031,21033],{"className":21032},[1152],[57,21034,21036,21067],{"className":21035},[1087,1088],[57,21037,21039,21064],{"className":21038},[1092],[57,21040,21042,21053],{"className":21041,"style":17412},[1096],[57,21043,21044,21047],{"style":18101},[57,21045],{"className":21046,"style":1493},[1104],[57,21048,21050],{"className":21049},[1109,1110,1111,1112],[57,21051,1733],{"className":21052},[1075,1148,1112],[57,21054,21055,21058],{"style":1489},[57,21056],{"className":21057,"style":1493},[1104],[57,21059,21061],{"className":21060},[1109,1110,1111,1112],[57,21062,1288],{"className":21063},[1075,1148,1112],[57,21065,1183],{"className":21066},[1182],[57,21068,21070],{"className":21069},[1092],[57,21071,21073],{"className":21072,"style":18131},[1096],[57,21074],{},[57,21076,1768],{"className":21077},[1269],[14,21079,21080,21081,21164,21165,21263,21264,21292,21293,21321,21322,21325,21326,21263,21412,21440,21441,222],{},"然而 ",[57,21082,21084,21104],{"className":21083},[1019],[57,21085,21087],{"className":21086},[1023],[1025,21088,21089],{"xmlns":1027},[1029,21090,21091,21101],{},[1032,21092,21093],{},[2545,21094,21095,21097,21099],{},[1047,21096,1423],{},[1047,21098,1733],{},[1047,21100,1288],{},[1054,21102,21103],{"encoding":1056},"R_t^n",[57,21105,21107],{"className":21106,"ariaHidden":1062},[1061],[57,21108,21110,21113],{"className":21109},[1066],[57,21111],{"className":21112,"style":20197},[1070],[57,21114,21116,21119],{"className":21115},[1075],[57,21117,1423],{"className":21118,"style":14898},[1075,1148],[57,21120,21122],{"className":21121},[1152],[57,21123,21125,21156],{"className":21124},[1087,1088],[57,21126,21128,21153],{"className":21127},[1092],[57,21129,21131,21142],{"className":21130,"style":17412},[1096],[57,21132,21133,21136],{"style":20218},[57,21134],{"className":21135,"style":1493},[1104],[57,21137,21139],{"className":21138},[1109,1110,1111,1112],[57,21140,1733],{"className":21141},[1075,1148,1112],[57,21143,21144,21147],{"style":1489},[57,21145],{"className":21146,"style":1493},[1104],[57,21148,21150],{"className":21149},[1109,1110,1111,1112],[57,21151,1288],{"className":21152},[1075,1148,1112],[57,21154,1183],{"className":21155},[1182],[57,21157,21159],{"className":21158},[1092],[57,21160,21162],{"className":21161,"style":18131},[1096],[57,21163],{}," 仅来自一次随机采样，方差很大、训练不稳定。一个自然的改进是引入价值函数：用 ",[57,21166,21168,21196],{"className":21167},[1019],[57,21169,21171],{"className":21170},[1023],[1025,21172,21173],{"xmlns":1027},[1029,21174,21175,21193],{},[1032,21176,21177,21183,21185,21187,21189,21191],{},[1044,21178,21179,21181],{},[1047,21180,1412],{},[1047,21182,14745],{},[1414,21184,1754],{"stretchy":1753},[1047,21186,13535],{},[1414,21188,1759],{"separator":1062},[1047,21190,18],{},[1414,21192,1768],{"stretchy":1753},[1054,21194,21195],{"encoding":1056},"Q_\\theta(s,a)",[57,21197,21199],{"className":21198,"ariaHidden":1062},[1061],[57,21200,21202,21205,21245,21248,21251,21254,21257,21260],{"className":21201},[1066],[57,21203],{"className":21204,"style":1831},[1070],[57,21206,21208,21211],{"className":21207},[1075],[57,21209,1412],{"className":21210},[1075,1148],[57,21212,21214],{"className":21213},[1152],[57,21215,21217,21237],{"className":21216},[1087,1088],[57,21218,21220,21234],{"className":21219},[1092],[57,21221,21223],{"className":21222,"style":2208},[1096],[57,21224,21225,21228],{"style":2211},[57,21226],{"className":21227,"style":1493},[1104],[57,21229,21231],{"className":21230},[1109,1110,1111,1112],[57,21232,14745],{"className":21233,"style":3932},[1075,1148,1112],[57,21235,1183],{"className":21236},[1182],[57,21238,21240],{"className":21239},[1092],[57,21241,21243],{"className":21242,"style":2230},[1096],[57,21244],{},[57,21246,1754],{"className":21247},[1079],[57,21249,13535],{"className":21250},[1075,1148],[57,21252,1759],{"className":21253},[1849],[57,21255],{"className":21256,"style":1853},[1452],[57,21258,18],{"className":21259},[1075,1148],[57,21261,1768],{"className":21262},[1269]," 表示在状态 ",[57,21265,21267,21280],{"className":21266},[1019],[57,21268,21270],{"className":21269},[1023],[1025,21271,21272],{"xmlns":1027},[1029,21273,21274,21278],{},[1032,21275,21276],{},[1047,21277,13535],{},[1054,21279,13535],{"encoding":1056},[57,21281,21283],{"className":21282,"ariaHidden":1062},[1061],[57,21284,21286,21289],{"className":21285},[1066],[57,21287],{"className":21288,"style":1300},[1070],[57,21290,13535],{"className":21291},[1075,1148]," 下采取动作 ",[57,21294,21296,21309],{"className":21295},[1019],[57,21297,21299],{"className":21298},[1023],[1025,21300,21301],{"xmlns":1027},[1029,21302,21303,21307],{},[1032,21304,21305],{},[1047,21306,18],{},[1054,21308,18],{"encoding":1056},[57,21310,21312],{"className":21311,"ariaHidden":1062},[1061],[57,21313,21315,21318],{"className":21314},[1066],[57,21316],{"className":21317,"style":1300},[1070],[57,21319,18],{"className":21320},[1075,1148]," 的期望回报，即 ",[11865,21323,21324],{},"动作价值函数","；用 ",[57,21327,21329,21353],{"className":21328},[1019],[57,21330,21332],{"className":21331},[1023],[1025,21333,21334],{"xmlns":1027},[1029,21335,21336,21350],{},[1032,21337,21338,21344,21346,21348],{},[1044,21339,21340,21342],{},[1047,21341,1625],{},[1047,21343,14745],{},[1414,21345,1754],{"stretchy":1753},[1047,21347,13535],{},[1414,21349,1768],{"stretchy":1753},[1054,21351,21352],{"encoding":1056},"V_\\theta(s)",[57,21354,21356],{"className":21355,"ariaHidden":1062},[1061],[57,21357,21359,21362,21403,21406,21409],{"className":21358},[1066],[57,21360],{"className":21361,"style":1831},[1070],[57,21363,21365,21368],{"className":21364},[1075],[57,21366,1625],{"className":21367,"style":1655},[1075,1148],[57,21369,21371],{"className":21370},[1152],[57,21372,21374,21395],{"className":21373},[1087,1088],[57,21375,21377,21392],{"className":21376},[1092],[57,21378,21380],{"className":21379,"style":2208},[1096],[57,21381,21383,21386],{"style":21382},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[57,21384],{"className":21385,"style":1493},[1104],[57,21387,21389],{"className":21388},[1109,1110,1111,1112],[57,21390,14745],{"className":21391,"style":3932},[1075,1148,1112],[57,21393,1183],{"className":21394},[1182],[57,21396,21398],{"className":21397},[1092],[57,21399,21401],{"className":21400,"style":2230},[1096],[57,21402],{},[57,21404,1754],{"className":21405},[1079],[57,21407,13535],{"className":21408},[1075,1148],[57,21410,1768],{"className":21411},[1269],[57,21413,21415,21428],{"className":21414},[1019],[57,21416,21418],{"className":21417},[1023],[1025,21419,21420],{"xmlns":1027},[1029,21421,21422,21426],{},[1032,21423,21424],{},[1047,21425,13535],{},[1054,21427,13535],{"encoding":1056},[57,21429,21431],{"className":21430,"ariaHidden":1062},[1061],[57,21432,21434,21437],{"className":21433},[1066],[57,21435],{"className":21436,"style":1300},[1070],[57,21438,13535],{"className":21439},[1075,1148]," 下的期望回报，即 ",[11865,21442,21443],{},"状态价值函数",[14,21445,21446,21447,21450,21451,21718,21719,21747,21748,21776],{},"定义 ",[11865,21448,21449],{},"优势函数（Advantage Function）"," ",[57,21452,21454,21514],{"className":21453},[1019],[57,21455,21457],{"className":21456},[1023],[1025,21458,21459],{"xmlns":1027},[1029,21460,21461,21511],{},[1032,21462,21463,21469,21471,21473,21475,21477,21479,21481,21487,21489,21491,21493,21495,21497,21499,21505,21507,21509],{},[1044,21464,21465,21467],{},[1047,21466,1730],{},[1047,21468,14745],{},[1414,21470,1754],{"stretchy":1753},[1047,21472,13535],{},[1414,21474,1759],{"separator":1062},[1047,21476,18],{},[1414,21478,1768],{"stretchy":1753},[1414,21480,281],{},[1044,21482,21483,21485],{},[1047,21484,1412],{},[1047,21486,14745],{},[1414,21488,1754],{"stretchy":1753},[1047,21490,13535],{},[1414,21492,1759],{"separator":1062},[1047,21494,18],{},[1414,21496,1768],{"stretchy":1753},[1414,21498,19155],{},[1044,21500,21501,21503],{},[1047,21502,1625],{},[1047,21504,14745],{},[1414,21506,1754],{"stretchy":1753},[1047,21508,13535],{},[1414,21510,1768],{"stretchy":1753},[1054,21512,21513],{"encoding":1056},"A_\\theta(s,a) = Q_\\theta(s,a) - V_\\theta(s)",[57,21515,21517,21590,21663],{"className":21516,"ariaHidden":1062},[1061],[57,21518,21520,21523,21563,21566,21569,21572,21575,21578,21581,21584,21587],{"className":21519},[1066],[57,21521],{"className":21522,"style":1831},[1070],[57,21524,21526,21529],{"className":21525},[1075],[57,21527,1730],{"className":21528},[1075,1148],[57,21530,21532],{"className":21531},[1152],[57,21533,21535,21555],{"className":21534},[1087,1088],[57,21536,21538,21552],{"className":21537},[1092],[57,21539,21541],{"className":21540,"style":2208},[1096],[57,21542,21543,21546],{"style":2211},[57,21544],{"className":21545,"style":1493},[1104],[57,21547,21549],{"className":21548},[1109,1110,1111,1112],[57,21550,14745],{"className":21551,"style":3932},[1075,1148,1112],[57,21553,1183],{"className":21554},[1182],[57,21556,21558],{"className":21557},[1092],[57,21559,21561],{"className":21560,"style":2230},[1096],[57,21562],{},[57,21564,1754],{"className":21565},[1079],[57,21567,13535],{"className":21568},[1075,1148],[57,21570,1759],{"className":21571},[1849],[57,21573],{"className":21574,"style":1853},[1452],[57,21576,18],{"className":21577},[1075,1148],[57,21579,1768],{"className":21580},[1269],[57,21582],{"className":21583,"style":1453},[1452],[57,21585,281],{"className":21586},[1457],[57,21588],{"className":21589,"style":1453},[1452],[57,21591,21593,21596,21636,21639,21642,21645,21648,21651,21654,21657,21660],{"className":21592},[1066],[57,21594],{"className":21595,"style":1831},[1070],[57,21597,21599,21602],{"className":21598},[1075],[57,21600,1412],{"className":21601},[1075,1148],[57,21603,21605],{"className":21604},[1152],[57,21606,21608,21628],{"className":21607},[1087,1088],[57,21609,21611,21625],{"className":21610},[1092],[57,21612,21614],{"className":21613,"style":2208},[1096],[57,21615,21616,21619],{"style":2211},[57,21617],{"className":21618,"style":1493},[1104],[57,21620,21622],{"className":21621},[1109,1110,1111,1112],[57,21623,14745],{"className":21624,"style":3932},[1075,1148,1112],[57,21626,1183],{"className":21627},[1182],[57,21629,21631],{"className":21630},[1092],[57,21632,21634],{"className":21633,"style":2230},[1096],[57,21635],{},[57,21637,1754],{"className":21638},[1079],[57,21640,13535],{"className":21641},[1075,1148],[57,21643,1759],{"className":21644},[1849],[57,21646],{"className":21647,"style":1853},[1452],[57,21649,18],{"className":21650},[1075,1148],[57,21652,1768],{"className":21653},[1269],[57,21655],{"className":21656,"style":2595},[1452],[57,21658,19155],{"className":21659},[1506],[57,21661],{"className":21662,"style":2595},[1452],[57,21664,21666,21669,21709,21712,21715],{"className":21665},[1066],[57,21667],{"className":21668,"style":1831},[1070],[57,21670,21672,21675],{"className":21671},[1075],[57,21673,1625],{"className":21674,"style":1655},[1075,1148],[57,21676,21678],{"className":21677},[1152],[57,21679,21681,21701],{"className":21680},[1087,1088],[57,21682,21684,21698],{"className":21683},[1092],[57,21685,21687],{"className":21686,"style":2208},[1096],[57,21688,21689,21692],{"style":21382},[57,21690],{"className":21691,"style":1493},[1104],[57,21693,21695],{"className":21694},[1109,1110,1111,1112],[57,21696,14745],{"className":21697,"style":3932},[1075,1148,1112],[57,21699,1183],{"className":21700},[1182],[57,21702,21704],{"className":21703},[1092],[57,21705,21707],{"className":21706,"style":2230},[1096],[57,21708],{},[57,21710,1754],{"className":21711},[1079],[57,21713,13535],{"className":21714},[1075,1148],[57,21716,1768],{"className":21717},[1269],"，它衡量在状态 ",[57,21720,21722,21735],{"className":21721},[1019],[57,21723,21725],{"className":21724},[1023],[1025,21726,21727],{"xmlns":1027},[1029,21728,21729,21733],{},[1032,21730,21731],{},[1047,21732,13535],{},[1054,21734,13535],{"encoding":1056},[57,21736,21738],{"className":21737,"ariaHidden":1062},[1061],[57,21739,21741,21744],{"className":21740},[1066],[57,21742],{"className":21743,"style":1300},[1070],[57,21745,13535],{"className":21746},[1075,1148]," 下，动作 ",[57,21749,21751,21764],{"className":21750},[1019],[57,21752,21754],{"className":21753},[1023],[1025,21755,21756],{"xmlns":1027},[1029,21757,21758,21762],{},[1032,21759,21760],{},[1047,21761,18],{},[1054,21763,18],{"encoding":1056},[57,21765,21767],{"className":21766,"ariaHidden":1062},[1061],[57,21768,21770,21773],{"className":21769},[1066],[57,21771],{"className":21772,"style":1300},[1070],[57,21774,18],{"className":21775},[1075,1148]," 相对于\"该状态下的平均水平\"有多大优势。",[14,21778,21779],{},"将上述策略梯度公式改写为：",[14,21781,21782],{},[57,21783,21785,21899],{"className":21784},[1019],[57,21786,21788],{"className":21787},[1023],[1025,21789,21790],{"xmlns":1027},[1029,21791,21792,21896],{},[1032,21793,21794,21802,21816,21834,21840,21842,21850,21852,21860,21862,21864,21866,21868,21874,21876,21884,21886,21894],{},[15163,21795,21796],{"scriptlevel":560,"displaystyle":1062},[1035,21797,21798,21800],{},[1038,21799,574],{},[1047,21801,7990],{},[2545,21803,21804,21806,21814],{},[1414,21805,2549],{},[1032,21807,21808,21810,21812],{},[1047,21809,1288],{},[1414,21811,281],{},[1038,21813,574],{},[1047,21815,7990],{},[2545,21817,21818,21820,21828],{},[1414,21819,2549],{},[1032,21821,21822,21824,21826],{},[1047,21823,1733],{},[1414,21825,281],{},[1038,21827,574],{},[1044,21829,21830,21832],{},[1047,21831,738],{},[1047,21833,1288],{},[1044,21835,21836,21838],{},[1047,21837,1730],{},[1047,21839,14745],{},[1414,21841,1754],{"stretchy":1753},[2545,21843,21844,21846,21848],{},[1047,21845,13535],{},[1047,21847,1733],{},[1047,21849,1288],{},[1414,21851,1759],{"separator":1062},[2545,21853,21854,21856,21858],{},[1047,21855,18],{},[1047,21857,1733],{},[1047,21859,1288],{},[1414,21861,1768],{"stretchy":1753},[1047,21863,15169],{"mathvariant":1729},[1047,21865,15594],{},[1414,21867,1776],{},[1044,21869,21870,21872],{},[1047,21871,6333],{},[1047,21873,14745],{},[1414,21875,1754],{"stretchy":1753},[2545,21877,21878,21880,21882],{},[1047,21879,18],{},[1047,21881,1733],{},[1047,21883,1288],{},[1047,21885,13675],{"mathvariant":1729},[2545,21887,21888,21890,21892],{},[1047,21889,13535],{},[1047,21891,1733],{},[1047,21893,1288],{},[1414,21895,1768],{"stretchy":1753},[1054,21897,21898],{"encoding":1056},"\\dfrac{1}{N}\\sum\\limits_{n=1}^N \\sum\\limits_{t=1}^{T_n} A_\\theta(s_t^n,a_t^n)\\nabla \\log P_\\theta(a_t^n|s_t^n)",[57,21900,21902],{"className":21901,"ariaHidden":1062},[1061],[57,21903,21905,21908,21970,21973,22037,22040,22144,22147,22187,22190,22241,22244,22247,22298,22301,22304,22307,22312,22315,22355,22358,22409,22412,22463],{"className":21904},[1066],[57,21906],{"className":21907,"style":19273},[1070],[57,21909,21911,21914,21967],{"className":21910},[1075],[57,21912],{"className":21913},[1079,1080],[57,21915,21917],{"className":21916},[1035],[57,21918,21920,21959],{"className":21919},[1087,1088],[57,21921,21923,21956],{"className":21922},[1092],[57,21924,21926,21937,21945],{"className":21925,"style":17068},[1096],[57,21927,21928,21931],{"style":16608},[57,21929],{"className":21930,"style":1105},[1104],[57,21932,21934],{"className":21933},[1075],[57,21935,7990],{"className":21936,"style":17080},[1075,1148],[57,21938,21939,21942],{"style":1230},[57,21940],{"className":21941,"style":1105},[1104],[57,21943],{"className":21944,"style":1238},[1237],[57,21946,21947,21950],{"style":16674},[57,21948],{"className":21949,"style":1105},[1104],[57,21951,21953],{"className":21952},[1075],[57,21954,574],{"className":21955},[1075],[57,21957,1183],{"className":21958},[1182],[57,21960,21962],{"className":21961},[1092],[57,21963,21965],{"className":21964,"style":17109},[1096],[57,21966],{},[57,21968],{"className":21969},[1269,1080],[57,21971],{"className":21972,"style":1853},[1452],[57,21974,21976],{"className":21975},[1888,15027],[57,21977,21979,22029],{"className":21978},[1087,1088],[57,21980,21982,22026],{"className":21981},[1092],[57,21983,21985,22005,22015],{"className":21984,"style":19362},[1096],[57,21986,21987,21990],{"style":19365},[57,21988],{"className":21989,"style":1105},[1104],[57,21991,21993],{"className":21992},[1109,1110,1111,1112],[57,21994,21996,21999,22002],{"className":21995},[1075,1112],[57,21997,1288],{"className":21998},[1075,1148,1112],[57,22000,281],{"className":22001},[1457,1112],[57,22003,574],{"className":22004},[1075,1112],[57,22006,22007,22010],{"style":1134},[57,22008],{"className":22009,"style":1105},[1104],[57,22011,22012],{},[57,22013,2549],{"className":22014},[1888,2633,2634],[57,22016,22017,22020],{"style":19396},[57,22018],{"className":22019,"style":1105},[1104],[57,22021,22023],{"className":22022},[1109,1110,1111,1112],[57,22024,7990],{"className":22025,"style":17080},[1075,1148,1112],[57,22027,1183],{"className":22028},[1182],[57,22030,22032],{"className":22031},[1092],[57,22033,22035],{"className":22034,"style":19415},[1096],[57,22036],{},[57,22038],{"className":22039,"style":1853},[1452],[57,22041,22043],{"className":22042},[1888,15027],[57,22044,22046,22136],{"className":22045},[1087,1088],[57,22047,22049,22133],{"className":22048},[1092],[57,22050,22052,22072,22082],{"className":22051,"style":19433},[1096],[57,22053,22054,22057],{"style":19365},[57,22055],{"className":22056,"style":1105},[1104],[57,22058,22060],{"className":22059},[1109,1110,1111,1112],[57,22061,22063,22066,22069],{"className":22062},[1075,1112],[57,22064,1733],{"className":22065},[1075,1148,1112],[57,22067,281],{"className":22068},[1457,1112],[57,22070,574],{"className":22071},[1075,1112],[57,22073,22074,22077],{"style":1134},[57,22075],{"className":22076,"style":1105},[1104],[57,22078,22079],{},[57,22080,2549],{"className":22081},[1888,2633,2634],[57,22083,22084,22087],{"style":19466},[57,22085],{"className":22086,"style":1105},[1104],[57,22088,22090],{"className":22089},[1109,1110,1111,1112],[57,22091,22093],{"className":22092},[1075,1112],[57,22094,22096,22099],{"className":22095},[1075,1112],[57,22097,738],{"className":22098,"style":3907},[1075,1148,1112],[57,22100,22102],{"className":22101},[1152],[57,22103,22105,22125],{"className":22104},[1087,1088],[57,22106,22108,22122],{"className":22107},[1092],[57,22109,22111],{"className":22110,"style":5022},[1096],[57,22112,22113,22116],{"style":18002},[57,22114],{"className":22115,"style":1169},[1104],[57,22117,22119],{"className":22118},[1109,1173,1174,1112],[57,22120,1288],{"className":22121},[1075,1148,1112],[57,22123,1183],{"className":22124},[1182],[57,22126,22128],{"className":22127},[1092],[57,22129,22131],{"className":22130,"style":5044},[1096],[57,22132],{},[57,22134,1183],{"className":22135},[1182],[57,22137,22139],{"className":22138},[1092],[57,22140,22142],{"className":22141,"style":19415},[1096],[57,22143],{},[57,22145],{"className":22146,"style":1853},[1452],[57,22148,22150,22153],{"className":22149},[1075],[57,22151,1730],{"className":22152},[1075,1148],[57,22154,22156],{"className":22155},[1152],[57,22157,22159,22179],{"className":22158},[1087,1088],[57,22160,22162,22176],{"className":22161},[1092],[57,22163,22165],{"className":22164,"style":2208},[1096],[57,22166,22167,22170],{"style":2211},[57,22168],{"className":22169,"style":1493},[1104],[57,22171,22173],{"className":22172},[1109,1110,1111,1112],[57,22174,14745],{"className":22175,"style":3932},[1075,1148,1112],[57,22177,1183],{"className":22178},[1182],[57,22180,22182],{"className":22181},[1092],[57,22183,22185],{"className":22184,"style":2230},[1096],[57,22186],{},[57,22188,1754],{"className":22189},[1079],[57,22191,22193,22196],{"className":22192},[1075],[57,22194,13535],{"className":22195},[1075,1148],[57,22197,22199],{"className":22198},[1152],[57,22200,22202,22233],{"className":22201},[1087,1088],[57,22203,22205,22230],{"className":22204},[1092],[57,22206,22208,22219],{"className":22207,"style":17412},[1096],[57,22209,22210,22213],{"style":18101},[57,22211],{"className":22212,"style":1493},[1104],[57,22214,22216],{"className":22215},[1109,1110,1111,1112],[57,22217,1733],{"className":22218},[1075,1148,1112],[57,22220,22221,22224],{"style":1489},[57,22222],{"className":22223,"style":1493},[1104],[57,22225,22227],{"className":22226},[1109,1110,1111,1112],[57,22228,1288],{"className":22229},[1075,1148,1112],[57,22231,1183],{"className":22232},[1182],[57,22234,22236],{"className":22235},[1092],[57,22237,22239],{"className":22238,"style":18131},[1096],[57,22240],{},[57,22242,1759],{"className":22243},[1849],[57,22245],{"className":22246,"style":1853},[1452],[57,22248,22250,22253],{"className":22249},[1075],[57,22251,18],{"className":22252},[1075,1148],[57,22254,22256],{"className":22255},[1152],[57,22257,22259,22290],{"className":22258},[1087,1088],[57,22260,22262,22287],{"className":22261},[1092],[57,22263,22265,22276],{"className":22264,"style":17412},[1096],[57,22266,22267,22270],{"style":18101},[57,22268],{"className":22269,"style":1493},[1104],[57,22271,22273],{"className":22272},[1109,1110,1111,1112],[57,22274,1733],{"className":22275},[1075,1148,1112],[57,22277,22278,22281],{"style":1489},[57,22279],{"className":22280,"style":1493},[1104],[57,22282,22284],{"className":22283},[1109,1110,1111,1112],[57,22285,1288],{"className":22286},[1075,1148,1112],[57,22288,1183],{"className":22289},[1182],[57,22291,22293],{"className":22292},[1092],[57,22294,22296],{"className":22295,"style":18131},[1096],[57,22297],{},[57,22299,1768],{"className":22300},[1269],[57,22302,15169],{"className":22303},[1075],[57,22305],{"className":22306,"style":1853},[1452],[57,22308,17640,22310],{"className":22309},[1888],[57,22311,17644],{"style":17643},[57,22313],{"className":22314,"style":1853},[1452],[57,22316,22318,22321],{"className":22317},[1075],[57,22319,6333],{"className":22320,"style":3907},[1075,1148],[57,22322,22324],{"className":22323},[1152],[57,22325,22327,22347],{"className":22326},[1087,1088],[57,22328,22330,22344],{"className":22329},[1092],[57,22331,22333],{"className":22332,"style":2208},[1096],[57,22334,22335,22338],{"style":8131},[57,22336],{"className":22337,"style":1493},[1104],[57,22339,22341],{"className":22340},[1109,1110,1111,1112],[57,22342,14745],{"className":22343,"style":3932},[1075,1148,1112],[57,22345,1183],{"className":22346},[1182],[57,22348,22350],{"className":22349},[1092],[57,22351,22353],{"className":22352,"style":2230},[1096],[57,22354],{},[57,22356,1754],{"className":22357},[1079],[57,22359,22361,22364],{"className":22360},[1075],[57,22362,18],{"className":22363},[1075,1148],[57,22365,22367],{"className":22366},[1152],[57,22368,22370,22401],{"className":22369},[1087,1088],[57,22371,22373,22398],{"className":22372},[1092],[57,22374,22376,22387],{"className":22375,"style":17412},[1096],[57,22377,22378,22381],{"style":18101},[57,22379],{"className":22380,"style":1493},[1104],[57,22382,22384],{"className":22383},[1109,1110,1111,1112],[57,22385,1733],{"className":22386},[1075,1148,1112],[57,22388,22389,22392],{"style":1489},[57,22390],{"className":22391,"style":1493},[1104],[57,22393,22395],{"className":22394},[1109,1110,1111,1112],[57,22396,1288],{"className":22397},[1075,1148,1112],[57,22399,1183],{"className":22400},[1182],[57,22402,22404],{"className":22403},[1092],[57,22405,22407],{"className":22406,"style":18131},[1096],[57,22408],{},[57,22410,13675],{"className":22411},[1075],[57,22413,22415,22418],{"className":22414},[1075],[57,22416,13535],{"className":22417},[1075,1148],[57,22419,22421],{"className":22420},[1152],[57,22422,22424,22455],{"className":22423},[1087,1088],[57,22425,22427,22452],{"className":22426},[1092],[57,22428,22430,22441],{"className":22429,"style":17412},[1096],[57,22431,22432,22435],{"style":18101},[57,22433],{"className":22434,"style":1493},[1104],[57,22436,22438],{"className":22437},[1109,1110,1111,1112],[57,22439,1733],{"className":22440},[1075,1148,1112],[57,22442,22443,22446],{"style":1489},[57,22444],{"className":22445,"style":1493},[1104],[57,22447,22449],{"className":22448},[1109,1110,1111,1112],[57,22450,1288],{"className":22451},[1075,1148,1112],[57,22453,1183],{"className":22454},[1182],[57,22456,22458],{"className":22457},[1092],[57,22459,22461],{"className":22460,"style":18131},[1096],[57,22462],{},[57,22464,1768],{"className":22465},[1269],[14,22467,22468],{},"利用 Bellman 关系，可以把动作价值函数与状态价值函数联系起来：",[57,22470,22472],{"className":22471},[15139],[57,22473,22475,22680],{"className":22474},[1019],[57,22476,22478],{"className":22477},[1023],[1025,22479,22480],{"xmlns":1027,"display":10210},[1029,22481,22482,22677],{},[1032,22483,22484,22490,22492,22498,22500,22502,22504,22506,22512,22514,22516,22522,22524,22536,22538,22541,22547,22549,22555,22557,22559,22561,22563,22569,22571,22573,22579,22581,22593,22595,22597,22603,22605,22611,22613,22615,22621,22623,22635,22637,22639,22651,22653,22655,22661,22663,22675],{},[1044,22485,22486,22488],{},[1047,22487,1412],{},[1047,22489,14745],{},[1414,22491,1754],{"stretchy":1753},[1044,22493,22494,22496],{},[1047,22495,13535],{},[1047,22497,1733],{},[1414,22499,1759],{"separator":1062},[1047,22501,18],{},[1414,22503,1768],{"stretchy":1753},[1414,22505,281],{},[1044,22507,22508,22510],{},[1047,22509,8500],{},[1047,22511,1733],{},[1414,22513,6487],{},[1047,22515,19786],{},[1044,22517,22518,22520],{},[1047,22519,1625],{},[1047,22521,14745],{},[1414,22523,1754],{"stretchy":1753},[1044,22525,22526,22528],{},[1047,22527,13535],{},[1032,22529,22530,22532,22534],{},[1047,22531,1733],{},[1414,22533,6487],{},[1038,22535,574],{},[1414,22537,1768],{"stretchy":1753},[1452,22539],{"linebreak":22540},"newline",[1044,22542,22543,22545],{},[1047,22544,1730],{},[1047,22546,14745],{},[1414,22548,1754],{"stretchy":1753},[1044,22550,22551,22553],{},[1047,22552,13535],{},[1047,22554,1733],{},[1414,22556,1759],{"separator":1062},[1047,22558,18],{},[1414,22560,1768],{"stretchy":1753},[1414,22562,281],{},[1044,22564,22565,22567],{},[1047,22566,8500],{},[1047,22568,1733],{},[1414,22570,6487],{},[1047,22572,19786],{},[1044,22574,22575,22577],{},[1047,22576,1625],{},[1047,22578,14745],{},[1414,22580,1754],{"stretchy":1753},[1044,22582,22583,22585],{},[1047,22584,13535],{},[1032,22586,22587,22589,22591],{},[1047,22588,1733],{},[1414,22590,6487],{},[1038,22592,574],{},[1414,22594,1768],{"stretchy":1753},[1414,22596,19155],{},[1044,22598,22599,22601],{},[1047,22600,1625],{},[1047,22602,14745],{},[1414,22604,1754],{"stretchy":1753},[1044,22606,22607,22609],{},[1047,22608,13535],{},[1047,22610,1733],{},[1414,22612,1768],{"stretchy":1753},[1452,22614],{"linebreak":22540},[1044,22616,22617,22619],{},[1047,22618,1625],{},[1047,22620,14745],{},[1414,22622,1754],{"stretchy":1753},[1044,22624,22625,22627],{},[1047,22626,13535],{},[1032,22628,22629,22631,22633],{},[1047,22630,1733],{},[1414,22632,6487],{},[1038,22634,574],{},[1414,22636,1768],{"stretchy":1753},[1414,22638,15464],{},[1044,22640,22641,22643],{},[1047,22642,8500],{},[1032,22644,22645,22647,22649],{},[1047,22646,1733],{},[1414,22648,6487],{},[1038,22650,574],{},[1414,22652,6487],{},[1047,22654,19786],{},[1044,22656,22657,22659],{},[1047,22658,1625],{},[1047,22660,14745],{},[1414,22662,1754],{"stretchy":1753},[1044,22664,22665,22667],{},[1047,22666,13535],{},[1032,22668,22669,22671,22673],{},[1047,22670,1733],{},[1414,22672,6487],{},[1038,22674,590],{},[1414,22676,1768],{"stretchy":1753},[1054,22678,22679],{"encoding":1056}," Q_\\theta(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) \\\\\n\n A_\\theta(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n V_\\theta(s_{t+1}) \\approx r_{t+1} + \\gamma V_\\theta(s_{t+2})",[57,22681,22683,22793,22849,22953,22956,23066,23121,23234,23326,23329,23439,23504],{"className":22682,"ariaHidden":1062},[1061],[57,22684,22686,22689,22729,22732,22772,22775,22778,22781,22784,22787,22790],{"className":22685},[1066],[57,22687],{"className":22688,"style":1831},[1070],[57,22690,22692,22695],{"className":22691},[1075],[57,22693,1412],{"className":22694},[1075,1148],[57,22696,22698],{"className":22697},[1152],[57,22699,22701,22721],{"className":22700},[1087,1088],[57,22702,22704,22718],{"className":22703},[1092],[57,22705,22707],{"className":22706,"style":2208},[1096],[57,22708,22709,22712],{"style":2211},[57,22710],{"className":22711,"style":1493},[1104],[57,22713,22715],{"className":22714},[1109,1110,1111,1112],[57,22716,14745],{"className":22717,"style":3932},[1075,1148,1112],[57,22719,1183],{"className":22720},[1182],[57,22722,22724],{"className":22723},[1092],[57,22725,22727],{"className":22726,"style":2230},[1096],[57,22728],{},[57,22730,1754],{"className":22731},[1079],[57,22733,22735,22738],{"className":22734},[1075],[57,22736,13535],{"className":22737},[1075,1148],[57,22739,22741],{"className":22740},[1152],[57,22742,22744,22764],{"className":22743},[1087,1088],[57,22745,22747,22761],{"className":22746},[1092],[57,22748,22750],{"className":22749,"style":13464},[1096],[57,22751,22752,22755],{"style":2211},[57,22753],{"className":22754,"style":1493},[1104],[57,22756,22758],{"className":22757},[1109,1110,1111,1112],[57,22759,1733],{"className":22760},[1075,1148,1112],[57,22762,1183],{"className":22763},[1182],[57,22765,22767],{"className":22766},[1092],[57,22768,22770],{"className":22769,"style":2230},[1096],[57,22771],{},[57,22773,1759],{"className":22774},[1849],[57,22776],{"className":22777,"style":1853},[1452],[57,22779,18],{"className":22780},[1075,1148],[57,22782,1768],{"className":22783},[1269],[57,22785],{"className":22786,"style":1453},[1452],[57,22788,281],{"className":22789},[1457],[57,22791],{"className":22792,"style":1453},[1452],[57,22794,22796,22800,22840,22843,22846],{"className":22795},[1066],[57,22797],{"className":22798,"style":22799},[1070],"height:0.7333em;vertical-align:-0.15em;",[57,22801,22803,22806],{"className":22802},[1075],[57,22804,8500],{"className":22805,"style":3932},[1075,1148],[57,22807,22809],{"className":22808},[1152],[57,22810,22812,22832],{"className":22811},[1087,1088],[57,22813,22815,22829],{"className":22814},[1092],[57,22816,22818],{"className":22817,"style":13464},[1096],[57,22819,22820,22823],{"style":13853},[57,22821],{"className":22822,"style":1493},[1104],[57,22824,22826],{"className":22825},[1109,1110,1111,1112],[57,22827,1733],{"className":22828},[1075,1148,1112],[57,22830,1183],{"className":22831},[1182],[57,22833,22835],{"className":22834},[1092],[57,22836,22838],{"className":22837,"style":2230},[1096],[57,22839],{},[57,22841],{"className":22842,"style":2595},[1452],[57,22844,6487],{"className":22845},[1506],[57,22847],{"className":22848,"style":2595},[1452],[57,22850,22852,22855,22858,22898,22901,22950],{"className":22851},[1066],[57,22853],{"className":22854,"style":1831},[1070],[57,22856,19786],{"className":22857,"style":20033},[1075,1148],[57,22859,22861,22864],{"className":22860},[1075],[57,22862,1625],{"className":22863,"style":1655},[1075,1148],[57,22865,22867],{"className":22866},[1152],[57,22868,22870,22890],{"className":22869},[1087,1088],[57,22871,22873,22887],{"className":22872},[1092],[57,22874,22876],{"className":22875,"style":2208},[1096],[57,22877,22878,22881],{"style":21382},[57,22879],{"className":22880,"style":1493},[1104],[57,22882,22884],{"className":22883},[1109,1110,1111,1112],[57,22885,14745],{"className":22886,"style":3932},[1075,1148,1112],[57,22888,1183],{"className":22889},[1182],[57,22891,22893],{"className":22892},[1092],[57,22894,22896],{"className":22895,"style":2230},[1096],[57,22897],{},[57,22899,1754],{"className":22900},[1079],[57,22902,22904,22907],{"className":22903},[1075],[57,22905,13535],{"className":22906},[1075,1148],[57,22908,22910],{"className":22909},[1152],[57,22911,22913,22942],{"className":22912},[1087,1088],[57,22914,22916,22939],{"className":22915},[1092],[57,22917,22919],{"className":22918,"style":3813},[1096],[57,22920,22921,22924],{"style":2211},[57,22922],{"className":22923,"style":1493},[1104],[57,22925,22927],{"className":22926},[1109,1110,1111,1112],[57,22928,22930,22933,22936],{"className":22929},[1075,1112],[57,22931,1733],{"className":22932},[1075,1148,1112],[57,22934,6487],{"className":22935},[1506,1112],[57,22937,574],{"className":22938},[1075,1112],[57,22940,1183],{"className":22941},[1182],[57,22943,22945],{"className":22944},[1092],[57,22946,22948],{"className":22947,"style":7608},[1096],[57,22949],{},[57,22951,1768],{"className":22952},[1269],[57,22954],{"className":22955},[1452,22540],[57,22957,22959,22962,23002,23005,23045,23048,23051,23054,23057,23060,23063],{"className":22958},[1066],[57,22960],{"className":22961,"style":1831},[1070],[57,22963,22965,22968],{"className":22964},[1075],[57,22966,1730],{"className":22967},[1075,1148],[57,22969,22971],{"className":22970},[1152],[57,22972,22974,22994],{"className":22973},[1087,1088],[57,22975,22977,22991],{"className":22976},[1092],[57,22978,22980],{"className":22979,"style":2208},[1096],[57,22981,22982,22985],{"style":2211},[57,22983],{"className":22984,"style":1493},[1104],[57,22986,22988],{"className":22987},[1109,1110,1111,1112],[57,22989,14745],{"className":22990,"style":3932},[1075,1148,1112],[57,22992,1183],{"className":22993},[1182],[57,22995,22997],{"className":22996},[1092],[57,22998,23000],{"className":22999,"style":2230},[1096],[57,23001],{},[57,23003,1754],{"className":23004},[1079],[57,23006,23008,23011],{"className":23007},[1075],[57,23009,13535],{"className":23010},[1075,1148],[57,23012,23014],{"className":23013},[1152],[57,23015,23017,23037],{"className":23016},[1087,1088],[57,23018,23020,23034],{"className":23019},[1092],[57,23021,23023],{"className":23022,"style":13464},[1096],[57,23024,23025,23028],{"style":2211},[57,23026],{"className":23027,"style":1493},[1104],[57,23029,23031],{"className":23030},[1109,1110,1111,1112],[57,23032,1733],{"className":23033},[1075,1148,1112],[57,23035,1183],{"className":23036},[1182],[57,23038,23040],{"className":23039},[1092],[57,23041,23043],{"className":23042,"style":2230},[1096],[57,23044],{},[57,23046,1759],{"className":23047},[1849],[57,23049],{"className":23050,"style":1853},[1452],[57,23052,18],{"className":23053},[1075,1148],[57,23055,1768],{"className":23056},[1269],[57,23058],{"className":23059,"style":1453},[1452],[57,23061,281],{"className":23062},[1457],[57,23064],{"className":23065,"style":1453},[1452],[57,23067,23069,23072,23112,23115,23118],{"className":23068},[1066],[57,23070],{"className":23071,"style":22799},[1070],[57,23073,23075,23078],{"className":23074},[1075],[57,23076,8500],{"className":23077,"style":3932},[1075,1148],[57,23079,23081],{"className":23080},[1152],[57,23082,23084,23104],{"className":23083},[1087,1088],[57,23085,23087,23101],{"className":23086},[1092],[57,23088,23090],{"className":23089,"style":13464},[1096],[57,23091,23092,23095],{"style":13853},[57,23093],{"className":23094,"style":1493},[1104],[57,23096,23098],{"className":23097},[1109,1110,1111,1112],[57,23099,1733],{"className":23100},[1075,1148,1112],[57,23102,1183],{"className":23103},[1182],[57,23105,23107],{"className":23106},[1092],[57,23108,23110],{"className":23109,"style":2230},[1096],[57,23111],{},[57,23113],{"className":23114,"style":2595},[1452],[57,23116,6487],{"className":23117},[1506],[57,23119],{"className":23120,"style":2595},[1452],[57,23122,23124,23127,23130,23170,23173,23222,23225,23228,23231],{"className":23123},[1066],[57,23125],{"className":23126,"style":1831},[1070],[57,23128,19786],{"className":23129,"style":20033},[1075,1148],[57,23131,23133,23136],{"className":23132},[1075],[57,23134,1625],{"className":23135,"style":1655},[1075,1148],[57,23137,23139],{"className":23138},[1152],[57,23140,23142,23162],{"className":23141},[1087,1088],[57,23143,23145,23159],{"className":23144},[1092],[57,23146,23148],{"className":23147,"style":2208},[1096],[57,23149,23150,23153],{"style":21382},[57,23151],{"className":23152,"style":1493},[1104],[57,23154,23156],{"className":23155},[1109,1110,1111,1112],[57,23157,14745],{"className":23158,"style":3932},[1075,1148,1112],[57,23160,1183],{"className":23161},[1182],[57,23163,23165],{"className":23164},[1092],[57,23166,23168],{"className":23167,"style":2230},[1096],[57,23169],{},[57,23171,1754],{"className":23172},[1079],[57,23174,23176,23179],{"className":23175},[1075],[57,23177,13535],{"className":23178},[1075,1148],[57,23180,23182],{"className":23181},[1152],[57,23183,23185,23214],{"className":23184},[1087,1088],[57,23186,23188,23211],{"className":23187},[1092],[57,23189,23191],{"className":23190,"style":3813},[1096],[57,23192,23193,23196],{"style":2211},[57,23194],{"className":23195,"style":1493},[1104],[57,23197,23199],{"className":23198},[1109,1110,1111,1112],[57,23200,23202,23205,23208],{"className":23201},[1075,1112],[57,23203,1733],{"className":23204},[1075,1148,1112],[57,23206,6487],{"className":23207},[1506,1112],[57,23209,574],{"className":23210},[1075,1112],[57,23212,1183],{"className":23213},[1182],[57,23215,23217],{"className":23216},[1092],[57,23218,23220],{"className":23219,"style":7608},[1096],[57,23221],{},[57,23223,1768],{"className":23224},[1269],[57,23226],{"className":23227,"style":2595},[1452],[57,23229,19155],{"className":23230},[1506],[57,23232],{"className":23233,"style":2595},[1452],[57,23235,23237,23240,23280,23283,23323],{"className":23236},[1066],[57,23238],{"className":23239,"style":1831},[1070],[57,23241,23243,23246],{"className":23242},[1075],[57,23244,1625],{"className":23245,"style":1655},[1075,1148],[57,23247,23249],{"className":23248},[1152],[57,23250,23252,23272],{"className":23251},[1087,1088],[57,23253,23255,23269],{"className":23254},[1092],[57,23256,23258],{"className":23257,"style":2208},[1096],[57,23259,23260,23263],{"style":21382},[57,23261],{"className":23262,"style":1493},[1104],[57,23264,23266],{"className":23265},[1109,1110,1111,1112],[57,23267,14745],{"className":23268,"style":3932},[1075,1148,1112],[57,23270,1183],{"className":23271},[1182],[57,23273,23275],{"className":23274},[1092],[57,23276,23278],{"className":23277,"style":2230},[1096],[57,23279],{},[57,23281,1754],{"className":23282},[1079],[57,23284,23286,23289],{"className":23285},[1075],[57,23287,13535],{"className":23288},[1075,1148],[57,23290,23292],{"className":23291},[1152],[57,23293,23295,23315],{"className":23294},[1087,1088],[57,23296,23298,23312],{"className":23297},[1092],[57,23299,23301],{"className":23300,"style":13464},[1096],[57,23302,23303,23306],{"style":2211},[57,23304],{"className":23305,"style":1493},[1104],[57,23307,23309],{"className":23308},[1109,1110,1111,1112],[57,23310,1733],{"className":23311},[1075,1148,1112],[57,23313,1183],{"className":23314},[1182],[57,23316,23318],{"className":23317},[1092],[57,23319,23321],{"className":23320,"style":2230},[1096],[57,23322],{},[57,23324,1768],{"className":23325},[1269],[57,23327],{"className":23328},[1452,22540],[57,23330,23332,23335,23375,23378,23427,23430,23433,23436],{"className":23331},[1066],[57,23333],{"className":23334,"style":1831},[1070],[57,23336,23338,23341],{"className":23337},[1075],[57,23339,1625],{"className":23340,"style":1655},[1075,1148],[57,23342,23344],{"className":23343},[1152],[57,23345,23347,23367],{"className":23346},[1087,1088],[57,23348,23350,23364],{"className":23349},[1092],[57,23351,23353],{"className":23352,"style":2208},[1096],[57,23354,23355,23358],{"style":21382},[57,23356],{"className":23357,"style":1493},[1104],[57,23359,23361],{"className":23360},[1109,1110,1111,1112],[57,23362,14745],{"className":23363,"style":3932},[1075,1148,1112],[57,23365,1183],{"className":23366},[1182],[57,23368,23370],{"className":23369},[1092],[57,23371,23373],{"className":23372,"style":2230},[1096],[57,23374],{},[57,23376,1754],{"className":23377},[1079],[57,23379,23381,23384],{"className":23380},[1075],[57,23382,13535],{"className":23383},[1075,1148],[57,23385,23387],{"className":23386},[1152],[57,23388,23390,23419],{"className":23389},[1087,1088],[57,23391,23393,23416],{"className":23392},[1092],[57,23394,23396],{"className":23395,"style":3813},[1096],[57,23397,23398,23401],{"style":2211},[57,23399],{"className":23400,"style":1493},[1104],[57,23402,23404],{"className":23403},[1109,1110,1111,1112],[57,23405,23407,23410,23413],{"className":23406},[1075,1112],[57,23408,1733],{"className":23409},[1075,1148,1112],[57,23411,6487],{"className":23412},[1506,1112],[57,23414,574],{"className":23415},[1075,1112],[57,23417,1183],{"className":23418},[1182],[57,23420,23422],{"className":23421},[1092],[57,23423,23425],{"className":23424,"style":7608},[1096],[57,23426],{},[57,23428,1768],{"className":23429},[1269],[57,23431],{"className":23432,"style":1453},[1452],[57,23434,15464],{"className":23435},[1457],[57,23437],{"className":23438,"style":1453},[1452],[57,23440,23442,23446,23495,23498,23501],{"className":23441},[1066],[57,23443],{"className":23444,"style":23445},[1070],"height:0.7917em;vertical-align:-0.2083em;",[57,23447,23449,23452],{"className":23448},[1075],[57,23450,8500],{"className":23451,"style":3932},[1075,1148],[57,23453,23455],{"className":23454},[1152],[57,23456,23458,23487],{"className":23457},[1087,1088],[57,23459,23461,23484],{"className":23460},[1092],[57,23462,23464],{"className":23463,"style":3813},[1096],[57,23465,23466,23469],{"style":13853},[57,23467],{"className":23468,"style":1493},[1104],[57,23470,23472],{"className":23471},[1109,1110,1111,1112],[57,23473,23475,23478,23481],{"className":23474},[1075,1112],[57,23476,1733],{"className":23477},[1075,1148,1112],[57,23479,6487],{"className":23480},[1506,1112],[57,23482,574],{"className":23483},[1075,1112],[57,23485,1183],{"className":23486},[1182],[57,23488,23490],{"className":23489},[1092],[57,23491,23493],{"className":23492,"style":7608},[1096],[57,23494],{},[57,23496],{"className":23497,"style":2595},[1452],[57,23499,6487],{"className":23500},[1506],[57,23502],{"className":23503,"style":2595},[1452],[57,23505,23507,23510,23513,23553,23556,23605],{"className":23506},[1066],[57,23508],{"className":23509,"style":1831},[1070],[57,23511,19786],{"className":23512,"style":20033},[1075,1148],[57,23514,23516,23519],{"className":23515},[1075],[57,23517,1625],{"className":23518,"style":1655},[1075,1148],[57,23520,23522],{"className":23521},[1152],[57,23523,23525,23545],{"className":23524},[1087,1088],[57,23526,23528,23542],{"className":23527},[1092],[57,23529,23531],{"className":23530,"style":2208},[1096],[57,23532,23533,23536],{"style":21382},[57,23534],{"className":23535,"style":1493},[1104],[57,23537,23539],{"className":23538},[1109,1110,1111,1112],[57,23540,14745],{"className":23541,"style":3932},[1075,1148,1112],[57,23543,1183],{"className":23544},[1182],[57,23546,23548],{"className":23547},[1092],[57,23549,23551],{"className":23550,"style":2230},[1096],[57,23552],{},[57,23554,1754],{"className":23555},[1079],[57,23557,23559,23562],{"className":23558},[1075],[57,23560,13535],{"className":23561},[1075,1148],[57,23563,23565],{"className":23564},[1152],[57,23566,23568,23597],{"className":23567},[1087,1088],[57,23569,23571,23594],{"className":23570},[1092],[57,23572,23574],{"className":23573,"style":3813},[1096],[57,23575,23576,23579],{"style":2211},[57,23577],{"className":23578,"style":1493},[1104],[57,23580,23582],{"className":23581},[1109,1110,1111,1112],[57,23583,23585,23588,23591],{"className":23584},[1075,1112],[57,23586,1733],{"className":23587},[1075,1148,1112],[57,23589,6487],{"className":23590},[1506,1112],[57,23592,590],{"className":23593},[1075,1112],[57,23595,1183],{"className":23596},[1182],[57,23598,23600],{"className":23599},[1092],[57,23601,23603],{"className":23602,"style":7608},[1096],[57,23604],{},[57,23606,1768],{"className":23607},[1269],[14,23609,23610,23611,23709,23710,23780],{},"经过这一步代换，",[57,23612,23614,23642],{"className":23613},[1019],[57,23615,23617],{"className":23616},[1023],[1025,23618,23619],{"xmlns":1027},[1029,23620,23621,23639],{},[1032,23622,23623,23629,23631,23633,23635,23637],{},[1044,23624,23625,23627],{},[1047,23626,1730],{},[1047,23628,14745],{},[1414,23630,1754],{"stretchy":1753},[1047,23632,13535],{},[1414,23634,1759],{"separator":1062},[1047,23636,18],{},[1414,23638,1768],{"stretchy":1753},[1054,23640,23641],{"encoding":1056},"A_\\theta(s,a)",[57,23643,23645],{"className":23644,"ariaHidden":1062},[1061],[57,23646,23648,23651,23691,23694,23697,23700,23703,23706],{"className":23647},[1066],[57,23649],{"className":23650,"style":1831},[1070],[57,23652,23654,23657],{"className":23653},[1075],[57,23655,1730],{"className":23656},[1075,1148],[57,23658,23660],{"className":23659},[1152],[57,23661,23663,23683],{"className":23662},[1087,1088],[57,23664,23666,23680],{"className":23665},[1092],[57,23667,23669],{"className":23668,"style":2208},[1096],[57,23670,23671,23674],{"style":2211},[57,23672],{"className":23673,"style":1493},[1104],[57,23675,23677],{"className":23676},[1109,1110,1111,1112],[57,23678,14745],{"className":23679,"style":3932},[1075,1148,1112],[57,23681,1183],{"className":23682},[1182],[57,23684,23686],{"className":23685},[1092],[57,23687,23689],{"className":23688,"style":2230},[1096],[57,23690],{},[57,23692,1754],{"className":23693},[1079],[57,23695,13535],{"className":23696},[1075,1148],[57,23698,1759],{"className":23699},[1849],[57,23701],{"className":23702,"style":1853},[1452],[57,23704,18],{"className":23705},[1075,1148],[57,23707,1768],{"className":23708},[1269]," 的估计只依赖一个价值函数 ",[57,23711,23713,23731],{"className":23712},[1019],[57,23714,23716],{"className":23715},[1023],[1025,23717,23718],{"xmlns":1027},[1029,23719,23720,23728],{},[1032,23721,23722],{},[1044,23723,23724,23726],{},[1047,23725,1625],{},[1047,23727,14745],{},[1054,23729,23730],{"encoding":1056},"V_\\theta",[57,23732,23734],{"className":23733,"ariaHidden":1062},[1061],[57,23735,23737,23740],{"className":23736},[1066],[57,23738],{"className":23739,"style":7644},[1070],[57,23741,23743,23746],{"className":23742},[1075],[57,23744,1625],{"className":23745,"style":1655},[1075,1148],[57,23747,23749],{"className":23748},[1152],[57,23750,23752,23772],{"className":23751},[1087,1088],[57,23753,23755,23769],{"className":23754},[1092],[57,23756,23758],{"className":23757,"style":2208},[1096],[57,23759,23760,23763],{"style":21382},[57,23761],{"className":23762,"style":1493},[1104],[57,23764,23766],{"className":23765},[1109,1110,1111,1112],[57,23767,14745],{"className":23768,"style":3932},[1075,1148,1112],[57,23770,1183],{"className":23771},[1182],[57,23773,23775],{"className":23774},[1092],[57,23776,23778],{"className":23777,"style":2230},[1096],[57,23779],{},"，公式整体的复杂度也得以降低。",[14,23782,23783,23784,23853],{},"那么应该向后展开（rollout）多少步再用 ",[57,23785,23787,23804],{"className":23786},[1019],[57,23788,23790],{"className":23789},[1023],[1025,23791,23792],{"xmlns":1027},[1029,23793,23794,23802],{},[1032,23795,23796],{},[1044,23797,23798,23800],{},[1047,23799,1625],{},[1047,23801,14745],{},[1054,23803,23730],{"encoding":1056},[57,23805,23807],{"className":23806,"ariaHidden":1062},[1061],[57,23808,23810,23813],{"className":23809},[1066],[57,23811],{"className":23812,"style":7644},[1070],[57,23814,23816,23819],{"className":23815},[1075],[57,23817,1625],{"className":23818,"style":1655},[1075,1148],[57,23820,23822],{"className":23821},[1152],[57,23823,23825,23845],{"className":23824},[1087,1088],[57,23826,23828,23842],{"className":23827},[1092],[57,23829,23831],{"className":23830,"style":2208},[1096],[57,23832,23833,23836],{"style":21382},[57,23834],{"className":23835,"style":1493},[1104],[57,23837,23839],{"className":23838},[1109,1110,1111,1112],[57,23840,14745],{"className":23841,"style":3932},[1075,1148,1112],[57,23843,1183],{"className":23844},[1182],[57,23846,23848],{"className":23847},[1092],[57,23849,23851],{"className":23850,"style":2230},[1096],[57,23852],{}," 截断呢？展开越多步，估计就越接近真实回报，偏差越小，但方差也会越大；反之展开越少，偏差大、方差小。这就是经典的 bias-variance trade-off：",[57,23855,23857],{"className":23856},[15139],[57,23858,23860,24310],{"className":23859},[1019],[57,23861,23863],{"className":23862},[1023],[1025,23864,23865],{"xmlns":1027,"display":10210},[1029,23866,23867,24307],{},[1032,23868,23869,23877,23879,23885,23887,23889,23891,23893,23899,23901,23903,23909,23911,23923,23925,23927,23933,23935,23941,23943,23945,23953,23955,23961,23963,23965,23967,23969,23975,23977,23979,23991,23993,23999,24005,24007,24019,24021,24023,24029,24031,24037,24039,24041,24049,24051,24057,24059,24061,24063,24065,24071,24073,24075,24078,24090,24092,24098,24100,24112,24114,24120,24126,24128,24140,24142,24144,24150,24152,24158,24160,24162,24174,24176,24184,24186,24192,24194,24196,24198,24200,24206,24208,24210,24212,24224,24226,24232,24234,24246,24248,24254,24256,24268,24270,24273,24275,24281,24283,24289,24291,24297,24299,24305],{},[2545,23870,23871,23873,23875],{},[1047,23872,1730],{},[1047,23874,14745],{},[1038,23876,574],{},[1414,23878,1754],{"stretchy":1753},[1044,23880,23881,23883],{},[1047,23882,13535],{},[1047,23884,1733],{},[1414,23886,1759],{"separator":1062},[1047,23888,18],{},[1414,23890,1768],{"stretchy":1753},[1414,23892,281],{},[1044,23894,23895,23897],{},[1047,23896,8500],{},[1047,23898,1733],{},[1414,23900,6487],{},[1047,23902,19786],{},[1044,23904,23905,23907],{},[1047,23906,1625],{},[1047,23908,14745],{},[1414,23910,1754],{"stretchy":1753},[1044,23912,23913,23915],{},[1047,23914,13535],{},[1032,23916,23917,23919,23921],{},[1047,23918,1733],{},[1414,23920,6487],{},[1038,23922,574],{},[1414,23924,1768],{"stretchy":1753},[1414,23926,19155],{},[1044,23928,23929,23931],{},[1047,23930,1625],{},[1047,23932,14745],{},[1414,23934,1754],{"stretchy":1753},[1044,23936,23937,23939],{},[1047,23938,13535],{},[1047,23940,1733],{},[1414,23942,1768],{"stretchy":1753},[1452,23944],{"linebreak":22540},[2545,23946,23947,23949,23951],{},[1047,23948,1730],{},[1047,23950,14745],{},[1038,23952,590],{},[1414,23954,1754],{"stretchy":1753},[1044,23956,23957,23959],{},[1047,23958,13535],{},[1047,23960,1733],{},[1414,23962,1759],{"separator":1062},[1047,23964,18],{},[1414,23966,1768],{"stretchy":1753},[1414,23968,281],{},[1044,23970,23971,23973],{},[1047,23972,8500],{},[1047,23974,1733],{},[1414,23976,6487],{},[1047,23978,19786],{},[1044,23980,23981,23983],{},[1047,23982,8500],{},[1032,23984,23985,23987,23989],{},[1047,23986,1733],{},[1414,23988,6487],{},[1038,23990,574],{},[1414,23992,6487],{},[1418,23994,23995,23997],{},[1047,23996,19786],{},[1038,23998,590],{},[1044,24000,24001,24003],{},[1047,24002,1625],{},[1047,24004,14745],{},[1414,24006,1754],{"stretchy":1753},[1044,24008,24009,24011],{},[1047,24010,13535],{},[1032,24012,24013,24015,24017],{},[1047,24014,1733],{},[1414,24016,6487],{},[1038,24018,590],{},[1414,24020,1768],{"stretchy":1753},[1414,24022,19155],{},[1044,24024,24025,24027],{},[1047,24026,1625],{},[1047,24028,14745],{},[1414,24030,1754],{"stretchy":1753},[1044,24032,24033,24035],{},[1047,24034,13535],{},[1047,24036,1733],{},[1414,24038,1768],{"stretchy":1753},[1452,24040],{"linebreak":22540},[2545,24042,24043,24045,24047],{},[1047,24044,1730],{},[1047,24046,14745],{},[1038,24048,606],{},[1414,24050,1754],{"stretchy":1753},[1044,24052,24053,24055],{},[1047,24054,13535],{},[1047,24056,1733],{},[1414,24058,1759],{"separator":1062},[1047,24060,18],{},[1414,24062,1768],{"stretchy":1753},[1414,24064,281],{},[1044,24066,24067,24069],{},[1047,24068,8500],{},[1047,24070,1733],{},[1414,24072,6487],{},[1047,24074,19786],{},[1414,24076,24077],{},"∗",[1044,24079,24080,24082],{},[1047,24081,8500],{},[1032,24083,24084,24086,24088],{},[1047,24085,1733],{},[1414,24087,6487],{},[1038,24089,574],{},[1414,24091,6487],{},[1418,24093,24094,24096],{},[1047,24095,19786],{},[1038,24097,590],{},[1414,24099,24077],{},[1044,24101,24102,24104],{},[1047,24103,8500],{},[1032,24105,24106,24108,24110],{},[1047,24107,1733],{},[1414,24109,6487],{},[1038,24111,590],{},[1414,24113,6487],{},[1418,24115,24116,24118],{},[1047,24117,19786],{},[1038,24119,606],{},[1044,24121,24122,24124],{},[1047,24123,1625],{},[1047,24125,14745],{},[1414,24127,1754],{"stretchy":1753},[1044,24129,24130,24132],{},[1047,24131,13535],{},[1032,24133,24134,24136,24138],{},[1047,24135,1733],{},[1414,24137,6487],{},[1038,24139,606],{},[1414,24141,1768],{"stretchy":1753},[1414,24143,19155],{},[1044,24145,24146,24148],{},[1047,24147,1625],{},[1047,24149,14745],{},[1414,24151,1754],{"stretchy":1753},[1044,24153,24154,24156],{},[1047,24155,13535],{},[1047,24157,1733],{},[1414,24159,1768],{"stretchy":1753},[1452,24161],{"linebreak":22540},[1032,24163,24164,24167],{},[1047,24165,24166],{"mathvariant":1729},"⋮",[24168,24169,24170],"mpadded",{"height":15155,"voffset":15155},[1452,24171],{"mathbackground":24172,"width":15155,"height":24173},"black","1.5em",[1452,24175],{"linebreak":22540},[2545,24177,24178,24180,24182],{},[1047,24179,1730],{},[1047,24181,14745],{},[1047,24183,738],{},[1414,24185,1754],{"stretchy":1753},[1044,24187,24188,24190],{},[1047,24189,13535],{},[1047,24191,1733],{},[1414,24193,1759],{"separator":1062},[1047,24195,18],{},[1414,24197,1768],{"stretchy":1753},[1414,24199,281],{},[1044,24201,24202,24204],{},[1047,24203,8500],{},[1047,24205,1733],{},[1414,24207,6487],{},[1047,24209,19786],{},[1414,24211,24077],{},[1044,24213,24214,24216],{},[1047,24215,8500],{},[1032,24217,24218,24220,24222],{},[1047,24219,1733],{},[1414,24221,6487],{},[1038,24223,574],{},[1414,24225,6487],{},[1418,24227,24228,24230],{},[1047,24229,19786],{},[1038,24231,590],{},[1414,24233,24077],{},[1044,24235,24236,24238],{},[1047,24237,8500],{},[1032,24239,24240,24242,24244],{},[1047,24241,1733],{},[1414,24243,6487],{},[1038,24245,590],{},[1414,24247,6487],{},[1418,24249,24250,24252],{},[1047,24251,19786],{},[1038,24253,606],{},[1414,24255,24077],{},[1044,24257,24258,24260],{},[1047,24259,8500],{},[1032,24261,24262,24264,24266],{},[1047,24263,1733],{},[1414,24265,6487],{},[1038,24267,606],{},[1414,24269,6487],{},[1414,24271,24272],{},"⋯",[1414,24274,6487],{},[1418,24276,24277,24279],{},[1047,24278,19786],{},[1047,24280,738],{},[1414,24282,24077],{},[1044,24284,24285,24287],{},[1047,24286,8500],{},[1047,24288,738],{},[1414,24290,19155],{},[1044,24292,24293,24295],{},[1047,24294,1625],{},[1047,24296,14745],{},[1414,24298,1754],{"stretchy":1753},[1044,24300,24301,24303],{},[1047,24302,13535],{},[1047,24304,1733],{},[1414,24306,1768],{"stretchy":1753},[1054,24308,24309],{"encoding":1056}," A_\\theta^1(s_t, a) = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n A_\\theta^2(s_t, a) = r_t + \\gamma r_{t+1} + \\gamma^2 V_\\theta(s_{t+2}) - V_\\theta(s_t)\\\\\n A_\\theta^3(s_t, a) = r_t + \\gamma * r_{t+1} + \\gamma^2 * r_{t+2} + \\gamma^3 V_\\theta(s_{t+3}) - V_\\theta(s_t)\\\\\n \\vdots \\\\\n A_\\theta^T(s_t, a) = r_t + \\gamma * r_{t+1} + \\gamma^2 * r_{t+2} + \\gamma^3 * r_{t+3} + \\cdots + \\gamma^T * r_T - V_\\theta(s_t)",[57,24311,24313,24436,24491,24604,24696,24699,24820,24875,24942,25081,25173,25176,25297,25352,25371,25435,25480,25544,25683,25775,25778,25796,25799,25922,25977,25995,26059,26103,26167,26211,26275,26293,26338,26393],{"className":24312,"ariaHidden":1062},[1061],[57,24314,24316,24320,24372,24375,24415,24418,24421,24424,24427,24430,24433],{"className":24315},[1066],[57,24317],{"className":24318,"style":24319},[1070],"height:1.1141em;vertical-align:-0.25em;",[57,24321,24323,24326],{"className":24322},[1075],[57,24324,1730],{"className":24325},[1075,1148],[57,24327,24329],{"className":24328},[1152],[57,24330,24332,24364],{"className":24331},[1087,1088],[57,24333,24335,24361],{"className":24334},[1092],[57,24336,24339,24350],{"className":24337,"style":24338},[1096],"height:0.8641em;",[57,24340,24341,24344],{"style":18101},[57,24342],{"className":24343,"style":1493},[1104],[57,24345,24347],{"className":24346},[1109,1110,1111,1112],[57,24348,14745],{"className":24349,"style":3932},[1075,1148,1112],[57,24351,24352,24355],{"style":17216},[57,24353],{"className":24354,"style":1493},[1104],[57,24356,24358],{"className":24357},[1109,1110,1111,1112],[57,24359,574],{"className":24360},[1075,1112],[57,24362,1183],{"className":24363},[1182],[57,24365,24367],{"className":24366},[1092],[57,24368,24370],{"className":24369,"style":18131},[1096],[57,24371],{},[57,24373,1754],{"className":24374},[1079],[57,24376,24378,24381],{"className":24377},[1075],[57,24379,13535],{"className":24380},[1075,1148],[57,24382,24384],{"className":24383},[1152],[57,24385,24387,24407],{"className":24386},[1087,1088],[57,24388,24390,24404],{"className":24389},[1092],[57,24391,24393],{"className":24392,"style":13464},[1096],[57,24394,24395,24398],{"style":2211},[57,24396],{"className":24397,"style":1493},[1104],[57,24399,24401],{"className":24400},[1109,1110,1111,1112],[57,24402,1733],{"className":24403},[1075,1148,1112],[57,24405,1183],{"className":24406},[1182],[57,24408,24410],{"className":24409},[1092],[57,24411,24413],{"className":24412,"style":2230},[1096],[57,24414],{},[57,24416,1759],{"className":24417},[1849],[57,24419],{"className":24420,"style":1853},[1452],[57,24422,18],{"className":24423},[1075,1148],[57,24425,1768],{"className":24426},[1269],[57,24428],{"className":24429,"style":1453},[1452],[57,24431,281],{"className":24432},[1457],[57,24434],{"className":24435,"style":1453},[1452],[57,24437,24439,24442,24482,24485,24488],{"className":24438},[1066],[57,24440],{"className":24441,"style":22799},[1070],[57,24443,24445,24448],{"className":24444},[1075],[57,24446,8500],{"className":24447,"style":3932},[1075,1148],[57,24449,24451],{"className":24450},[1152],[57,24452,24454,24474],{"className":24453},[1087,1088],[57,24455,24457,24471],{"className":24456},[1092],[57,24458,24460],{"className":24459,"style":13464},[1096],[57,24461,24462,24465],{"style":13853},[57,24463],{"className":24464,"style":1493},[1104],[57,24466,24468],{"className":24467},[1109,1110,1111,1112],[57,24469,1733],{"className":24470},[1075,1148,1112],[57,24472,1183],{"className":24473},[1182],[57,24475,24477],{"className":24476},[1092],[57,24478,24480],{"className":24479,"style":2230},[1096],[57,24481],{},[57,24483],{"className":24484,"style":2595},[1452],[57,24486,6487],{"className":24487},[1506],[57,24489],{"className":24490,"style":2595},[1452],[57,24492,24494,24497,24500,24540,24543,24592,24595,24598,24601],{"className":24493},[1066],[57,24495],{"className":24496,"style":1831},[1070],[57,24498,19786],{"className":24499,"style":20033},[1075,1148],[57,24501,24503,24506],{"className":24502},[1075],[57,24504,1625],{"className":24505,"style":1655},[1075,1148],[57,24507,24509],{"className":24508},[1152],[57,24510,24512,24532],{"className":24511},[1087,1088],[57,24513,24515,24529],{"className":24514},[1092],[57,24516,24518],{"className":24517,"style":2208},[1096],[57,24519,24520,24523],{"style":21382},[57,24521],{"className":24522,"style":1493},[1104],[57,24524,24526],{"className":24525},[1109,1110,1111,1112],[57,24527,14745],{"className":24528,"style":3932},[1075,1148,1112],[57,24530,1183],{"className":24531},[1182],[57,24533,24535],{"className":24534},[1092],[57,24536,24538],{"className":24537,"style":2230},[1096],[57,24539],{},[57,24541,1754],{"className":24542},[1079],[57,24544,24546,24549],{"className":24545},[1075],[57,24547,13535],{"className":24548},[1075,1148],[57,24550,24552],{"className":24551},[1152],[57,24553,24555,24584],{"className":24554},[1087,1088],[57,24556,24558,24581],{"className":24557},[1092],[57,24559,24561],{"className":24560,"style":3813},[1096],[57,24562,24563,24566],{"style":2211},[57,24564],{"className":24565,"style":1493},[1104],[57,24567,24569],{"className":24568},[1109,1110,1111,1112],[57,24570,24572,24575,24578],{"className":24571},[1075,1112],[57,24573,1733],{"className":24574},[1075,1148,1112],[57,24576,6487],{"className":24577},[1506,1112],[57,24579,574],{"className":24580},[1075,1112],[57,24582,1183],{"className":24583},[1182],[57,24585,24587],{"className":24586},[1092],[57,24588,24590],{"className":24589,"style":7608},[1096],[57,24591],{},[57,24593,1768],{"className":24594},[1269],[57,24596],{"className":24597,"style":2595},[1452],[57,24599,19155],{"className":24600},[1506],[57,24602],{"className":24603,"style":2595},[1452],[57,24605,24607,24610,24650,24653,24693],{"className":24606},[1066],[57,24608],{"className":24609,"style":1831},[1070],[57,24611,24613,24616],{"className":24612},[1075],[57,24614,1625],{"className":24615,"style":1655},[1075,1148],[57,24617,24619],{"className":24618},[1152],[57,24620,24622,24642],{"className":24621},[1087,1088],[57,24623,24625,24639],{"className":24624},[1092],[57,24626,24628],{"className":24627,"style":2208},[1096],[57,24629,24630,24633],{"style":21382},[57,24631],{"className":24632,"style":1493},[1104],[57,24634,24636],{"className":24635},[1109,1110,1111,1112],[57,24637,14745],{"className":24638,"style":3932},[1075,1148,1112],[57,24640,1183],{"className":24641},[1182],[57,24643,24645],{"className":24644},[1092],[57,24646,24648],{"className":24647,"style":2230},[1096],[57,24649],{},[57,24651,1754],{"className":24652},[1079],[57,24654,24656,24659],{"className":24655},[1075],[57,24657,13535],{"className":24658},[1075,1148],[57,24660,24662],{"className":24661},[1152],[57,24663,24665,24685],{"className":24664},[1087,1088],[57,24666,24668,24682],{"className":24667},[1092],[57,24669,24671],{"className":24670,"style":13464},[1096],[57,24672,24673,24676],{"style":2211},[57,24674],{"className":24675,"style":1493},[1104],[57,24677,24679],{"className":24678},[1109,1110,1111,1112],[57,24680,1733],{"className":24681},[1075,1148,1112],[57,24683,1183],{"className":24684},[1182],[57,24686,24688],{"className":24687},[1092],[57,24689,24691],{"className":24690,"style":2230},[1096],[57,24692],{},[57,24694,1768],{"className":24695},[1269],[57,24697],{"className":24698},[1452,22540],[57,24700,24702,24705,24756,24759,24799,24802,24805,24808,24811,24814,24817],{"className":24701},[1066],[57,24703],{"className":24704,"style":24319},[1070],[57,24706,24708,24711],{"className":24707},[1075],[57,24709,1730],{"className":24710},[1075,1148],[57,24712,24714],{"className":24713},[1152],[57,24715,24717,24748],{"className":24716},[1087,1088],[57,24718,24720,24745],{"className":24719},[1092],[57,24721,24723,24734],{"className":24722,"style":24338},[1096],[57,24724,24725,24728],{"style":18101},[57,24726],{"className":24727,"style":1493},[1104],[57,24729,24731],{"className":24730},[1109,1110,1111,1112],[57,24732,14745],{"className":24733,"style":3932},[1075,1148,1112],[57,24735,24736,24739],{"style":17216},[57,24737],{"className":24738,"style":1493},[1104],[57,24740,24742],{"className":24741},[1109,1110,1111,1112],[57,24743,590],{"className":24744},[1075,1112],[57,24746,1183],{"className":24747},[1182],[57,24749,24751],{"className":24750},[1092],[57,24752,24754],{"className":24753,"style":18131},[1096],[57,24755],{},[57,24757,1754],{"className":24758},[1079],[57,24760,24762,24765],{"className":24761},[1075],[57,24763,13535],{"className":24764},[1075,1148],[57,24766,24768],{"className":24767},[1152],[57,24769,24771,24791],{"className":24770},[1087,1088],[57,24772,24774,24788],{"className":24773},[1092],[57,24775,24777],{"className":24776,"style":13464},[1096],[57,24778,24779,24782],{"style":2211},[57,24780],{"className":24781,"style":1493},[1104],[57,24783,24785],{"className":24784},[1109,1110,1111,1112],[57,24786,1733],{"className":24787},[1075,1148,1112],[57,24789,1183],{"className":24790},[1182],[57,24792,24794],{"className":24793},[1092],[57,24795,24797],{"className":24796,"style":2230},[1096],[57,24798],{},[57,24800,1759],{"className":24801},[1849],[57,24803],{"className":24804,"style":1853},[1452],[57,24806,18],{"className":24807},[1075,1148],[57,24809,1768],{"className":24810},[1269],[57,24812],{"className":24813,"style":1453},[1452],[57,24815,281],{"className":24816},[1457],[57,24818],{"className":24819,"style":1453},[1452],[57,24821,24823,24826,24866,24869,24872],{"className":24822},[1066],[57,24824],{"className":24825,"style":22799},[1070],[57,24827,24829,24832],{"className":24828},[1075],[57,24830,8500],{"className":24831,"style":3932},[1075,1148],[57,24833,24835],{"className":24834},[1152],[57,24836,24838,24858],{"className":24837},[1087,1088],[57,24839,24841,24855],{"className":24840},[1092],[57,24842,24844],{"className":24843,"style":13464},[1096],[57,24845,24846,24849],{"style":13853},[57,24847],{"className":24848,"style":1493},[1104],[57,24850,24852],{"className":24851},[1109,1110,1111,1112],[57,24853,1733],{"className":24854},[1075,1148,1112],[57,24856,1183],{"className":24857},[1182],[57,24859,24861],{"className":24860},[1092],[57,24862,24864],{"className":24863,"style":2230},[1096],[57,24865],{},[57,24867],{"className":24868,"style":2595},[1452],[57,24870,6487],{"className":24871},[1506],[57,24873],{"className":24874,"style":2595},[1452],[57,24876,24878,24881,24884,24933,24936,24939],{"className":24877},[1066],[57,24879],{"className":24880,"style":23445},[1070],[57,24882,19786],{"className":24883,"style":20033},[1075,1148],[57,24885,24887,24890],{"className":24886},[1075],[57,24888,8500],{"className":24889,"style":3932},[1075,1148],[57,24891,24893],{"className":24892},[1152],[57,24894,24896,24925],{"className":24895},[1087,1088],[57,24897,24899,24922],{"className":24898},[1092],[57,24900,24902],{"className":24901,"style":3813},[1096],[57,24903,24904,24907],{"style":13853},[57,24905],{"className":24906,"style":1493},[1104],[57,24908,24910],{"className":24909},[1109,1110,1111,1112],[57,24911,24913,24916,24919],{"className":24912},[1075,1112],[57,24914,1733],{"className":24915},[1075,1148,1112],[57,24917,6487],{"className":24918},[1506,1112],[57,24920,574],{"className":24921},[1075,1112],[57,24923,1183],{"className":24924},[1182],[57,24926,24928],{"className":24927},[1092],[57,24929,24931],{"className":24930,"style":7608},[1096],[57,24932],{},[57,24934],{"className":24935,"style":2595},[1452],[57,24937,6487],{"className":24938},[1506],[57,24940],{"className":24941,"style":2595},[1452],[57,24943,24945,24948,24977,25017,25020,25069,25072,25075,25078],{"className":24944},[1066],[57,24946],{"className":24947,"style":24319},[1070],[57,24949,24951,24954],{"className":24950},[1075],[57,24952,19786],{"className":24953,"style":20033},[1075,1148],[57,24955,24957],{"className":24956},[1152],[57,24958,24960],{"className":24959},[1087],[57,24961,24963],{"className":24962},[1092],[57,24964,24966],{"className":24965,"style":24338},[1096],[57,24967,24968,24971],{"style":17216},[57,24969],{"className":24970,"style":1493},[1104],[57,24972,24974],{"className":24973},[1109,1110,1111,1112],[57,24975,590],{"className":24976},[1075,1112],[57,24978,24980,24983],{"className":24979},[1075],[57,24981,1625],{"className":24982,"style":1655},[1075,1148],[57,24984,24986],{"className":24985},[1152],[57,24987,24989,25009],{"className":24988},[1087,1088],[57,24990,24992,25006],{"className":24991},[1092],[57,24993,24995],{"className":24994,"style":2208},[1096],[57,24996,24997,25000],{"style":21382},[57,24998],{"className":24999,"style":1493},[1104],[57,25001,25003],{"className":25002},[1109,1110,1111,1112],[57,25004,14745],{"className":25005,"style":3932},[1075,1148,1112],[57,25007,1183],{"className":25008},[1182],[57,25010,25012],{"className":25011},[1092],[57,25013,25015],{"className":25014,"style":2230},[1096],[57,25016],{},[57,25018,1754],{"className":25019},[1079],[57,25021,25023,25026],{"className":25022},[1075],[57,25024,13535],{"className":25025},[1075,1148],[57,25027,25029],{"className":25028},[1152],[57,25030,25032,25061],{"className":25031},[1087,1088],[57,25033,25035,25058],{"className":25034},[1092],[57,25036,25038],{"className":25037,"style":3813},[1096],[57,25039,25040,25043],{"style":2211},[57,25041],{"className":25042,"style":1493},[1104],[57,25044,25046],{"className":25045},[1109,1110,1111,1112],[57,25047,25049,25052,25055],{"className":25048},[1075,1112],[57,25050,1733],{"className":25051},[1075,1148,1112],[57,25053,6487],{"className":25054},[1506,1112],[57,25056,590],{"className":25057},[1075,1112],[57,25059,1183],{"className":25060},[1182],[57,25062,25064],{"className":25063},[1092],[57,25065,25067],{"className":25066,"style":7608},[1096],[57,25068],{},[57,25070,1768],{"className":25071},[1269],[57,25073],{"className":25074,"style":2595},[1452],[57,25076,19155],{"className":25077},[1506],[57,25079],{"className":25080,"style":2595},[1452],[57,25082,25084,25087,25127,25130,25170],{"className":25083},[1066],[57,25085],{"className":25086,"style":1831},[1070],[57,25088,25090,25093],{"className":25089},[1075],[57,25091,1625],{"className":25092,"style":1655},[1075,1148],[57,25094,25096],{"className":25095},[1152],[57,25097,25099,25119],{"className":25098},[1087,1088],[57,25100,25102,25116],{"className":25101},[1092],[57,25103,25105],{"className":25104,"style":2208},[1096],[57,25106,25107,25110],{"style":21382},[57,25108],{"className":25109,"style":1493},[1104],[57,25111,25113],{"className":25112},[1109,1110,1111,1112],[57,25114,14745],{"className":25115,"style":3932},[1075,1148,1112],[57,25117,1183],{"className":25118},[1182],[57,25120,25122],{"className":25121},[1092],[57,25123,25125],{"className":25124,"style":2230},[1096],[57,25126],{},[57,25128,1754],{"className":25129},[1079],[57,25131,25133,25136],{"className":25132},[1075],[57,25134,13535],{"className":25135},[1075,1148],[57,25137,25139],{"className":25138},[1152],[57,25140,25142,25162],{"className":25141},[1087,1088],[57,25143,25145,25159],{"className":25144},[1092],[57,25146,25148],{"className":25147,"style":13464},[1096],[57,25149,25150,25153],{"style":2211},[57,25151],{"className":25152,"style":1493},[1104],[57,25154,25156],{"className":25155},[1109,1110,1111,1112],[57,25157,1733],{"className":25158},[1075,1148,1112],[57,25160,1183],{"className":25161},[1182],[57,25163,25165],{"className":25164},[1092],[57,25166,25168],{"className":25167,"style":2230},[1096],[57,25169],{},[57,25171,1768],{"className":25172},[1269],[57,25174],{"className":25175},[1452,22540],[57,25177,25179,25182,25233,25236,25276,25279,25282,25285,25288,25291,25294],{"className":25178},[1066],[57,25180],{"className":25181,"style":24319},[1070],[57,25183,25185,25188],{"className":25184},[1075],[57,25186,1730],{"className":25187},[1075,1148],[57,25189,25191],{"className":25190},[1152],[57,25192,25194,25225],{"className":25193},[1087,1088],[57,25195,25197,25222],{"className":25196},[1092],[57,25198,25200,25211],{"className":25199,"style":24338},[1096],[57,25201,25202,25205],{"style":18101},[57,25203],{"className":25204,"style":1493},[1104],[57,25206,25208],{"className":25207},[1109,1110,1111,1112],[57,25209,14745],{"className":25210,"style":3932},[1075,1148,1112],[57,25212,25213,25216],{"style":17216},[57,25214],{"className":25215,"style":1493},[1104],[57,25217,25219],{"className":25218},[1109,1110,1111,1112],[57,25220,606],{"className":25221},[1075,1112],[57,25223,1183],{"className":25224},[1182],[57,25226,25228],{"className":25227},[1092],[57,25229,25231],{"className":25230,"style":18131},[1096],[57,25232],{},[57,25234,1754],{"className":25235},[1079],[57,25237,25239,25242],{"className":25238},[1075],[57,25240,13535],{"className":25241},[1075,1148],[57,25243,25245],{"className":25244},[1152],[57,25246,25248,25268],{"className":25247},[1087,1088],[57,25249,25251,25265],{"className":25250},[1092],[57,25252,25254],{"className":25253,"style":13464},[1096],[57,25255,25256,25259],{"style":2211},[57,25257],{"className":25258,"style":1493},[1104],[57,25260,25262],{"className":25261},[1109,1110,1111,1112],[57,25263,1733],{"className":25264},[1075,1148,1112],[57,25266,1183],{"className":25267},[1182],[57,25269,25271],{"className":25270},[1092],[57,25272,25274],{"className":25273,"style":2230},[1096],[57,25275],{},[57,25277,1759],{"className":25278},[1849],[57,25280],{"className":25281,"style":1853},[1452],[57,25283,18],{"className":25284},[1075,1148],[57,25286,1768],{"className":25287},[1269],[57,25289],{"className":25290,"style":1453},[1452],[57,25292,281],{"className":25293},[1457],[57,25295],{"className":25296,"style":1453},[1452],[57,25298,25300,25303,25343,25346,25349],{"className":25299},[1066],[57,25301],{"className":25302,"style":22799},[1070],[57,25304,25306,25309],{"className":25305},[1075],[57,25307,8500],{"className":25308,"style":3932},[1075,1148],[57,25310,25312],{"className":25311},[1152],[57,25313,25315,25335],{"className":25314},[1087,1088],[57,25316,25318,25332],{"className":25317},[1092],[57,25319,25321],{"className":25320,"style":13464},[1096],[57,25322,25323,25326],{"style":13853},[57,25324],{"className":25325,"style":1493},[1104],[57,25327,25329],{"className":25328},[1109,1110,1111,1112],[57,25330,1733],{"className":25331},[1075,1148,1112],[57,25333,1183],{"className":25334},[1182],[57,25336,25338],{"className":25337},[1092],[57,25339,25341],{"className":25340,"style":2230},[1096],[57,25342],{},[57,25344],{"className":25345,"style":2595},[1452],[57,25347,6487],{"className":25348},[1506],[57,25350],{"className":25351,"style":2595},[1452],[57,25353,25355,25359,25362,25365,25368],{"className":25354},[1066],[57,25356],{"className":25357,"style":25358},[1070],"height:0.6597em;vertical-align:-0.1944em;",[57,25360,19786],{"className":25361,"style":20033},[1075,1148],[57,25363],{"className":25364,"style":2595},[1452],[57,25366,24077],{"className":25367},[1506],[57,25369],{"className":25370,"style":2595},[1452],[57,25372,25374,25377,25426,25429,25432],{"className":25373},[1066],[57,25375],{"className":25376,"style":23445},[1070],[57,25378,25380,25383],{"className":25379},[1075],[57,25381,8500],{"className":25382,"style":3932},[1075,1148],[57,25384,25386],{"className":25385},[1152],[57,25387,25389,25418],{"className":25388},[1087,1088],[57,25390,25392,25415],{"className":25391},[1092],[57,25393,25395],{"className":25394,"style":3813},[1096],[57,25396,25397,25400],{"style":13853},[57,25398],{"className":25399,"style":1493},[1104],[57,25401,25403],{"className":25402},[1109,1110,1111,1112],[57,25404,25406,25409,25412],{"className":25405},[1075,1112],[57,25407,1733],{"className":25408},[1075,1148,1112],[57,25410,6487],{"className":25411},[1506,1112],[57,25413,574],{"className":25414},[1075,1112],[57,25416,1183],{"className":25417},[1182],[57,25419,25421],{"className":25420},[1092],[57,25422,25424],{"className":25423,"style":7608},[1096],[57,25425],{},[57,25427],{"className":25428,"style":2595},[1452],[57,25430,6487],{"className":25431},[1506],[57,25433],{"className":25434,"style":2595},[1452],[57,25436,25438,25442,25471,25474,25477],{"className":25437},[1066],[57,25439],{"className":25440,"style":25441},[1070],"height:1.0585em;vertical-align:-0.1944em;",[57,25443,25445,25448],{"className":25444},[1075],[57,25446,19786],{"className":25447,"style":20033},[1075,1148],[57,25449,25451],{"className":25450},[1152],[57,25452,25454],{"className":25453},[1087],[57,25455,25457],{"className":25456},[1092],[57,25458,25460],{"className":25459,"style":24338},[1096],[57,25461,25462,25465],{"style":17216},[57,25463],{"className":25464,"style":1493},[1104],[57,25466,25468],{"className":25467},[1109,1110,1111,1112],[57,25469,590],{"className":25470},[1075,1112],[57,25472],{"className":25473,"style":2595},[1452],[57,25475,24077],{"className":25476},[1506],[57,25478],{"className":25479,"style":2595},[1452],[57,25481,25483,25486,25535,25538,25541],{"className":25482},[1066],[57,25484],{"className":25485,"style":23445},[1070],[57,25487,25489,25492],{"className":25488},[1075],[57,25490,8500],{"className":25491,"style":3932},[1075,1148],[57,25493,25495],{"className":25494},[1152],[57,25496,25498,25527],{"className":25497},[1087,1088],[57,25499,25501,25524],{"className":25500},[1092],[57,25502,25504],{"className":25503,"style":3813},[1096],[57,25505,25506,25509],{"style":13853},[57,25507],{"className":25508,"style":1493},[1104],[57,25510,25512],{"className":25511},[1109,1110,1111,1112],[57,25513,25515,25518,25521],{"className":25514},[1075,1112],[57,25516,1733],{"className":25517},[1075,1148,1112],[57,25519,6487],{"className":25520},[1506,1112],[57,25522,590],{"className":25523},[1075,1112],[57,25525,1183],{"className":25526},[1182],[57,25528,25530],{"className":25529},[1092],[57,25531,25533],{"className":25532,"style":7608},[1096],[57,25534],{},[57,25536],{"className":25537,"style":2595},[1452],[57,25539,6487],{"className":25540},[1506],[57,25542],{"className":25543,"style":2595},[1452],[57,25545,25547,25550,25579,25619,25622,25671,25674,25677,25680],{"className":25546},[1066],[57,25548],{"className":25549,"style":24319},[1070],[57,25551,25553,25556],{"className":25552},[1075],[57,25554,19786],{"className":25555,"style":20033},[1075,1148],[57,25557,25559],{"className":25558},[1152],[57,25560,25562],{"className":25561},[1087],[57,25563,25565],{"className":25564},[1092],[57,25566,25568],{"className":25567,"style":24338},[1096],[57,25569,25570,25573],{"style":17216},[57,25571],{"className":25572,"style":1493},[1104],[57,25574,25576],{"className":25575},[1109,1110,1111,1112],[57,25577,606],{"className":25578},[1075,1112],[57,25580,25582,25585],{"className":25581},[1075],[57,25583,1625],{"className":25584,"style":1655},[1075,1148],[57,25586,25588],{"className":25587},[1152],[57,25589,25591,25611],{"className":25590},[1087,1088],[57,25592,25594,25608],{"className":25593},[1092],[57,25595,25597],{"className":25596,"style":2208},[1096],[57,25598,25599,25602],{"style":21382},[57,25600],{"className":25601,"style":1493},[1104],[57,25603,25605],{"className":25604},[1109,1110,1111,1112],[57,25606,14745],{"className":25607,"style":3932},[1075,1148,1112],[57,25609,1183],{"className":25610},[1182],[57,25612,25614],{"className":25613},[1092],[57,25615,25617],{"className":25616,"style":2230},[1096],[57,25618],{},[57,25620,1754],{"className":25621},[1079],[57,25623,25625,25628],{"className":25624},[1075],[57,25626,13535],{"className":25627},[1075,1148],[57,25629,25631],{"className":25630},[1152],[57,25632,25634,25663],{"className":25633},[1087,1088],[57,25635,25637,25660],{"className":25636},[1092],[57,25638,25640],{"className":25639,"style":3813},[1096],[57,25641,25642,25645],{"style":2211},[57,25643],{"className":25644,"style":1493},[1104],[57,25646,25648],{"className":25647},[1109,1110,1111,1112],[57,25649,25651,25654,25657],{"className":25650},[1075,1112],[57,25652,1733],{"className":25653},[1075,1148,1112],[57,25655,6487],{"className":25656},[1506,1112],[57,25658,606],{"className":25659},[1075,1112],[57,25661,1183],{"className":25662},[1182],[57,25664,25666],{"className":25665},[1092],[57,25667,25669],{"className":25668,"style":7608},[1096],[57,25670],{},[57,25672,1768],{"className":25673},[1269],[57,25675],{"className":25676,"style":2595},[1452],[57,25678,19155],{"className":25679},[1506],[57,25681],{"className":25682,"style":2595},[1452],[57,25684,25686,25689,25729,25732,25772],{"className":25685},[1066],[57,25687],{"className":25688,"style":1831},[1070],[57,25690,25692,25695],{"className":25691},[1075],[57,25693,1625],{"className":25694,"style":1655},[1075,1148],[57,25696,25698],{"className":25697},[1152],[57,25699,25701,25721],{"className":25700},[1087,1088],[57,25702,25704,25718],{"className":25703},[1092],[57,25705,25707],{"className":25706,"style":2208},[1096],[57,25708,25709,25712],{"style":21382},[57,25710],{"className":25711,"style":1493},[1104],[57,25713,25715],{"className":25714},[1109,1110,1111,1112],[57,25716,14745],{"className":25717,"style":3932},[1075,1148,1112],[57,25719,1183],{"className":25720},[1182],[57,25722,25724],{"className":25723},[1092],[57,25725,25727],{"className":25726,"style":2230},[1096],[57,25728],{},[57,25730,1754],{"className":25731},[1079],[57,25733,25735,25738],{"className":25734},[1075],[57,25736,13535],{"className":25737},[1075,1148],[57,25739,25741],{"className":25740},[1152],[57,25742,25744,25764],{"className":25743},[1087,1088],[57,25745,25747,25761],{"className":25746},[1092],[57,25748,25750],{"className":25749,"style":13464},[1096],[57,25751,25752,25755],{"style":2211},[57,25753],{"className":25754,"style":1493},[1104],[57,25756,25758],{"className":25757},[1109,1110,1111,1112],[57,25759,1733],{"className":25760},[1075,1148,1112],[57,25762,1183],{"className":25763},[1182],[57,25765,25767],{"className":25766},[1092],[57,25768,25770],{"className":25769,"style":2230},[1096],[57,25771],{},[57,25773,1768],{"className":25774},[1269],[57,25776],{"className":25777},[1452,22540],[57,25779,25781,25785],{"className":25780},[1066],[57,25782],{"className":25783,"style":25784},[1070],"height:1.53em;vertical-align:-0.03em;",[57,25786,25788,25791],{"className":25787},[1075],[57,25789,24166],{"className":25790},[1075],[57,25792],{"className":25793,"style":25795},[1075,25794],"rule","border-right-width:0em;border-top-width:1.5em;bottom:0em;",[57,25797],{"className":25798},[1452,22540],[57,25800,25802,25806,25858,25861,25901,25904,25907,25910,25913,25916,25919],{"className":25801},[1066],[57,25803],{"className":25804,"style":25805},[1070],"height:1.1413em;vertical-align:-0.25em;",[57,25807,25809,25812],{"className":25808},[1075],[57,25810,1730],{"className":25811},[1075,1148],[57,25813,25815],{"className":25814},[1152],[57,25816,25818,25850],{"className":25817},[1087,1088],[57,25819,25821,25847],{"className":25820},[1092],[57,25822,25825,25836],{"className":25823,"style":25824},[1096],"height:0.8913em;",[57,25826,25827,25830],{"style":18101},[57,25828],{"className":25829,"style":1493},[1104],[57,25831,25833],{"className":25832},[1109,1110,1111,1112],[57,25834,14745],{"className":25835,"style":3932},[1075,1148,1112],[57,25837,25838,25841],{"style":17216},[57,25839],{"className":25840,"style":1493},[1104],[57,25842,25844],{"className":25843},[1109,1110,1111,1112],[57,25845,738],{"className":25846,"style":3907},[1075,1148,1112],[57,25848,1183],{"className":25849},[1182],[57,25851,25853],{"className":25852},[1092],[57,25854,25856],{"className":25855,"style":18131},[1096],[57,25857],{},[57,25859,1754],{"className":25860},[1079],[57,25862,25864,25867],{"className":25863},[1075],[57,25865,13535],{"className":25866},[1075,1148],[57,25868,25870],{"className":25869},[1152],[57,25871,25873,25893],{"className":25872},[1087,1088],[57,25874,25876,25890],{"className":25875},[1092],[57,25877,25879],{"className":25878,"style":13464},[1096],[57,25880,25881,25884],{"style":2211},[57,25882],{"className":25883,"style":1493},[1104],[57,25885,25887],{"className":25886},[1109,1110,1111,1112],[57,25888,1733],{"className":25889},[1075,1148,1112],[57,25891,1183],{"className":25892},[1182],[57,25894,25896],{"className":25895},[1092],[57,25897,25899],{"className":25898,"style":2230},[1096],[57,25900],{},[57,25902,1759],{"className":25903},[1849],[57,25905],{"className":25906,"style":1853},[1452],[57,25908,18],{"className":25909},[1075,1148],[57,25911,1768],{"className":25912},[1269],[57,25914],{"className":25915,"style":1453},[1452],[57,25917,281],{"className":25918},[1457],[57,25920],{"className":25921,"style":1453},[1452],[57,25923,25925,25928,25968,25971,25974],{"className":25924},[1066],[57,25926],{"className":25927,"style":22799},[1070],[57,25929,25931,25934],{"className":25930},[1075],[57,25932,8500],{"className":25933,"style":3932},[1075,1148],[57,25935,25937],{"className":25936},[1152],[57,25938,25940,25960],{"className":25939},[1087,1088],[57,25941,25943,25957],{"className":25942},[1092],[57,25944,25946],{"className":25945,"style":13464},[1096],[57,25947,25948,25951],{"style":13853},[57,25949],{"className":25950,"style":1493},[1104],[57,25952,25954],{"className":25953},[1109,1110,1111,1112],[57,25955,1733],{"className":25956},[1075,1148,1112],[57,25958,1183],{"className":25959},[1182],[57,25961,25963],{"className":25962},[1092],[57,25964,25966],{"className":25965,"style":2230},[1096],[57,25967],{},[57,25969],{"className":25970,"style":2595},[1452],[57,25972,6487],{"className":25973},[1506],[57,25975],{"className":25976,"style":2595},[1452],[57,25978,25980,25983,25986,25989,25992],{"className":25979},[1066],[57,25981],{"className":25982,"style":25358},[1070],[57,25984,19786],{"className":25985,"style":20033},[1075,1148],[57,25987],{"className":25988,"style":2595},[1452],[57,25990,24077],{"className":25991},[1506],[57,25993],{"className":25994,"style":2595},[1452],[57,25996,25998,26001,26050,26053,26056],{"className":25997},[1066],[57,25999],{"className":26000,"style":23445},[1070],[57,26002,26004,26007],{"className":26003},[1075],[57,26005,8500],{"className":26006,"style":3932},[1075,1148],[57,26008,26010],{"className":26009},[1152],[57,26011,26013,26042],{"className":26012},[1087,1088],[57,26014,26016,26039],{"className":26015},[1092],[57,26017,26019],{"className":26018,"style":3813},[1096],[57,26020,26021,26024],{"style":13853},[57,26022],{"className":26023,"style":1493},[1104],[57,26025,26027],{"className":26026},[1109,1110,1111,1112],[57,26028,26030,26033,26036],{"className":26029},[1075,1112],[57,26031,1733],{"className":26032},[1075,1148,1112],[57,26034,6487],{"className":26035},[1506,1112],[57,26037,574],{"className":26038},[1075,1112],[57,26040,1183],{"className":26041},[1182],[57,26043,26045],{"className":26044},[1092],[57,26046,26048],{"className":26047,"style":7608},[1096],[57,26049],{},[57,26051],{"className":26052,"style":2595},[1452],[57,26054,6487],{"className":26055},[1506],[57,26057],{"className":26058,"style":2595},[1452],[57,26060,26062,26065,26094,26097,26100],{"className":26061},[1066],[57,26063],{"className":26064,"style":25441},[1070],[57,26066,26068,26071],{"className":26067},[1075],[57,26069,19786],{"className":26070,"style":20033},[1075,1148],[57,26072,26074],{"className":26073},[1152],[57,26075,26077],{"className":26076},[1087],[57,26078,26080],{"className":26079},[1092],[57,26081,26083],{"className":26082,"style":24338},[1096],[57,26084,26085,26088],{"style":17216},[57,26086],{"className":26087,"style":1493},[1104],[57,26089,26091],{"className":26090},[1109,1110,1111,1112],[57,26092,590],{"className":26093},[1075,1112],[57,26095],{"className":26096,"style":2595},[1452],[57,26098,24077],{"className":26099},[1506],[57,26101],{"className":26102,"style":2595},[1452],[57,26104,26106,26109,26158,26161,26164],{"className":26105},[1066],[57,26107],{"className":26108,"style":23445},[1070],[57,26110,26112,26115],{"className":26111},[1075],[57,26113,8500],{"className":26114,"style":3932},[1075,1148],[57,26116,26118],{"className":26117},[1152],[57,26119,26121,26150],{"className":26120},[1087,1088],[57,26122,26124,26147],{"className":26123},[1092],[57,26125,26127],{"className":26126,"style":3813},[1096],[57,26128,26129,26132],{"style":13853},[57,26130],{"className":26131,"style":1493},[1104],[57,26133,26135],{"className":26134},[1109,1110,1111,1112],[57,26136,26138,26141,26144],{"className":26137},[1075,1112],[57,26139,1733],{"className":26140},[1075,1148,1112],[57,26142,6487],{"className":26143},[1506,1112],[57,26145,590],{"className":26146},[1075,1112],[57,26148,1183],{"className":26149},[1182],[57,26151,26153],{"className":26152},[1092],[57,26154,26156],{"className":26155,"style":7608},[1096],[57,26157],{},[57,26159],{"className":26160,"style":2595},[1452],[57,26162,6487],{"className":26163},[1506],[57,26165],{"className":26166,"style":2595},[1452],[57,26168,26170,26173,26202,26205,26208],{"className":26169},[1066],[57,26171],{"className":26172,"style":25441},[1070],[57,26174,26176,26179],{"className":26175},[1075],[57,26177,19786],{"className":26178,"style":20033},[1075,1148],[57,26180,26182],{"className":26181},[1152],[57,26183,26185],{"className":26184},[1087],[57,26186,26188],{"className":26187},[1092],[57,26189,26191],{"className":26190,"style":24338},[1096],[57,26192,26193,26196],{"style":17216},[57,26194],{"className":26195,"style":1493},[1104],[57,26197,26199],{"className":26198},[1109,1110,1111,1112],[57,26200,606],{"className":26201},[1075,1112],[57,26203],{"className":26204,"style":2595},[1452],[57,26206,24077],{"className":26207},[1506],[57,26209],{"className":26210,"style":2595},[1452],[57,26212,26214,26217,26266,26269,26272],{"className":26213},[1066],[57,26215],{"className":26216,"style":23445},[1070],[57,26218,26220,26223],{"className":26219},[1075],[57,26221,8500],{"className":26222,"style":3932},[1075,1148],[57,26224,26226],{"className":26225},[1152],[57,26227,26229,26258],{"className":26228},[1087,1088],[57,26230,26232,26255],{"className":26231},[1092],[57,26233,26235],{"className":26234,"style":3813},[1096],[57,26236,26237,26240],{"style":13853},[57,26238],{"className":26239,"style":1493},[1104],[57,26241,26243],{"className":26242},[1109,1110,1111,1112],[57,26244,26246,26249,26252],{"className":26245},[1075,1112],[57,26247,1733],{"className":26248},[1075,1148,1112],[57,26250,6487],{"className":26251},[1506,1112],[57,26253,606],{"className":26254},[1075,1112],[57,26256,1183],{"className":26257},[1182],[57,26259,26261],{"className":26260},[1092],[57,26262,26264],{"className":26263,"style":7608},[1096],[57,26265],{},[57,26267],{"className":26268,"style":2595},[1452],[57,26270,6487],{"className":26271},[1506],[57,26273],{"className":26274,"style":2595},[1452],[57,26276,26278,26281,26284,26287,26290],{"className":26277},[1066],[57,26279],{"className":26280,"style":8676},[1070],[57,26282,24272],{"className":26283},[1898],[57,26285],{"className":26286,"style":2595},[1452],[57,26288,6487],{"className":26289},[1506],[57,26291],{"className":26292,"style":2595},[1452],[57,26294,26296,26300,26329,26332,26335],{"className":26295},[1066],[57,26297],{"className":26298,"style":26299},[1070],"height:1.0858em;vertical-align:-0.1944em;",[57,26301,26303,26306],{"className":26302},[1075],[57,26304,19786],{"className":26305,"style":20033},[1075,1148],[57,26307,26309],{"className":26308},[1152],[57,26310,26312],{"className":26311},[1087],[57,26313,26315],{"className":26314},[1092],[57,26316,26318],{"className":26317,"style":25824},[1096],[57,26319,26320,26323],{"style":17216},[57,26321],{"className":26322,"style":1493},[1104],[57,26324,26326],{"className":26325},[1109,1110,1111,1112],[57,26327,738],{"className":26328,"style":3907},[1075,1148,1112],[57,26330],{"className":26331,"style":2595},[1452],[57,26333,24077],{"className":26334},[1506],[57,26336],{"className":26337,"style":2595},[1452],[57,26339,26341,26344,26384,26387,26390],{"className":26340},[1066],[57,26342],{"className":26343,"style":22799},[1070],[57,26345,26347,26350],{"className":26346},[1075],[57,26348,8500],{"className":26349,"style":3932},[1075,1148],[57,26351,26353],{"className":26352},[1152],[57,26354,26356,26376],{"className":26355},[1087,1088],[57,26357,26359,26373],{"className":26358},[1092],[57,26360,26362],{"className":26361,"style":14225},[1096],[57,26363,26364,26367],{"style":13853},[57,26365],{"className":26366,"style":1493},[1104],[57,26368,26370],{"className":26369},[1109,1110,1111,1112],[57,26371,738],{"className":26372,"style":3907},[1075,1148,1112],[57,26374,1183],{"className":26375},[1182],[57,26377,26379],{"className":26378},[1092],[57,26380,26382],{"className":26381,"style":2230},[1096],[57,26383],{},[57,26385],{"className":26386,"style":2595},[1452],[57,26388,19155],{"className":26389},[1506],[57,26391],{"className":26392,"style":2595},[1452],[57,26394,26396,26399,26439,26442,26482],{"className":26395},[1066],[57,26397],{"className":26398,"style":1831},[1070],[57,26400,26402,26405],{"className":26401},[1075],[57,26403,1625],{"className":26404,"style":1655},[1075,1148],[57,26406,26408],{"className":26407},[1152],[57,26409,26411,26431],{"className":26410},[1087,1088],[57,26412,26414,26428],{"className":26413},[1092],[57,26415,26417],{"className":26416,"style":2208},[1096],[57,26418,26419,26422],{"style":21382},[57,26420],{"className":26421,"style":1493},[1104],[57,26423,26425],{"className":26424},[1109,1110,1111,1112],[57,26426,14745],{"className":26427,"style":3932},[1075,1148,1112],[57,26429,1183],{"className":26430},[1182],[57,26432,26434],{"className":26433},[1092],[57,26435,26437],{"className":26436,"style":2230},[1096],[57,26438],{},[57,26440,1754],{"className":26441},[1079],[57,26443,26445,26448],{"className":26444},[1075],[57,26446,13535],{"className":26447},[1075,1148],[57,26449,26451],{"className":26450},[1152],[57,26452,26454,26474],{"className":26453},[1087,1088],[57,26455,26457,26471],{"className":26456},[1092],[57,26458,26460],{"className":26459,"style":13464},[1096],[57,26461,26462,26465],{"style":2211},[57,26463],{"className":26464,"style":1493},[1104],[57,26466,26468],{"className":26467},[1109,1110,1111,1112],[57,26469,1733],{"className":26470},[1075,1148,1112],[57,26472,1183],{"className":26473},[1182],[57,26475,26477],{"className":26476},[1092],[57,26478,26480],{"className":26479,"style":2230},[1096],[57,26481],{},[57,26483,1768],{"className":26484},[1269],[14,26486,26487,26488,26575,26576,727],{},"为了让公式更简洁，引入一个中间量 ",[57,26489,26491,26512],{"className":26490},[1019],[57,26492,26494],{"className":26493},[1023],[1025,26495,26496],{"xmlns":1027},[1029,26497,26498,26509],{},[1032,26499,26500],{},[2545,26501,26502,26505,26507],{},[1047,26503,26504],{},"δ",[1047,26506,1733],{},[1047,26508,1625],{},[1054,26510,26511],{"encoding":1056},"\\delta_t^V",[57,26513,26515],{"className":26514,"ariaHidden":1062},[1061],[57,26516,26518,26522],{"className":26517},[1066],[57,26519],{"className":26520,"style":26521},[1070],"height:1.0883em;vertical-align:-0.247em;",[57,26523,26525,26529],{"className":26524},[1075],[57,26526,26504],{"className":26527,"style":26528},[1075,1148],"margin-right:0.03785em;",[57,26530,26532],{"className":26531},[1152],[57,26533,26535,26567],{"className":26534},[1087,1088],[57,26536,26538,26564],{"className":26537},[1092],[57,26539,26541,26553],{"className":26540,"style":3920},[1096],[57,26542,26544,26547],{"style":26543},"top:-2.453em;margin-left:-0.0379em;margin-right:0.05em;",[57,26545],{"className":26546,"style":1493},[1104],[57,26548,26550],{"className":26549},[1109,1110,1111,1112],[57,26551,1733],{"className":26552},[1075,1148,1112],[57,26554,26555,26558],{"style":1489},[57,26556],{"className":26557,"style":1493},[1104],[57,26559,26561],{"className":26560},[1109,1110,1111,1112],[57,26562,1625],{"className":26563,"style":1655},[1075,1148,1112],[57,26565,1183],{"className":26566},[1182],[57,26568,26570],{"className":26569},[1092],[57,26571,26573],{"className":26572,"style":18131},[1096],[57,26574],{},"，表示 ",[11865,26577,26578,26579,26607],{},"第 ",[57,26580,26582,26595],{"className":26581},[1019],[57,26583,26585],{"className":26584},[1023],[1025,26586,26587],{"xmlns":1027},[1029,26588,26589,26593],{},[1032,26590,26591],{},[1047,26592,1733],{},[1054,26594,1733],{"encoding":1056},[57,26596,26598],{"className":26597,"ariaHidden":1062},[1061],[57,26599,26601,26604],{"className":26600},[1066],[57,26602],{"className":26603,"style":13512},[1070],[57,26605,1733],{"className":26606},[1075,1148]," 步采取该动作所带来的 TD 残差（temporal-difference error）",[57,26609,26611],{"className":26610},[15139],[57,26612,26614,26940],{"className":26613},[1019],[57,26615,26617],{"className":26616},[1023],[1025,26618,26619],{"xmlns":1027,"display":10210},[1029,26620,26621,26937],{},[1032,26622,26623,26631,26633,26639,26641,26643,26645,26651,26653,26665,26667,26669,26675,26677,26683,26685,26687,26701,26703,26715,26717,26719,26721,26727,26729,26741,26743,26745,26751,26753,26765,26767,26769,26777,26779,26785,26787,26789,26791,26793,26801,26803,26811,26813,26819,26821,26823,26825,26827,26835,26837,26839,26853,26855,26863,26865,26871,26873,26875,26877,26879,26887,26889,26891,26905,26907,26913,26927,26929],{},[2545,26624,26625,26627,26629],{},[1047,26626,26504],{},[1047,26628,1733],{},[1047,26630,1625],{},[1414,26632,281],{},[1044,26634,26635,26637],{},[1047,26636,8500],{},[1047,26638,1733],{},[1414,26640,6487],{},[1047,26642,19786],{},[1414,26644,24077],{},[1044,26646,26647,26649],{},[1047,26648,1625],{},[1047,26650,14745],{},[1414,26652,1754],{"stretchy":1753},[1044,26654,26655,26657],{},[1047,26656,13535],{},[1032,26658,26659,26661,26663],{},[1047,26660,1733],{},[1414,26662,6487],{},[1038,26664,574],{},[1414,26666,1768],{"stretchy":1753},[1414,26668,19155],{},[1044,26670,26671,26673],{},[1047,26672,1625],{},[1047,26674,14745],{},[1414,26676,1754],{"stretchy":1753},[1044,26678,26679,26681],{},[1047,26680,13535],{},[1047,26682,1733],{},[1414,26684,1768],{"stretchy":1753},[1452,26686],{"linebreak":22540},[2545,26688,26689,26691,26699],{},[1047,26690,26504],{},[1032,26692,26693,26695,26697],{},[1047,26694,1733],{},[1414,26696,6487],{},[1038,26698,574],{},[1047,26700,1625],{},[1414,26702,281],{},[1044,26704,26705,26707],{},[1047,26706,8500],{},[1032,26708,26709,26711,26713],{},[1047,26710,1733],{},[1414,26712,6487],{},[1038,26714,574],{},[1414,26716,6487],{},[1047,26718,19786],{},[1414,26720,24077],{},[1044,26722,26723,26725],{},[1047,26724,1625],{},[1047,26726,14745],{},[1414,26728,1754],{"stretchy":1753},[1044,26730,26731,26733],{},[1047,26732,13535],{},[1032,26734,26735,26737,26739],{},[1047,26736,1733],{},[1414,26738,6487],{},[1038,26740,590],{},[1414,26742,1768],{"stretchy":1753},[1414,26744,19155],{},[1044,26746,26747,26749],{},[1047,26748,1625],{},[1047,26750,14745],{},[1414,26752,1754],{"stretchy":1753},[1044,26754,26755,26757],{},[1047,26756,13535],{},[1032,26758,26759,26761,26763],{},[1047,26760,1733],{},[1414,26762,6487],{},[1038,26764,574],{},[1414,26766,1768],{"stretchy":1753},[1452,26768],{"linebreak":22540},[2545,26770,26771,26773,26775],{},[1047,26772,1730],{},[1047,26774,14745],{},[1038,26776,574],{},[1414,26778,1754],{"stretchy":1753},[1044,26780,26781,26783],{},[1047,26782,13535],{},[1047,26784,1733],{},[1414,26786,1759],{"separator":1062},[1047,26788,18],{},[1414,26790,1768],{"stretchy":1753},[1414,26792,281],{},[2545,26794,26795,26797,26799],{},[1047,26796,26504],{},[1047,26798,1733],{},[1047,26800,1625],{},[1452,26802],{"linebreak":22540},[2545,26804,26805,26807,26809],{},[1047,26806,1730],{},[1047,26808,14745],{},[1038,26810,590],{},[1414,26812,1754],{"stretchy":1753},[1044,26814,26815,26817],{},[1047,26816,13535],{},[1047,26818,1733],{},[1414,26820,1759],{"separator":1062},[1047,26822,18],{},[1414,26824,1768],{"stretchy":1753},[1414,26826,281],{},[2545,26828,26829,26831,26833],{},[1047,26830,26504],{},[1047,26832,1733],{},[1047,26834,1625],{},[1414,26836,6487],{},[1047,26838,19786],{},[2545,26840,26841,26843,26851],{},[1047,26842,26504],{},[1032,26844,26845,26847,26849],{},[1047,26846,1733],{},[1414,26848,6487],{},[1038,26850,574],{},[1047,26852,1625],{},[1452,26854],{"linebreak":22540},[2545,26856,26857,26859,26861],{},[1047,26858,1730],{},[1047,26860,14745],{},[1038,26862,606],{},[1414,26864,1754],{"stretchy":1753},[1044,26866,26867,26869],{},[1047,26868,13535],{},[1047,26870,1733],{},[1414,26872,1759],{"separator":1062},[1047,26874,18],{},[1414,26876,1768],{"stretchy":1753},[1414,26878,281],{},[2545,26880,26881,26883,26885],{},[1047,26882,26504],{},[1047,26884,1733],{},[1047,26886,1625],{},[1414,26888,6487],{},[1047,26890,19786],{},[2545,26892,26893,26895,26903],{},[1047,26894,26504],{},[1032,26896,26897,26899,26901],{},[1047,26898,1733],{},[1414,26900,6487],{},[1038,26902,574],{},[1047,26904,1625],{},[1414,26906,6487],{},[1418,26908,26909,26911],{},[1047,26910,19786],{},[1038,26912,590],{},[2545,26914,26915,26917,26925],{},[1047,26916,26504],{},[1032,26918,26919,26921,26923],{},[1047,26920,1733],{},[1414,26922,6487],{},[1038,26924,590],{},[1047,26926,1625],{},[1452,26928],{"linebreak":22540},[1032,26930,26931,26933],{},[1047,26932,24166],{"mathvariant":1729},[24168,26934,26935],{"height":15155,"voffset":15155},[1452,26936],{"mathbackground":24172,"width":15155,"height":24173},[1054,26938,26939],{"encoding":1056},"\n\\delta_t^V = r_t + \\gamma * V_\\theta(s_{t+1}) - V_\\theta(s_t)\\\\\n\n\\delta_{t+1}^V = r_{t+1} + \\gamma * V_\\theta(s_{t+2}) - V_\\theta(s_{t+1})\\\\\n\nA_\\theta^1(s_t, a) = \\delta_t^V\\\\\n\nA_\\theta^2(s_t, a) = \\delta_t^V + \\gamma \\delta_{t+1}^V\\\\\n\nA_\\theta^3(s_t, a) = \\delta_t^V + \\gamma \\delta_{t+1}^V + \\gamma^2 \\delta_{t+2}^V\\\\\n\n\\vdots",[57,26941,26943,27010,27065,27083,27193,27285,27288,27365,27429,27447,27557,27658,27661,27782,27839,27842,27963,28029,28098,28101,28222,28288,28366,28461,28464],{"className":26942,"ariaHidden":1062},[1061],[57,26944,26946,26950,27001,27004,27007],{"className":26945},[1066],[57,26947],{"className":26948,"style":26949},[1070],"height:1.1383em;vertical-align:-0.247em;",[57,26951,26953,26956],{"className":26952},[1075],[57,26954,26504],{"className":26955,"style":26528},[1075,1148],[57,26957,26959],{"className":26958},[1152],[57,26960,26962,26993],{"className":26961},[1087,1088],[57,26963,26965,26990],{"className":26964},[1092],[57,26966,26968,26979],{"className":26967,"style":25824},[1096],[57,26969,26970,26973],{"style":26543},[57,26971],{"className":26972,"style":1493},[1104],[57,26974,26976],{"className":26975},[1109,1110,1111,1112],[57,26977,1733],{"className":26978},[1075,1148,1112],[57,26980,26981,26984],{"style":17216},[57,26982],{"className":26983,"style":1493},[1104],[57,26985,26987],{"className":26986},[1109,1110,1111,1112],[57,26988,1625],{"className":26989,"style":1655},[1075,1148,1112],[57,26991,1183],{"className":26992},[1182],[57,26994,26996],{"className":26995},[1092],[57,26997,26999],{"className":26998,"style":18131},[1096],[57,27000],{},[57,27002],{"className":27003,"style":1453},[1452],[57,27005,281],{"className":27006},[1457],[57,27008],{"className":27009,"style":1453},[1452],[57,27011,27013,27016,27056,27059,27062],{"className":27012},[1066],[57,27014],{"className":27015,"style":22799},[1070],[57,27017,27019,27022],{"className":27018},[1075],[57,27020,8500],{"className":27021,"style":3932},[1075,1148],[57,27023,27025],{"className":27024},[1152],[57,27026,27028,27048],{"className":27027},[1087,1088],[57,27029,27031,27045],{"className":27030},[1092],[57,27032,27034],{"className":27033,"style":13464},[1096],[57,27035,27036,27039],{"style":13853},[57,27037],{"className":27038,"style":1493},[1104],[57,27040,27042],{"className":27041},[1109,1110,1111,1112],[57,27043,1733],{"className":27044},[1075,1148,1112],[57,27046,1183],{"className":27047},[1182],[57,27049,27051],{"className":27050},[1092],[57,27052,27054],{"className":27053,"style":2230},[1096],[57,27055],{},[57,27057],{"className":27058,"style":2595},[1452],[57,27060,6487],{"className":27061},[1506],[57,27063],{"className":27064,"style":2595},[1452],[57,27066,27068,27071,27074,27077,27080],{"className":27067},[1066],[57,27069],{"className":27070,"style":25358},[1070],[57,27072,19786],{"className":27073,"style":20033},[1075,1148],[57,27075],{"className":27076,"style":2595},[1452],[57,27078,24077],{"className":27079},[1506],[57,27081],{"className":27082,"style":2595},[1452],[57,27084,27086,27089,27129,27132,27181,27184,27187,27190],{"className":27085},[1066],[57,27087],{"className":27088,"style":1831},[1070],[57,27090,27092,27095],{"className":27091},[1075],[57,27093,1625],{"className":27094,"style":1655},[1075,1148],[57,27096,27098],{"className":27097},[1152],[57,27099,27101,27121],{"className":27100},[1087,1088],[57,27102,27104,27118],{"className":27103},[1092],[57,27105,27107],{"className":27106,"style":2208},[1096],[57,27108,27109,27112],{"style":21382},[57,27110],{"className":27111,"style":1493},[1104],[57,27113,27115],{"className":27114},[1109,1110,1111,1112],[57,27116,14745],{"className":27117,"style":3932},[1075,1148,1112],[57,27119,1183],{"className":27120},[1182],[57,27122,27124],{"className":27123},[1092],[57,27125,27127],{"className":27126,"style":2230},[1096],[57,27128],{},[57,27130,1754],{"className":27131},[1079],[57,27133,27135,27138],{"className":27134},[1075],[57,27136,13535],{"className":27137},[1075,1148],[57,27139,27141],{"className":27140},[1152],[57,27142,27144,27173],{"className":27143},[1087,1088],[57,27145,27147,27170],{"className":27146},[1092],[57,27148,27150],{"className":27149,"style":3813},[1096],[57,27151,27152,27155],{"style":2211},[57,27153],{"className":27154,"style":1493},[1104],[57,27156,27158],{"className":27157},[1109,1110,1111,1112],[57,27159,27161,27164,27167],{"className":27160},[1075,1112],[57,27162,1733],{"className":27163},[1075,1148,1112],[57,27165,6487],{"className":27166},[1506,1112],[57,27168,574],{"className":27169},[1075,1112],[57,27171,1183],{"className":27172},[1182],[57,27174,27176],{"className":27175},[1092],[57,27177,27179],{"className":27178,"style":7608},[1096],[57,27180],{},[57,27182,1768],{"className":27183},[1269],[57,27185],{"className":27186,"style":2595},[1452],[57,27188,19155],{"className":27189},[1506],[57,27191],{"className":27192,"style":2595},[1452],[57,27194,27196,27199,27239,27242,27282],{"className":27195},[1066],[57,27197],{"className":27198,"style":1831},[1070],[57,27200,27202,27205],{"className":27201},[1075],[57,27203,1625],{"className":27204,"style":1655},[1075,1148],[57,27206,27208],{"className":27207},[1152],[57,27209,27211,27231],{"className":27210},[1087,1088],[57,27212,27214,27228],{"className":27213},[1092],[57,27215,27217],{"className":27216,"style":2208},[1096],[57,27218,27219,27222],{"style":21382},[57,27220],{"className":27221,"style":1493},[1104],[57,27223,27225],{"className":27224},[1109,1110,1111,1112],[57,27226,14745],{"className":27227,"style":3932},[1075,1148,1112],[57,27229,1183],{"className":27230},[1182],[57,27232,27234],{"className":27233},[1092],[57,27235,27237],{"className":27236,"style":2230},[1096],[57,27238],{},[57,27240,1754],{"className":27241},[1079],[57,27243,27245,27248],{"className":27244},[1075],[57,27246,13535],{"className":27247},[1075,1148],[57,27249,27251],{"className":27250},[1152],[57,27252,27254,27274],{"className":27253},[1087,1088],[57,27255,27257,27271],{"className":27256},[1092],[57,27258,27260],{"className":27259,"style":13464},[1096],[57,27261,27262,27265],{"style":2211},[57,27263],{"className":27264,"style":1493},[1104],[57,27266,27268],{"className":27267},[1109,1110,1111,1112],[57,27269,1733],{"className":27270},[1075,1148,1112],[57,27272,1183],{"className":27273},[1182],[57,27275,27277],{"className":27276},[1092],[57,27278,27280],{"className":27279,"style":2230},[1096],[57,27281],{},[57,27283,1768],{"className":27284},[1269],[57,27286],{"className":27287},[1452,22540],[57,27289,27291,27295,27356,27359,27362],{"className":27290},[1066],[57,27292],{"className":27293,"style":27294},[1070],"height:1.1967em;vertical-align:-0.3053em;",[57,27296,27298,27301],{"className":27297},[1075],[57,27299,26504],{"className":27300,"style":26528},[1075,1148],[57,27302,27304],{"className":27303},[1152],[57,27305,27307,27347],{"className":27306},[1087,1088],[57,27308,27310,27344],{"className":27309},[1092],[57,27311,27313,27333],{"className":27312,"style":25824},[1096],[57,27314,27315,27318],{"style":26543},[57,27316],{"className":27317,"style":1493},[1104],[57,27319,27321],{"className":27320},[1109,1110,1111,1112],[57,27322,27324,27327,27330],{"className":27323},[1075,1112],[57,27325,1733],{"className":27326},[1075,1148,1112],[57,27328,6487],{"className":27329},[1506,1112],[57,27331,574],{"className":27332},[1075,1112],[57,27334,27335,27338],{"style":17216},[57,27336],{"className":27337,"style":1493},[1104],[57,27339,27341],{"className":27340},[1109,1110,1111,1112],[57,27342,1625],{"className":27343,"style":1655},[1075,1148,1112],[57,27345,1183],{"className":27346},[1182],[57,27348,27350],{"className":27349},[1092],[57,27351,27354],{"className":27352,"style":27353},[1096],"height:0.3053em;",[57,27355],{},[57,27357],{"className":27358,"style":1453},[1452],[57,27360,281],{"className":27361},[1457],[57,27363],{"className":27364,"style":1453},[1452],[57,27366,27368,27371,27420,27423,27426],{"className":27367},[1066],[57,27369],{"className":27370,"style":23445},[1070],[57,27372,27374,27377],{"className":27373},[1075],[57,27375,8500],{"className":27376,"style":3932},[1075,1148],[57,27378,27380],{"className":27379},[1152],[57,27381,27383,27412],{"className":27382},[1087,1088],[57,27384,27386,27409],{"className":27385},[1092],[57,27387,27389],{"className":27388,"style":3813},[1096],[57,27390,27391,27394],{"style":13853},[57,27392],{"className":27393,"style":1493},[1104],[57,27395,27397],{"className":27396},[1109,1110,1111,1112],[57,27398,27400,27403,27406],{"className":27399},[1075,1112],[57,27401,1733],{"className":27402},[1075,1148,1112],[57,27404,6487],{"className":27405},[1506,1112],[57,27407,574],{"className":27408},[1075,1112],[57,27410,1183],{"className":27411},[1182],[57,27413,27415],{"className":27414},[1092],[57,27416,27418],{"className":27417,"style":7608},[1096],[57,27419],{},[57,27421],{"className":27422,"style":2595},[1452],[57,27424,6487],{"className":27425},[1506],[57,27427],{"className":27428,"style":2595},[1452],[57,27430,27432,27435,27438,27441,27444],{"className":27431},[1066],[57,27433],{"className":27434,"style":25358},[1070],[57,27436,19786],{"className":27437,"style":20033},[1075,1148],[57,27439],{"className":27440,"style":2595},[1452],[57,27442,24077],{"className":27443},[1506],[57,27445],{"className":27446,"style":2595},[1452],[57,27448,27450,27453,27493,27496,27545,27548,27551,27554],{"className":27449},[1066],[57,27451],{"className":27452,"style":1831},[1070],[57,27454,27456,27459],{"className":27455},[1075],[57,27457,1625],{"className":27458,"style":1655},[1075,1148],[57,27460,27462],{"className":27461},[1152],[57,27463,27465,27485],{"className":27464},[1087,1088],[57,27466,27468,27482],{"className":27467},[1092],[57,27469,27471],{"className":27470,"style":2208},[1096],[57,27472,27473,27476],{"style":21382},[57,27474],{"className":27475,"style":1493},[1104],[57,27477,27479],{"className":27478},[1109,1110,1111,1112],[57,27480,14745],{"className":27481,"style":3932},[1075,1148,1112],[57,27483,1183],{"className":27484},[1182],[57,27486,27488],{"className":27487},[1092],[57,27489,27491],{"className":27490,"style":2230},[1096],[57,27492],{},[57,27494,1754],{"className":27495},[1079],[57,27497,27499,27502],{"className":27498},[1075],[57,27500,13535],{"className":27501},[1075,1148],[57,27503,27505],{"className":27504},[1152],[57,27506,27508,27537],{"className":27507},[1087,1088],[57,27509,27511,27534],{"className":27510},[1092],[57,27512,27514],{"className":27513,"style":3813},[1096],[57,27515,27516,27519],{"style":2211},[57,27517],{"className":27518,"style":1493},[1104],[57,27520,27522],{"className":27521},[1109,1110,1111,1112],[57,27523,27525,27528,27531],{"className":27524},[1075,1112],[57,27526,1733],{"className":27527},[1075,1148,1112],[57,27529,6487],{"className":27530},[1506,1112],[57,27532,590],{"className":27533},[1075,1112],[57,27535,1183],{"className":27536},[1182],[57,27538,27540],{"className":27539},[1092],[57,27541,27543],{"className":27542,"style":7608},[1096],[57,27544],{},[57,27546,1768],{"className":27547},[1269],[57,27549],{"className":27550,"style":2595},[1452],[57,27552,19155],{"className":27553},[1506],[57,27555],{"className":27556,"style":2595},[1452],[57,27558,27560,27563,27603,27606,27655],{"className":27559},[1066],[57,27561],{"className":27562,"style":1831},[1070],[57,27564,27566,27569],{"className":27565},[1075],[57,27567,1625],{"className":27568,"style":1655},[1075,1148],[57,27570,27572],{"className":27571},[1152],[57,27573,27575,27595],{"className":27574},[1087,1088],[57,27576,27578,27592],{"className":27577},[1092],[57,27579,27581],{"className":27580,"style":2208},[1096],[57,27582,27583,27586],{"style":21382},[57,27584],{"className":27585,"style":1493},[1104],[57,27587,27589],{"className":27588},[1109,1110,1111,1112],[57,27590,14745],{"className":27591,"style":3932},[1075,1148,1112],[57,27593,1183],{"className":27594},[1182],[57,27596,27598],{"className":27597},[1092],[57,27599,27601],{"className":27600,"style":2230},[1096],[57,27602],{},[57,27604,1754],{"className":27605},[1079],[57,27607,27609,27612],{"className":27608},[1075],[57,27610,13535],{"className":27611},[1075,1148],[57,27613,27615],{"className":27614},[1152],[57,27616,27618,27647],{"className":27617},[1087,1088],[57,27619,27621,27644],{"className":27620},[1092],[57,27622,27624],{"className":27623,"style":3813},[1096],[57,27625,27626,27629],{"style":2211},[57,27627],{"className":27628,"style":1493},[1104],[57,27630,27632],{"className":27631},[1109,1110,1111,1112],[57,27633,27635,27638,27641],{"className":27634},[1075,1112],[57,27636,1733],{"className":27637},[1075,1148,1112],[57,27639,6487],{"className":27640},[1506,1112],[57,27642,574],{"className":27643},[1075,1112],[57,27645,1183],{"className":27646},[1182],[57,27648,27650],{"className":27649},[1092],[57,27651,27653],{"className":27652,"style":7608},[1096],[57,27654],{},[57,27656,1768],{"className":27657},[1269],[57,27659],{"className":27660},[1452,22540],[57,27662,27664,27667,27718,27721,27761,27764,27767,27770,27773,27776,27779],{"className":27663},[1066],[57,27665],{"className":27666,"style":24319},[1070],[57,27668,27670,27673],{"className":27669},[1075],[57,27671,1730],{"className":27672},[1075,1148],[57,27674,27676],{"className":27675},[1152],[57,27677,27679,27710],{"className":27678},[1087,1088],[57,27680,27682,27707],{"className":27681},[1092],[57,27683,27685,27696],{"className":27684,"style":24338},[1096],[57,27686,27687,27690],{"style":18101},[57,27688],{"className":27689,"style":1493},[1104],[57,27691,27693],{"className":27692},[1109,1110,1111,1112],[57,27694,14745],{"className":27695,"style":3932},[1075,1148,1112],[57,27697,27698,27701],{"style":17216},[57,27699],{"className":27700,"style":1493},[1104],[57,27702,27704],{"className":27703},[1109,1110,1111,1112],[57,27705,574],{"className":27706},[1075,1112],[57,27708,1183],{"className":27709},[1182],[57,27711,27713],{"className":27712},[1092],[57,27714,27716],{"className":27715,"style":18131},[1096],[57,27717],{},[57,27719,1754],{"className":27720},[1079],[57,27722,27724,27727],{"className":27723},[1075],[57,27725,13535],{"className":27726},[1075,1148],[57,27728,27730],{"className":27729},[1152],[57,27731,27733,27753],{"className":27732},[1087,1088],[57,27734,27736,27750],{"className":27735},[1092],[57,27737,27739],{"className":27738,"style":13464},[1096],[57,27740,27741,27744],{"style":2211},[57,27742],{"className":27743,"style":1493},[1104],[57,27745,27747],{"className":27746},[1109,1110,1111,1112],[57,27748,1733],{"className":27749},[1075,1148,1112],[57,27751,1183],{"className":27752},[1182],[57,27754,27756],{"className":27755},[1092],[57,27757,27759],{"className":27758,"style":2230},[1096],[57,27760],{},[57,27762,1759],{"className":27763},[1849],[57,27765],{"className":27766,"style":1853},[1452],[57,27768,18],{"className":27769},[1075,1148],[57,27771,1768],{"className":27772},[1269],[57,27774],{"className":27775,"style":1453},[1452],[57,27777,281],{"className":27778},[1457],[57,27780],{"className":27781,"style":1453},[1452],[57,27783,27785,27788],{"className":27784},[1066],[57,27786],{"className":27787,"style":26949},[1070],[57,27789,27791,27794],{"className":27790},[1075],[57,27792,26504],{"className":27793,"style":26528},[1075,1148],[57,27795,27797],{"className":27796},[1152],[57,27798,27800,27831],{"className":27799},[1087,1088],[57,27801,27803,27828],{"className":27802},[1092],[57,27804,27806,27817],{"className":27805,"style":25824},[1096],[57,27807,27808,27811],{"style":26543},[57,27809],{"className":27810,"style":1493},[1104],[57,27812,27814],{"className":27813},[1109,1110,1111,1112],[57,27815,1733],{"className":27816},[1075,1148,1112],[57,27818,27819,27822],{"style":17216},[57,27820],{"className":27821,"style":1493},[1104],[57,27823,27825],{"className":27824},[1109,1110,1111,1112],[57,27826,1625],{"className":27827,"style":1655},[1075,1148,1112],[57,27829,1183],{"className":27830},[1182],[57,27832,27834],{"className":27833},[1092],[57,27835,27837],{"className":27836,"style":18131},[1096],[57,27838],{},[57,27840],{"className":27841},[1452,22540],[57,27843,27845,27848,27899,27902,27942,27945,27948,27951,27954,27957,27960],{"className":27844},[1066],[57,27846],{"className":27847,"style":24319},[1070],[57,27849,27851,27854],{"className":27850},[1075],[57,27852,1730],{"className":27853},[1075,1148],[57,27855,27857],{"className":27856},[1152],[57,27858,27860,27891],{"className":27859},[1087,1088],[57,27861,27863,27888],{"className":27862},[1092],[57,27864,27866,27877],{"className":27865,"style":24338},[1096],[57,27867,27868,27871],{"style":18101},[57,27869],{"className":27870,"style":1493},[1104],[57,27872,27874],{"className":27873},[1109,1110,1111,1112],[57,27875,14745],{"className":27876,"style":3932},[1075,1148,1112],[57,27878,27879,27882],{"style":17216},[57,27880],{"className":27881,"style":1493},[1104],[57,27883,27885],{"className":27884},[1109,1110,1111,1112],[57,27886,590],{"className":27887},[1075,1112],[57,27889,1183],{"className":27890},[1182],[57,27892,27894],{"className":27893},[1092],[57,27895,27897],{"className":27896,"style":18131},[1096],[57,27898],{},[57,27900,1754],{"className":27901},[1079],[57,27903,27905,27908],{"className":27904},[1075],[57,27906,13535],{"className":27907},[1075,1148],[57,27909,27911],{"className":27910},[1152],[57,27912,27914,27934],{"className":27913},[1087,1088],[57,27915,27917,27931],{"className":27916},[1092],[57,27918,27920],{"className":27919,"style":13464},[1096],[57,27921,27922,27925],{"style":2211},[57,27923],{"className":27924,"style":1493},[1104],[57,27926,27928],{"className":27927},[1109,1110,1111,1112],[57,27929,1733],{"className":27930},[1075,1148,1112],[57,27932,1183],{"className":27933},[1182],[57,27935,27937],{"className":27936},[1092],[57,27938,27940],{"className":27939,"style":2230},[1096],[57,27941],{},[57,27943,1759],{"className":27944},[1849],[57,27946],{"className":27947,"style":1853},[1452],[57,27949,18],{"className":27950},[1075,1148],[57,27952,1768],{"className":27953},[1269],[57,27955],{"className":27956,"style":1453},[1452],[57,27958,281],{"className":27959},[1457],[57,27961],{"className":27962,"style":1453},[1452],[57,27964,27966,27969,28020,28023,28026],{"className":27965},[1066],[57,27967],{"className":27968,"style":26949},[1070],[57,27970,27972,27975],{"className":27971},[1075],[57,27973,26504],{"className":27974,"style":26528},[1075,1148],[57,27976,27978],{"className":27977},[1152],[57,27979,27981,28012],{"className":27980},[1087,1088],[57,27982,27984,28009],{"className":27983},[1092],[57,27985,27987,27998],{"className":27986,"style":25824},[1096],[57,27988,27989,27992],{"style":26543},[57,27990],{"className":27991,"style":1493},[1104],[57,27993,27995],{"className":27994},[1109,1110,1111,1112],[57,27996,1733],{"className":27997},[1075,1148,1112],[57,27999,28000,28003],{"style":17216},[57,28001],{"className":28002,"style":1493},[1104],[57,28004,28006],{"className":28005},[1109,1110,1111,1112],[57,28007,1625],{"className":28008,"style":1655},[1075,1148,1112],[57,28010,1183],{"className":28011},[1182],[57,28013,28015],{"className":28014},[1092],[57,28016,28018],{"className":28017,"style":18131},[1096],[57,28019],{},[57,28021],{"className":28022,"style":2595},[1452],[57,28024,6487],{"className":28025},[1506],[57,28027],{"className":28028,"style":2595},[1452],[57,28030,28032,28035,28038],{"className":28031},[1066],[57,28033],{"className":28034,"style":27294},[1070],[57,28036,19786],{"className":28037,"style":20033},[1075,1148],[57,28039,28041,28044],{"className":28040},[1075],[57,28042,26504],{"className":28043,"style":26528},[1075,1148],[57,28045,28047],{"className":28046},[1152],[57,28048,28050,28090],{"className":28049},[1087,1088],[57,28051,28053,28087],{"className":28052},[1092],[57,28054,28056,28076],{"className":28055,"style":25824},[1096],[57,28057,28058,28061],{"style":26543},[57,28059],{"className":28060,"style":1493},[1104],[57,28062,28064],{"className":28063},[1109,1110,1111,1112],[57,28065,28067,28070,28073],{"className":28066},[1075,1112],[57,28068,1733],{"className":28069},[1075,1148,1112],[57,28071,6487],{"className":28072},[1506,1112],[57,28074,574],{"className":28075},[1075,1112],[57,28077,28078,28081],{"style":17216},[57,28079],{"className":28080,"style":1493},[1104],[57,28082,28084],{"className":28083},[1109,1110,1111,1112],[57,28085,1625],{"className":28086,"style":1655},[1075,1148,1112],[57,28088,1183],{"className":28089},[1182],[57,28091,28093],{"className":28092},[1092],[57,28094,28096],{"className":28095,"style":27353},[1096],[57,28097],{},[57,28099],{"className":28100},[1452,22540],[57,28102,28104,28107,28158,28161,28201,28204,28207,28210,28213,28216,28219],{"className":28103},[1066],[57,28105],{"className":28106,"style":24319},[1070],[57,28108,28110,28113],{"className":28109},[1075],[57,28111,1730],{"className":28112},[1075,1148],[57,28114,28116],{"className":28115},[1152],[57,28117,28119,28150],{"className":28118},[1087,1088],[57,28120,28122,28147],{"className":28121},[1092],[57,28123,28125,28136],{"className":28124,"style":24338},[1096],[57,28126,28127,28130],{"style":18101},[57,28128],{"className":28129,"style":1493},[1104],[57,28131,28133],{"className":28132},[1109,1110,1111,1112],[57,28134,14745],{"className":28135,"style":3932},[1075,1148,1112],[57,28137,28138,28141],{"style":17216},[57,28139],{"className":28140,"style":1493},[1104],[57,28142,28144],{"className":28143},[1109,1110,1111,1112],[57,28145,606],{"className":28146},[1075,1112],[57,28148,1183],{"className":28149},[1182],[57,28151,28153],{"className":28152},[1092],[57,28154,28156],{"className":28155,"style":18131},[1096],[57,28157],{},[57,28159,1754],{"className":28160},[1079],[57,28162,28164,28167],{"className":28163},[1075],[57,28165,13535],{"className":28166},[1075,1148],[57,28168,28170],{"className":28169},[1152],[57,28171,28173,28193],{"className":28172},[1087,1088],[57,28174,28176,28190],{"className":28175},[1092],[57,28177,28179],{"className":28178,"style":13464},[1096],[57,28180,28181,28184],{"style":2211},[57,28182],{"className":28183,"style":1493},[1104],[57,28185,28187],{"className":28186},[1109,1110,1111,1112],[57,28188,1733],{"className":28189},[1075,1148,1112],[57,28191,1183],{"className":28192},[1182],[57,28194,28196],{"className":28195},[1092],[57,28197,28199],{"className":28198,"style":2230},[1096],[57,28200],{},[57,28202,1759],{"className":28203},[1849],[57,28205],{"className":28206,"style":1853},[1452],[57,28208,18],{"className":28209},[1075,1148],[57,28211,1768],{"className":28212},[1269],[57,28214],{"className":28215,"style":1453},[1452],[57,28217,281],{"className":28218},[1457],[57,28220],{"className":28221,"style":1453},[1452],[57,28223,28225,28228,28279,28282,28285],{"className":28224},[1066],[57,28226],{"className":28227,"style":26949},[1070],[57,28229,28231,28234],{"className":28230},[1075],[57,28232,26504],{"className":28233,"style":26528},[1075,1148],[57,28235,28237],{"className":28236},[1152],[57,28238,28240,28271],{"className":28239},[1087,1088],[57,28241,28243,28268],{"className":28242},[1092],[57,28244,28246,28257],{"className":28245,"style":25824},[1096],[57,28247,28248,28251],{"style":26543},[57,28249],{"className":28250,"style":1493},[1104],[57,28252,28254],{"className":28253},[1109,1110,1111,1112],[57,28255,1733],{"className":28256},[1075,1148,1112],[57,28258,28259,28262],{"style":17216},[57,28260],{"className":28261,"style":1493},[1104],[57,28263,28265],{"className":28264},[1109,1110,1111,1112],[57,28266,1625],{"className":28267,"style":1655},[1075,1148,1112],[57,28269,1183],{"className":28270},[1182],[57,28272,28274],{"className":28273},[1092],[57,28275,28277],{"className":28276,"style":18131},[1096],[57,28278],{},[57,28280],{"className":28281,"style":2595},[1452],[57,28283,6487],{"className":28284},[1506],[57,28286],{"className":28287,"style":2595},[1452],[57,28289,28291,28294,28297,28357,28360,28363],{"className":28290},[1066],[57,28292],{"className":28293,"style":27294},[1070],[57,28295,19786],{"className":28296,"style":20033},[1075,1148],[57,28298,28300,28303],{"className":28299},[1075],[57,28301,26504],{"className":28302,"style":26528},[1075,1148],[57,28304,28306],{"className":28305},[1152],[57,28307,28309,28349],{"className":28308},[1087,1088],[57,28310,28312,28346],{"className":28311},[1092],[57,28313,28315,28335],{"className":28314,"style":25824},[1096],[57,28316,28317,28320],{"style":26543},[57,28318],{"className":28319,"style":1493},[1104],[57,28321,28323],{"className":28322},[1109,1110,1111,1112],[57,28324,28326,28329,28332],{"className":28325},[1075,1112],[57,28327,1733],{"className":28328},[1075,1148,1112],[57,28330,6487],{"className":28331},[1506,1112],[57,28333,574],{"className":28334},[1075,1112],[57,28336,28337,28340],{"style":17216},[57,28338],{"className":28339,"style":1493},[1104],[57,28341,28343],{"className":28342},[1109,1110,1111,1112],[57,28344,1625],{"className":28345,"style":1655},[1075,1148,1112],[57,28347,1183],{"className":28348},[1182],[57,28350,28352],{"className":28351},[1092],[57,28353,28355],{"className":28354,"style":27353},[1096],[57,28356],{},[57,28358],{"className":28359,"style":2595},[1452],[57,28361,6487],{"className":28362},[1506],[57,28364],{"className":28365,"style":2595},[1452],[57,28367,28369,28372,28401],{"className":28368},[1066],[57,28370],{"className":28371,"style":27294},[1070],[57,28373,28375,28378],{"className":28374},[1075],[57,28376,19786],{"className":28377,"style":20033},[1075,1148],[57,28379,28381],{"className":28380},[1152],[57,28382,28384],{"className":28383},[1087],[57,28385,28387],{"className":28386},[1092],[57,28388,28390],{"className":28389,"style":24338},[1096],[57,28391,28392,28395],{"style":17216},[57,28393],{"className":28394,"style":1493},[1104],[57,28396,28398],{"className":28397},[1109,1110,1111,1112],[57,28399,590],{"className":28400},[1075,1112],[57,28402,28404,28407],{"className":28403},[1075],[57,28405,26504],{"className":28406,"style":26528},[1075,1148],[57,28408,28410],{"className":28409},[1152],[57,28411,28413,28453],{"className":28412},[1087,1088],[57,28414,28416,28450],{"className":28415},[1092],[57,28417,28419,28439],{"className":28418,"style":25824},[1096],[57,28420,28421,28424],{"style":26543},[57,28422],{"className":28423,"style":1493},[1104],[57,28425,28427],{"className":28426},[1109,1110,1111,1112],[57,28428,28430,28433,28436],{"className":28429},[1075,1112],[57,28431,1733],{"className":28432},[1075,1148,1112],[57,28434,6487],{"className":28435},[1506,1112],[57,28437,590],{"className":28438},[1075,1112],[57,28440,28441,28444],{"style":17216},[57,28442],{"className":28443,"style":1493},[1104],[57,28445,28447],{"className":28446},[1109,1110,1111,1112],[57,28448,1625],{"className":28449,"style":1655},[1075,1148,1112],[57,28451,1183],{"className":28452},[1182],[57,28454,28456],{"className":28455},[1092],[57,28457,28459],{"className":28458,"style":27353},[1096],[57,28460],{},[57,28462],{"className":28463},[1452,22540],[57,28465,28467,28470],{"className":28466},[1066],[57,28468],{"className":28469,"style":25784},[1070],[57,28471,28473,28476],{"className":28472},[1075],[57,28474,24166],{"className":28475},[1075],[57,28477],{"className":28478,"style":25795},[1075,25794],[248,28480,28482],{"id":28481},"generalized-advantage-estimation-gae","Generalized Advantage Estimation (GAE)",[14,28484,28485],{},"那么\"展开几步\"这件事到底该怎么选？GAE 给出的答案是：小孩子才做选择，全都要！",[14,28487,28488,28489,28548,28549,28579],{},"GAE 把不同展开步数的优势估计 ",[57,28490,28492,28510],{"className":28491},[1019],[57,28493,28495],{"className":28494},[1023],[1025,28496,28497],{"xmlns":1027},[1029,28498,28499,28507],{},[1032,28500,28501],{},[1418,28502,28503,28505],{},[1047,28504,1730],{},[1047,28506,1052],{},[1054,28508,28509],{"encoding":1056},"A^k",[57,28511,28513],{"className":28512,"ariaHidden":1062},[1061],[57,28514,28516,28519],{"className":28515},[1066],[57,28517],{"className":28518,"style":1467},[1070],[57,28520,28522,28525],{"className":28521},[1075],[57,28523,1730],{"className":28524},[1075,1148],[57,28526,28528],{"className":28527},[1152],[57,28529,28531],{"className":28530},[1087],[57,28532,28534],{"className":28533},[1092],[57,28535,28537],{"className":28536,"style":1467},[1096],[57,28538,28539,28542],{"style":1489},[57,28540],{"className":28541,"style":1493},[1104],[57,28543,28545],{"className":28544},[1109,1110,1111,1112],[57,28546,1052],{"className":28547,"style":1178},[1075,1148,1112]," 用一个权重 ",[57,28550,28552,28567],{"className":28551},[1019],[57,28553,28555],{"className":28554},[1023],[1025,28556,28557],{"xmlns":1027},[1029,28558,28559,28564],{},[1032,28560,28561],{},[1047,28562,28563],{},"λ",[1054,28565,28566],{"encoding":1056},"\\lambda",[57,28568,28570],{"className":28569,"ariaHidden":1062},[1061],[57,28571,28573,28576],{"className":28572},[1066],[57,28574],{"className":28575,"style":1360},[1070],[57,28577,28563],{"className":28578},[1075,1148]," 做指数加权平均，再化简成一个简洁的求和形式：",[57,28581,28583],{"className":28582},[15139],[57,28584,28586,29176],{"className":28585},[1019],[57,28587,28589],{"className":28588},[1023],[1025,28590,28591],{"xmlns":1027,"display":10210},[1029,28592,28593,29173],{},[15151,28594,28595,28772,28914,29024,29114],{"rowspacing":15153,"columnalign":15154,"columnspacing":15155},[15157,28596,28597,28632],{},[15160,28598,28599],{},[15163,28600,28601],{"scriptlevel":560,"displaystyle":1062},[1032,28602,28603,28618,28620,28626,28628,28630],{},[2545,28604,28605,28607,28609],{},[1047,28606,1730],{},[1047,28608,14745],{},[1032,28610,28611,28614,28616],{},[1047,28612,28613],{},"G",[1047,28615,1730],{},[1047,28617,6336],{},[1414,28619,1754],{"stretchy":1753},[1044,28621,28622,28624],{},[1047,28623,13535],{},[1047,28625,1733],{},[1414,28627,1759],{"separator":1062},[1047,28629,18],{},[1414,28631,1768],{"stretchy":1753},[15160,28633,28634],{},[15163,28635,28636],{"scriptlevel":560,"displaystyle":1062},[1032,28637,28638,28640,28642,28644,28646,28648,28650,28652,28654,28662,28664,28666,28668,28676,28678,28684,28692,28694,28696,28698,28700,28703,28705,28707,28709,28712,28714,28716,28730,28732,28734,28742,28744,28747,28755,28757,28760,28768,28770],{},[1032,28639],{},[1414,28641,281],{},[1414,28643,1754],{"stretchy":1753},[1038,28645,574],{},[1414,28647,19155],{},[1047,28649,28563],{},[1414,28651,1768],{"stretchy":1753},[1414,28653,1754],{"stretchy":1753},[2545,28655,28656,28658,28660],{},[1047,28657,1730],{},[1047,28659,14745],{},[1038,28661,574],{},[1414,28663,6487],{},[1047,28665,28563],{},[1414,28667,24077],{},[2545,28669,28670,28672,28674],{},[1047,28671,1730],{},[1047,28673,14745],{},[1038,28675,590],{},[1414,28677,6487],{},[1418,28679,28680,28682],{},[1047,28681,28563],{},[1038,28683,590],{},[2545,28685,28686,28688,28690],{},[1047,28687,1730],{},[1047,28689,14745],{},[1038,28691,606],{},[1414,28693,6487],{},[1414,28695,24272],{},[3999,28697,8013],{},[1414,28699,1768],{"stretchy":1753},[1452,28701],{"width":28702},"1em",[1452,28704],{"width":28702},[1047,28706,28563],{},[1414,28708,281],{},[1038,28710,28711],{},"0.9",[1414,28713,7851],{},[1452,28715],{"width":28702},[2545,28717,28718,28720,28722],{},[1047,28719,1730],{},[1047,28721,14745],{},[1032,28723,28724,28726,28728],{},[1047,28725,28613],{},[1047,28727,1730],{},[1047,28729,6336],{},[1414,28731,281],{},[1038,28733,13195],{},[2545,28735,28736,28738,28740],{},[1047,28737,1730],{},[1047,28739,14745],{},[1038,28741,574],{},[1414,28743,6487],{},[1038,28745,28746],{},"0.09",[2545,28748,28749,28751,28753],{},[1047,28750,1730],{},[1047,28752,14745],{},[1038,28754,590],{},[1414,28756,6487],{},[1038,28758,28759],{},"0.081",[2545,28761,28762,28764,28766],{},[1047,28763,1730],{},[1047,28765,14745],{},[1038,28767,606],{},[1414,28769,6487],{},[1414,28771,24272],{},[15157,28773,28774,28780],{},[15160,28775,28776],{},[15163,28777,28778],{"scriptlevel":560,"displaystyle":1062},[1032,28779],{},[15160,28781,28782],{},[15163,28783,28784],{"scriptlevel":560,"displaystyle":1062},[1032,28785,28786,28788,28790,28792,28794,28796,28798,28800,28802,28810,28812,28814,28816,28818,28826,28828,28830,28844,28846,28848,28854,28856,28864,28866,28868,28882,28884,28890,28904,28906,28908,28910,28912],{},[1032,28787],{},[1414,28789,281],{},[1414,28791,1754],{"stretchy":1753},[1038,28793,574],{},[1414,28795,19155],{},[1047,28797,28563],{},[1414,28799,1768],{"stretchy":1753},[1414,28801,1754],{"stretchy":1753},[2545,28803,28804,28806,28808],{},[1047,28805,26504],{},[1047,28807,1733],{},[1047,28809,1625],{},[1414,28811,6487],{},[1047,28813,28563],{},[1414,28815,24077],{},[1414,28817,1754],{"stretchy":1753},[2545,28819,28820,28822,28824],{},[1047,28821,26504],{},[1047,28823,1733],{},[1047,28825,1625],{},[1414,28827,6487],{},[1047,28829,19786],{},[2545,28831,28832,28834,28842],{},[1047,28833,26504],{},[1032,28835,28836,28838,28840],{},[1047,28837,1733],{},[1414,28839,6487],{},[1038,28841,574],{},[1047,28843,1625],{},[1414,28845,1768],{"stretchy":1753},[1414,28847,6487],{},[1418,28849,28850,28852],{},[1047,28851,28563],{},[1038,28853,590],{},[1414,28855,1754],{"stretchy":1753},[2545,28857,28858,28860,28862],{},[1047,28859,26504],{},[1047,28861,1733],{},[1047,28863,1625],{},[1414,28865,6487],{},[1047,28867,19786],{},[2545,28869,28870,28872,28880],{},[1047,28871,26504],{},[1032,28873,28874,28876,28878],{},[1047,28875,1733],{},[1414,28877,6487],{},[1038,28879,574],{},[1047,28881,1625],{},[1414,28883,6487],{},[1418,28885,28886,28888],{},[1047,28887,19786],{},[1038,28889,590],{},[2545,28891,28892,28894,28902],{},[1047,28893,26504],{},[1032,28895,28896,28898,28900],{},[1047,28897,1733],{},[1414,28899,6487],{},[1038,28901,590],{},[1047,28903,1625],{},[1414,28905,1768],{"stretchy":1753},[1414,28907,6487],{},[1414,28909,24272],{},[3999,28911,8013],{},[1414,28913,1768],{"stretchy":1753},[15157,28915,28916,28922],{},[15160,28917,28918],{},[15163,28919,28920],{"scriptlevel":560,"displaystyle":1062},[1032,28921],{},[15160,28923,28924],{},[15163,28925,28926],{"scriptlevel":560,"displaystyle":1062},[1032,28927,28928,28930,28932,28934,28936,28938,28940,28942,28944,28952,28954,28956,28958,28960,28962,28968,28970,28972,28974,28976,28978,28980,28994,28996,28998,29000,29002,29008,29010,29012,29014,29016,29018,29020,29022],{},[1032,28929],{},[1414,28931,281],{},[1414,28933,1754],{"stretchy":1753},[1038,28935,574],{},[1414,28937,19155],{},[1047,28939,28563],{},[1414,28941,1768],{"stretchy":1753},[1414,28943,1754],{"stretchy":1753},[2545,28945,28946,28948,28950],{},[1047,28947,26504],{},[1047,28949,1733],{},[1047,28951,1625],{},[1414,28953,1754],{"stretchy":1753},[1038,28955,574],{},[1414,28957,6487],{},[1047,28959,28563],{},[1414,28961,6487],{},[1418,28963,28964,28966],{},[1047,28965,28563],{},[1038,28967,590],{},[1414,28969,6487],{},[1414,28971,24272],{},[3999,28973,8013],{},[1414,28975,1768],{"stretchy":1753},[1414,28977,6487],{},[1047,28979,19786],{},[2545,28981,28982,28984,28992],{},[1047,28983,26504],{},[1032,28985,28986,28988,28990],{},[1047,28987,1733],{},[1414,28989,6487],{},[1038,28991,574],{},[1047,28993,1625],{},[1414,28995,24077],{},[1414,28997,1754],{"stretchy":1753},[1047,28999,28563],{},[1414,29001,6487],{},[1418,29003,29004,29006],{},[1047,29005,28563],{},[1038,29007,590],{},[1414,29009,6487],{},[1414,29011,24272],{},[3999,29013,8013],{},[1414,29015,1768],{"stretchy":1753},[1414,29017,6487],{},[1414,29019,24272],{},[3999,29021,8013],{},[1414,29023,1768],{"stretchy":1753},[15157,29025,29026,29032],{},[15160,29027,29028],{},[15163,29029,29030],{"scriptlevel":560,"displaystyle":1062},[1032,29031],{},[15160,29033,29034],{},[15163,29035,29036],{"scriptlevel":560,"displaystyle":1062},[1032,29037,29038,29040,29042,29044,29046,29048,29050,29052],{},[1032,29039],{},[1414,29041,281],{},[1414,29043,1754],{"stretchy":1753},[1038,29045,574],{},[1414,29047,19155],{},[1047,29049,28563],{},[1414,29051,1768],{"stretchy":1753},[1032,29053,29054,29056,29064,29076,29078,29080,29094,29106,29108,29110,29112],{},[1414,29055,1754],{"fence":1062},[2545,29057,29058,29060,29062],{},[1047,29059,26504],{},[1047,29061,1733],{},[1047,29063,1625],{},[1035,29065,29066,29068],{},[1038,29067,574],{},[1032,29069,29070,29072,29074],{},[1038,29071,574],{},[1414,29073,19155],{},[1047,29075,28563],{},[1414,29077,6487],{},[1047,29079,19786],{},[2545,29081,29082,29084,29092],{},[1047,29083,26504],{},[1032,29085,29086,29088,29090],{},[1047,29087,1733],{},[1414,29089,6487],{},[1038,29091,574],{},[1047,29093,1625],{},[1035,29095,29096,29098],{},[1047,29097,28563],{},[1032,29099,29100,29102,29104],{},[1038,29101,574],{},[1414,29103,19155],{},[1047,29105,28563],{},[1414,29107,6487],{},[1414,29109,24272],{},[3999,29111,8013],{},[1414,29113,1768],{"fence":1062},[15157,29115,29116,29122],{},[15160,29117,29118],{},[15163,29119,29120],{"scriptlevel":560,"displaystyle":1062},[1032,29121],{},[15160,29123,29124],{},[15163,29125,29126],{"scriptlevel":560,"displaystyle":1062},[1032,29127,29128,29130,29132,29147,29149,29151,29153,29159],{},[1032,29129],{},[1414,29131,281],{},[15472,29133,29134,29136,29144],{},[1414,29135,2549],{},[1032,29137,29138,29140,29142],{},[1047,29139,8028],{},[1414,29141,281],{},[1038,29143,560],{},[1047,29145,29146],{"mathvariant":1729},"∞",[1414,29148,1754],{"stretchy":1753},[1047,29150,19786],{},[1047,29152,28563],{},[1418,29154,29155,29157],{},[1414,29156,1768],{"stretchy":1753},[1047,29158,8028],{},[2545,29160,29161,29163,29171],{},[1047,29162,26504],{},[1032,29164,29165,29167,29169],{},[1047,29166,1733],{},[1414,29168,6487],{},[1047,29170,8028],{},[1047,29172,1625],{},[1054,29174,29175],{"encoding":1056},"\\begin{aligned}\nA_\\theta^{GAE}(s_t, a) &= (1 - \\lambda)(A_\\theta^1 + \\lambda * A_\\theta^2 + \\lambda^2 A_\\theta^3 + \\cdots) \\quad\\quad \\lambda = 0.9: \\quad A_\\theta^{GAE} = 0.1A_\\theta^1 + 0.09A_\\theta^2 + 0.081A_\\theta^3 + \\cdots \\\\\n&= (1 - \\lambda)(\\delta_t^V + \\lambda * (\\delta_t^V + \\gamma \\delta_{t+1}^V) + \\lambda^2 (\\delta_t^V + \\gamma \\delta_{t+1}^V + \\gamma^2 \\delta_{t+2}^V) + \\cdots) \\\\\n&= (1 - \\lambda)(\\delta_t^V (1 + \\lambda + \\lambda^2 + \\cdots) + \\gamma \\delta_{t+1}^V * (\\lambda + \\lambda^2 + \\cdots) + \\cdots) \\\\\n&= (1 - \\lambda)\\left(\\delta_t^V \\frac{1}{1 - \\lambda} + \\gamma \\delta_{t+1}^V \\frac{\\lambda}{1 - \\lambda} + \\cdots\\right) \\\\\n&= \\sum\\limits_{b=0}^\\infty (\\gamma\\lambda)^b \\delta_{t+b}^V\n\\end{aligned}",[57,29177,29179],{"className":29178,"ariaHidden":1062},[1061],[57,29180,29182,29186],{"className":29181},[1066],[57,29183],{"className":29184,"style":29185},[1070],"height:10.6075em;vertical-align:-5.0538em;",[57,29187,29189],{"className":29188},[1075],[57,29190,29192,29378],{"className":29191},[15151],[57,29193,29195],{"className":29194},[15942],[57,29196,29198,29369],{"className":29197},[1087,1088],[57,29199,29201,29366],{"className":29200},[1092],[57,29202,29205,29330,29339,29348,29357],{"className":29203,"style":29204},[1096],"height:5.5538em;",[57,29206,29208,29212],{"style":29207},"top:-8.3138em;",[57,29209],{"className":29210,"style":29211},[1104],"height:3.6514em;",[57,29213,29215,29275,29278,29318,29321,29324,29327],{"className":29214},[1075],[57,29216,29218,29221],{"className":29217},[1075],[57,29219,1730],{"className":29220},[1075,1148],[57,29222,29224],{"className":29223},[1152],[57,29225,29227,29267],{"className":29226},[1087,1088],[57,29228,29230,29264],{"className":29229},[1092],[57,29231,29233,29244],{"className":29232,"style":25824},[1096],[57,29234,29235,29238],{"style":18101},[57,29236],{"className":29237,"style":1493},[1104],[57,29239,29241],{"className":29240},[1109,1110,1111,1112],[57,29242,14745],{"className":29243,"style":3932},[1075,1148,1112],[57,29245,29246,29249],{"style":17216},[57,29247],{"className":29248,"style":1493},[1104],[57,29250,29252],{"className":29251},[1109,1110,1111,1112],[57,29253,29255,29258,29261],{"className":29254},[1075,1112],[57,29256,28613],{"className":29257},[1075,1148,1112],[57,29259,1730],{"className":29260},[1075,1148,1112],[57,29262,6336],{"className":29263,"style":6749},[1075,1148,1112],[57,29265,1183],{"className":29266},[1182],[57,29268,29270],{"className":29269},[1092],[57,29271,29273],{"className":29272,"style":18131},[1096],[57,29274],{},[57,29276,1754],{"className":29277},[1079],[57,29279,29281,29284],{"className":29280},[1075],[57,29282,13535],{"className":29283},[1075,1148],[57,29285,29287],{"className":29286},[1152],[57,29288,29290,29310],{"className":29289},[1087,1088],[57,29291,29293,29307],{"className":29292},[1092],[57,29294,29296],{"className":29295,"style":13464},[1096],[57,29297,29298,29301],{"style":2211},[57,29299],{"className":29300,"style":1493},[1104],[57,29302,29304],{"className":29303},[1109,1110,1111,1112],[57,29305,1733],{"className":29306},[1075,1148,1112],[57,29308,1183],{"className":29309},[1182],[57,29311,29313],{"className":29312},[1092],[57,29314,29316],{"className":29315,"style":2230},[1096],[57,29317],{},[57,29319,1759],{"className":29320},[1849],[57,29322],{"className":29323,"style":1853},[1452],[57,29325,18],{"className":29326},[1075,1148],[57,29328,1768],{"className":29329},[1269],[57,29331,29333,29336],{"style":29332},"top:-6.7625em;",[57,29334],{"className":29335,"style":29211},[1104],[57,29337],{"className":29338},[1075],[57,29340,29342,29345],{"style":29341},"top:-5.2112em;",[57,29343],{"className":29344,"style":29211},[1104],[57,29346],{"className":29347},[1075],[57,29349,29351,29354],{"style":29350},"top:-3.1012em;",[57,29352],{"className":29353,"style":29211},[1104],[57,29355],{"className":29356},[1075],[57,29358,29360,29363],{"style":29359},"top:-0.1997em;",[57,29361],{"className":29362,"style":29211},[1104],[57,29364],{"className":29365},[1075],[57,29367,1183],{"className":29368},[1182],[57,29370,29372],{"className":29371},[1092],[57,29373,29376],{"className":29374,"style":29375},[1096],"height:5.0538em;",[57,29377],{},[57,29379,29381],{"className":29380},[16166],[57,29382,29384,31353],{"className":29383},[1087,1088],[57,29385,29387,31350],{"className":29386},[1092],[57,29388,29390,29959,30487,30817,31164],{"className":29389,"style":29204},[1096],[57,29391,29392,29395],{"style":29207},[57,29393],{"className":29394,"style":29211},[1104],[57,29396,29398,29401,29404,29407,29410,29413,29416,29419,29422,29425,29428,29431,29434,29485,29488,29491,29494,29497,29500,29503,29506,29557,29560,29563,29566,29595,29646,29649,29652,29655,29658,29661,29664,29668,29671,29674,29677,29680,29683,29686,29689,29692,29695,29698,29758,29761,29764,29767,29770,29821,29824,29827,29830,29833,29884,29887,29890,29893,29896,29947,29950,29953,29956],{"className":29397},[1075],[57,29399],{"className":29400},[1075],[57,29402],{"className":29403,"style":1453},[1452],[57,29405,281],{"className":29406},[1457],[57,29408],{"className":29409,"style":1453},[1452],[57,29411,1754],{"className":29412},[1079],[57,29414,574],{"className":29415},[1075],[57,29417],{"className":29418,"style":2595},[1452],[57,29420,19155],{"className":29421},[1506],[57,29423],{"className":29424,"style":2595},[1452],[57,29426,28563],{"className":29427},[1075,1148],[57,29429,1768],{"className":29430},[1269],[57,29432,1754],{"className":29433},[1079],[57,29435,29437,29440],{"className":29436},[1075],[57,29438,1730],{"className":29439},[1075,1148],[57,29441,29443],{"className":29442},[1152],[57,29444,29446,29477],{"className":29445},[1087,1088],[57,29447,29449,29474],{"className":29448},[1092],[57,29450,29452,29463],{"className":29451,"style":24338},[1096],[57,29453,29454,29457],{"style":18101},[57,29455],{"className":29456,"style":1493},[1104],[57,29458,29460],{"className":29459},[1109,1110,1111,1112],[57,29461,14745],{"className":29462,"style":3932},[1075,1148,1112],[57,29464,29465,29468],{"style":17216},[57,29466],{"className":29467,"style":1493},[1104],[57,29469,29471],{"className":29470},[1109,1110,1111,1112],[57,29472,574],{"className":29473},[1075,1112],[57,29475,1183],{"className":29476},[1182],[57,29478,29480],{"className":29479},[1092],[57,29481,29483],{"className":29482,"style":18131},[1096],[57,29484],{},[57,29486],{"className":29487,"style":2595},[1452],[57,29489,6487],{"className":29490},[1506],[57,29492],{"className":29493,"style":2595},[1452],[57,29495,28563],{"className":29496},[1075,1148],[57,29498],{"className":29499,"style":2595},[1452],[57,29501,24077],{"className":29502},[1506],[57,29504],{"className":29505,"style":2595},[1452],[57,29507,29509,29512],{"className":29508},[1075],[57,29510,1730],{"className":29511},[1075,1148],[57,29513,29515],{"className":29514},[1152],[57,29516,29518,29549],{"className":29517},[1087,1088],[57,29519,29521,29546],{"className":29520},[1092],[57,29522,29524,29535],{"className":29523,"style":24338},[1096],[57,29525,29526,29529],{"style":18101},[57,29527],{"className":29528,"style":1493},[1104],[57,29530,29532],{"className":29531},[1109,1110,1111,1112],[57,29533,14745],{"className":29534,"style":3932},[1075,1148,1112],[57,29536,29537,29540],{"style":17216},[57,29538],{"className":29539,"style":1493},[1104],[57,29541,29543],{"className":29542},[1109,1110,1111,1112],[57,29544,590],{"className":29545},[1075,1112],[57,29547,1183],{"className":29548},[1182],[57,29550,29552],{"className":29551},[1092],[57,29553,29555],{"className":29554,"style":18131},[1096],[57,29556],{},[57,29558],{"className":29559,"style":2595},[1452],[57,29561,6487],{"className":29562},[1506],[57,29564],{"className":29565,"style":2595},[1452],[57,29567,29569,29572],{"className":29568},[1075],[57,29570,28563],{"className":29571},[1075,1148],[57,29573,29575],{"className":29574},[1152],[57,29576,29578],{"className":29577},[1087],[57,29579,29581],{"className":29580},[1092],[57,29582,29584],{"className":29583,"style":24338},[1096],[57,29585,29586,29589],{"style":17216},[57,29587],{"className":29588,"style":1493},[1104],[57,29590,29592],{"className":29591},[1109,1110,1111,1112],[57,29593,590],{"className":29594},[1075,1112],[57,29596,29598,29601],{"className":29597},[1075],[57,29599,1730],{"className":29600},[1075,1148],[57,29602,29604],{"className":29603},[1152],[57,29605,29607,29638],{"className":29606},[1087,1088],[57,29608,29610,29635],{"className":29609},[1092],[57,29611,29613,29624],{"className":29612,"style":24338},[1096],[57,29614,29615,29618],{"style":18101},[57,29616],{"className":29617,"style":1493},[1104],[57,29619,29621],{"className":29620},[1109,1110,1111,1112],[57,29622,14745],{"className":29623,"style":3932},[1075,1148,1112],[57,29625,29626,29629],{"style":17216},[57,29627],{"className":29628,"style":1493},[1104],[57,29630,29632],{"className":29631},[1109,1110,1111,1112],[57,29633,606],{"className":29634},[1075,1112],[57,29636,1183],{"className":29637},[1182],[57,29639,29641],{"className":29640},[1092],[57,29642,29644],{"className":29643,"style":18131},[1096],[57,29645],{},[57,29647],{"className":29648,"style":2595},[1452],[57,29650,6487],{"className":29651},[1506],[57,29653],{"className":29654,"style":2595},[1452],[57,29656,24272],{"className":29657},[1898],[57,29659],{"className":29660,"style":1853},[1452],[57,29662,1768],{"className":29663},[1269],[57,29665],{"className":29666,"style":29667},[1452],"margin-right:1em;",[57,29669],{"className":29670,"style":29667},[1452],[57,29672,28563],{"className":29673},[1075,1148],[57,29675],{"className":29676,"style":1453},[1452],[57,29678,281],{"className":29679},[1457],[57,29681],{"className":29682,"style":1453},[1452],[57,29684,28711],{"className":29685},[1075],[57,29687],{"className":29688,"style":1453},[1452],[57,29690,7851],{"className":29691},[1457],[57,29693],{"className":29694,"style":29667},[1452],[57,29696],{"className":29697,"style":1453},[1452],[57,29699,29701,29704],{"className":29700},[1075],[57,29702,1730],{"className":29703},[1075,1148],[57,29705,29707],{"className":29706},[1152],[57,29708,29710,29750],{"className":29709},[1087,1088],[57,29711,29713,29747],{"className":29712},[1092],[57,29714,29716,29727],{"className":29715,"style":25824},[1096],[57,29717,29718,29721],{"style":18101},[57,29719],{"className":29720,"style":1493},[1104],[57,29722,29724],{"className":29723},[1109,1110,1111,1112],[57,29725,14745],{"className":29726,"style":3932},[1075,1148,1112],[57,29728,29729,29732],{"style":17216},[57,29730],{"className":29731,"style":1493},[1104],[57,29733,29735],{"className":29734},[1109,1110,1111,1112],[57,29736,29738,29741,29744],{"className":29737},[1075,1112],[57,29739,28613],{"className":29740},[1075,1148,1112],[57,29742,1730],{"className":29743},[1075,1148,1112],[57,29745,6336],{"className":29746,"style":6749},[1075,1148,1112],[57,29748,1183],{"className":29749},[1182],[57,29751,29753],{"className":29752},[1092],[57,29754,29756],{"className":29755,"style":18131},[1096],[57,29757],{},[57,29759],{"className":29760,"style":1453},[1452],[57,29762,281],{"className":29763},[1457],[57,29765],{"className":29766,"style":1453},[1452],[57,29768,13195],{"className":29769},[1075],[57,29771,29773,29776],{"className":29772},[1075],[57,29774,1730],{"className":29775},[1075,1148],[57,29777,29779],{"className":29778},[1152],[57,29780,29782,29813],{"className":29781},[1087,1088],[57,29783,29785,29810],{"className":29784},[1092],[57,29786,29788,29799],{"className":29787,"style":24338},[1096],[57,29789,29790,29793],{"style":18101},[57,29791],{"className":29792,"style":1493},[1104],[57,29794,29796],{"className":29795},[1109,1110,1111,1112],[57,29797,14745],{"className":29798,"style":3932},[1075,1148,1112],[57,29800,29801,29804],{"style":17216},[57,29802],{"className":29803,"style":1493},[1104],[57,29805,29807],{"className":29806},[1109,1110,1111,1112],[57,29808,574],{"className":29809},[1075,1112],[57,29811,1183],{"className":29812},[1182],[57,29814,29816],{"className":29815},[1092],[57,29817,29819],{"className":29818,"style":18131},[1096],[57,29820],{},[57,29822],{"className":29823,"style":2595},[1452],[57,29825,6487],{"className":29826},[1506],[57,29828],{"className":29829,"style":2595},[1452],[57,29831,28746],{"className":29832},[1075],[57,29834,29836,29839],{"className":29835},[1075],[57,29837,1730],{"className":29838},[1075,1148],[57,29840,29842],{"className":29841},[1152],[57,29843,29845,29876],{"className":29844},[1087,1088],[57,29846,29848,29873],{"className":29847},[1092],[57,29849,29851,29862],{"className":29850,"style":24338},[1096],[57,29852,29853,29856],{"style":18101},[57,29854],{"className":29855,"style":1493},[1104],[57,29857,29859],{"className":29858},[1109,1110,1111,1112],[57,29860,14745],{"className":29861,"style":3932},[1075,1148,1112],[57,29863,29864,29867],{"style":17216},[57,29865],{"className":29866,"style":1493},[1104],[57,29868,29870],{"className":29869},[1109,1110,1111,1112],[57,29871,590],{"className":29872},[1075,1112],[57,29874,1183],{"className":29875},[1182],[57,29877,29879],{"className":29878},[1092],[57,29880,29882],{"className":29881,"style":18131},[1096],[57,29883],{},[57,29885],{"className":29886,"style":2595},[1452],[57,29888,6487],{"className":29889},[1506],[57,29891],{"className":29892,"style":2595},[1452],[57,29894,28759],{"className":29895},[1075],[57,29897,29899,29902],{"className":29898},[1075],[57,29900,1730],{"className":29901},[1075,1148],[57,29903,29905],{"className":29904},[1152],[57,29906,29908,29939],{"className":29907},[1087,1088],[57,29909,29911,29936],{"className":29910},[1092],[57,29912,29914,29925],{"className":29913,"style":24338},[1096],[57,29915,29916,29919],{"style":18101},[57,29917],{"className":29918,"style":1493},[1104],[57,29920,29922],{"className":29921},[1109,1110,1111,1112],[57,29923,14745],{"className":29924,"style":3932},[1075,1148,1112],[57,29926,29927,29930],{"style":17216},[57,29928],{"className":29929,"style":1493},[1104],[57,29931,29933],{"className":29932},[1109,1110,1111,1112],[57,29934,606],{"className":29935},[1075,1112],[57,29937,1183],{"className":29938},[1182],[57,29940,29942],{"className":29941},[1092],[57,29943,29945],{"className":29944,"style":18131},[1096],[57,29946],{},[57,29948],{"className":29949,"style":2595},[1452],[57,29951,6487],{"className":29952},[1506],[57,29954],{"className":29955,"style":2595},[1452],[57,29957,24272],{"className":29958},[1898],[57,29960,29961,29964],{"style":29332},[57,29962],{"className":29963,"style":29211},[1104],[57,29965,29967,29970,29973,29976,29979,29982,29985,29988,29991,29994,29997,30000,30003,30054,30057,30060,30063,30066,30069,30072,30075,30078,30129,30132,30135,30138,30141,30201,30204,30207,30210,30213,30242,30245,30296,30299,30302,30305,30308,30368,30371,30374,30377,30406,30466,30469,30472,30475,30478,30481,30484],{"className":29966},[1075],[57,29968],{"className":29969},[1075],[57,29971],{"className":29972,"style":1453},[1452],[57,29974,281],{"className":29975},[1457],[57,29977],{"className":29978,"style":1453},[1452],[57,29980,1754],{"className":29981},[1079],[57,29983,574],{"className":29984},[1075],[57,29986],{"className":29987,"style":2595},[1452],[57,29989,19155],{"className":29990},[1506],[57,29992],{"className":29993,"style":2595},[1452],[57,29995,28563],{"className":29996},[1075,1148],[57,29998,1768],{"className":29999},[1269],[57,30001,1754],{"className":30002},[1079],[57,30004,30006,30009],{"className":30005},[1075],[57,30007,26504],{"className":30008,"style":26528},[1075,1148],[57,30010,30012],{"className":30011},[1152],[57,30013,30015,30046],{"className":30014},[1087,1088],[57,30016,30018,30043],{"className":30017},[1092],[57,30019,30021,30032],{"className":30020,"style":25824},[1096],[57,30022,30023,30026],{"style":26543},[57,30024],{"className":30025,"style":1493},[1104],[57,30027,30029],{"className":30028},[1109,1110,1111,1112],[57,30030,1733],{"className":30031},[1075,1148,1112],[57,30033,30034,30037],{"style":17216},[57,30035],{"className":30036,"style":1493},[1104],[57,30038,30040],{"className":30039},[1109,1110,1111,1112],[57,30041,1625],{"className":30042,"style":1655},[1075,1148,1112],[57,30044,1183],{"className":30045},[1182],[57,30047,30049],{"className":30048},[1092],[57,30050,30052],{"className":30051,"style":18131},[1096],[57,30053],{},[57,30055],{"className":30056,"style":2595},[1452],[57,30058,6487],{"className":30059},[1506],[57,30061],{"className":30062,"style":2595},[1452],[57,30064,28563],{"className":30065},[1075,1148],[57,30067],{"className":30068,"style":2595},[1452],[57,30070,24077],{"className":30071},[1506],[57,30073],{"className":30074,"style":2595},[1452],[57,30076,1754],{"className":30077},[1079],[57,30079,30081,30084],{"className":30080},[1075],[57,30082,26504],{"className":30083,"style":26528},[1075,1148],[57,30085,30087],{"className":30086},[1152],[57,30088,30090,30121],{"className":30089},[1087,1088],[57,30091,30093,30118],{"className":30092},[1092],[57,30094,30096,30107],{"className":30095,"style":25824},[1096],[57,30097,30098,30101],{"style":26543},[57,30099],{"className":30100,"style":1493},[1104],[57,30102,30104],{"className":30103},[1109,1110,1111,1112],[57,30105,1733],{"className":30106},[1075,1148,1112],[57,30108,30109,30112],{"style":17216},[57,30110],{"className":30111,"style":1493},[1104],[57,30113,30115],{"className":30114},[1109,1110,1111,1112],[57,30116,1625],{"className":30117,"style":1655},[1075,1148,1112],[57,30119,1183],{"className":30120},[1182],[57,30122,30124],{"className":30123},[1092],[57,30125,30127],{"className":30126,"style":18131},[1096],[57,30128],{},[57,30130],{"className":30131,"style":2595},[1452],[57,30133,6487],{"className":30134},[1506],[57,30136],{"className":30137,"style":2595},[1452],[57,30139,19786],{"className":30140,"style":20033},[1075,1148],[57,30142,30144,30147],{"className":30143},[1075],[57,30145,26504],{"className":30146,"style":26528},[1075,1148],[57,30148,30150],{"className":30149},[1152],[57,30151,30153,30193],{"className":30152},[1087,1088],[57,30154,30156,30190],{"className":30155},[1092],[57,30157,30159,30179],{"className":30158,"style":25824},[1096],[57,30160,30161,30164],{"style":26543},[57,30162],{"className":30163,"style":1493},[1104],[57,30165,30167],{"className":30166},[1109,1110,1111,1112],[57,30168,30170,30173,30176],{"className":30169},[1075,1112],[57,30171,1733],{"className":30172},[1075,1148,1112],[57,30174,6487],{"className":30175},[1506,1112],[57,30177,574],{"className":30178},[1075,1112],[57,30180,30181,30184],{"style":17216},[57,30182],{"className":30183,"style":1493},[1104],[57,30185,30187],{"className":30186},[1109,1110,1111,1112],[57,30188,1625],{"className":30189,"style":1655},[1075,1148,1112],[57,30191,1183],{"className":30192},[1182],[57,30194,30196],{"className":30195},[1092],[57,30197,30199],{"className":30198,"style":27353},[1096],[57,30200],{},[57,30202,1768],{"className":30203},[1269],[57,30205],{"className":30206,"style":2595},[1452],[57,30208,6487],{"className":30209},[1506],[57,30211],{"className":30212,"style":2595},[1452],[57,30214,30216,30219],{"className":30215},[1075],[57,30217,28563],{"className":30218},[1075,1148],[57,30220,30222],{"className":30221},[1152],[57,30223,30225],{"className":30224},[1087],[57,30226,30228],{"className":30227},[1092],[57,30229,30231],{"className":30230,"style":24338},[1096],[57,30232,30233,30236],{"style":17216},[57,30234],{"className":30235,"style":1493},[1104],[57,30237,30239],{"className":30238},[1109,1110,1111,1112],[57,30240,590],{"className":30241},[1075,1112],[57,30243,1754],{"className":30244},[1079],[57,30246,30248,30251],{"className":30247},[1075],[57,30249,26504],{"className":30250,"style":26528},[1075,1148],[57,30252,30254],{"className":30253},[1152],[57,30255,30257,30288],{"className":30256},[1087,1088],[57,30258,30260,30285],{"className":30259},[1092],[57,30261,30263,30274],{"className":30262,"style":25824},[1096],[57,30264,30265,30268],{"style":26543},[57,30266],{"className":30267,"style":1493},[1104],[57,30269,30271],{"className":30270},[1109,1110,1111,1112],[57,30272,1733],{"className":30273},[1075,1148,1112],[57,30275,30276,30279],{"style":17216},[57,30277],{"className":30278,"style":1493},[1104],[57,30280,30282],{"className":30281},[1109,1110,1111,1112],[57,30283,1625],{"className":30284,"style":1655},[1075,1148,1112],[57,30286,1183],{"className":30287},[1182],[57,30289,30291],{"className":30290},[1092],[57,30292,30294],{"className":30293,"style":18131},[1096],[57,30295],{},[57,30297],{"className":30298,"style":2595},[1452],[57,30300,6487],{"className":30301},[1506],[57,30303],{"className":30304,"style":2595},[1452],[57,30306,19786],{"className":30307,"style":20033},[1075,1148],[57,30309,30311,30314],{"className":30310},[1075],[57,30312,26504],{"className":30313,"style":26528},[1075,1148],[57,30315,30317],{"className":30316},[1152],[57,30318,30320,30360],{"className":30319},[1087,1088],[57,30321,30323,30357],{"className":30322},[1092],[57,30324,30326,30346],{"className":30325,"style":25824},[1096],[57,30327,30328,30331],{"style":26543},[57,30329],{"className":30330,"style":1493},[1104],[57,30332,30334],{"className":30333},[1109,1110,1111,1112],[57,30335,30337,30340,30343],{"className":30336},[1075,1112],[57,30338,1733],{"className":30339},[1075,1148,1112],[57,30341,6487],{"className":30342},[1506,1112],[57,30344,574],{"className":30345},[1075,1112],[57,30347,30348,30351],{"style":17216},[57,30349],{"className":30350,"style":1493},[1104],[57,30352,30354],{"className":30353},[1109,1110,1111,1112],[57,30355,1625],{"className":30356,"style":1655},[1075,1148,1112],[57,30358,1183],{"className":30359},[1182],[57,30361,30363],{"className":30362},[1092],[57,30364,30366],{"className":30365,"style":27353},[1096],[57,30367],{},[57,30369],{"className":30370,"style":2595},[1452],[57,30372,6487],{"className":30373},[1506],[57,30375],{"className":30376,"style":2595},[1452],[57,30378,30380,30383],{"className":30379},[1075],[57,30381,19786],{"className":30382,"style":20033},[1075,1148],[57,30384,30386],{"className":30385},[1152],[57,30387,30389],{"className":30388},[1087],[57,30390,30392],{"className":30391},[1092],[57,30393,30395],{"className":30394,"style":24338},[1096],[57,30396,30397,30400],{"style":17216},[57,30398],{"className":30399,"style":1493},[1104],[57,30401,30403],{"className":30402},[1109,1110,1111,1112],[57,30404,590],{"className":30405},[1075,1112],[57,30407,30409,30412],{"className":30408},[1075],[57,30410,26504],{"className":30411,"style":26528},[1075,1148],[57,30413,30415],{"className":30414},[1152],[57,30416,30418,30458],{"className":30417},[1087,1088],[57,30419,30421,30455],{"className":30420},[1092],[57,30422,30424,30444],{"className":30423,"style":25824},[1096],[57,30425,30426,30429],{"style":26543},[57,30427],{"className":30428,"style":1493},[1104],[57,30430,30432],{"className":30431},[1109,1110,1111,1112],[57,30433,30435,30438,30441],{"className":30434},[1075,1112],[57,30436,1733],{"className":30437},[1075,1148,1112],[57,30439,6487],{"className":30440},[1506,1112],[57,30442,590],{"className":30443},[1075,1112],[57,30445,30446,30449],{"style":17216},[57,30447],{"className":30448,"style":1493},[1104],[57,30450,30452],{"className":30451},[1109,1110,1111,1112],[57,30453,1625],{"className":30454,"style":1655},[1075,1148,1112],[57,30456,1183],{"className":30457},[1182],[57,30459,30461],{"className":30460},[1092],[57,30462,30464],{"className":30463,"style":27353},[1096],[57,30465],{},[57,30467,1768],{"className":30468},[1269],[57,30470],{"className":30471,"style":2595},[1452],[57,30473,6487],{"className":30474},[1506],[57,30476],{"className":30477,"style":2595},[1452],[57,30479,24272],{"className":30480},[1898],[57,30482],{"className":30483,"style":1853},[1452],[57,30485,1768],{"className":30486},[1269],[57,30488,30489,30492],{"style":29341},[57,30490],{"className":30491,"style":29211},[1104],[57,30493,30495,30498,30501,30504,30507,30510,30513,30516,30519,30522,30525,30528,30531,30582,30585,30588,30591,30594,30597,30600,30603,30606,30609,30638,30641,30644,30647,30650,30653,30656,30659,30662,30665,30668,30728,30731,30734,30737,30740,30743,30746,30749,30752,30781,30784,30787,30790,30793,30796,30799,30802,30805,30808,30811,30814],{"className":30494},[1075],[57,30496],{"className":30497},[1075],[57,30499],{"className":30500,"style":1453},[1452],[57,30502,281],{"className":30503},[1457],[57,30505],{"className":30506,"style":1453},[1452],[57,30508,1754],{"className":30509},[1079],[57,30511,574],{"className":30512},[1075],[57,30514],{"className":30515,"style":2595},[1452],[57,30517,19155],{"className":30518},[1506],[57,30520],{"className":30521,"style":2595},[1452],[57,30523,28563],{"className":30524},[1075,1148],[57,30526,1768],{"className":30527},[1269],[57,30529,1754],{"className":30530},[1079],[57,30532,30534,30537],{"className":30533},[1075],[57,30535,26504],{"className":30536,"style":26528},[1075,1148],[57,30538,30540],{"className":30539},[1152],[57,30541,30543,30574],{"className":30542},[1087,1088],[57,30544,30546,30571],{"className":30545},[1092],[57,30547,30549,30560],{"className":30548,"style":25824},[1096],[57,30550,30551,30554],{"style":26543},[57,30552],{"className":30553,"style":1493},[1104],[57,30555,30557],{"className":30556},[1109,1110,1111,1112],[57,30558,1733],{"className":30559},[1075,1148,1112],[57,30561,30562,30565],{"style":17216},[57,30563],{"className":30564,"style":1493},[1104],[57,30566,30568],{"className":30567},[1109,1110,1111,1112],[57,30569,1625],{"className":30570,"style":1655},[1075,1148,1112],[57,30572,1183],{"className":30573},[1182],[57,30575,30577],{"className":30576},[1092],[57,30578,30580],{"className":30579,"style":18131},[1096],[57,30581],{},[57,30583,1754],{"className":30584},[1079],[57,30586,574],{"className":30587},[1075],[57,30589],{"className":30590,"style":2595},[1452],[57,30592,6487],{"className":30593},[1506],[57,30595],{"className":30596,"style":2595},[1452],[57,30598,28563],{"className":30599},[1075,1148],[57,30601],{"className":30602,"style":2595},[1452],[57,30604,6487],{"className":30605},[1506],[57,30607],{"className":30608,"style":2595},[1452],[57,30610,30612,30615],{"className":30611},[1075],[57,30613,28563],{"className":30614},[1075,1148],[57,30616,30618],{"className":30617},[1152],[57,30619,30621],{"className":30620},[1087],[57,30622,30624],{"className":30623},[1092],[57,30625,30627],{"className":30626,"style":24338},[1096],[57,30628,30629,30632],{"style":17216},[57,30630],{"className":30631,"style":1493},[1104],[57,30633,30635],{"className":30634},[1109,1110,1111,1112],[57,30636,590],{"className":30637},[1075,1112],[57,30639],{"className":30640,"style":2595},[1452],[57,30642,6487],{"className":30643},[1506],[57,30645],{"className":30646,"style":2595},[1452],[57,30648,24272],{"className":30649},[1898],[57,30651],{"className":30652,"style":1853},[1452],[57,30654,1768],{"className":30655},[1269],[57,30657],{"className":30658,"style":2595},[1452],[57,30660,6487],{"className":30661},[1506],[57,30663],{"className":30664,"style":2595},[1452],[57,30666,19786],{"className":30667,"style":20033},[1075,1148],[57,30669,30671,30674],{"className":30670},[1075],[57,30672,26504],{"className":30673,"style":26528},[1075,1148],[57,30675,30677],{"className":30676},[1152],[57,30678,30680,30720],{"className":30679},[1087,1088],[57,30681,30683,30717],{"className":30682},[1092],[57,30684,30686,30706],{"className":30685,"style":25824},[1096],[57,30687,30688,30691],{"style":26543},[57,30689],{"className":30690,"style":1493},[1104],[57,30692,30694],{"className":30693},[1109,1110,1111,1112],[57,30695,30697,30700,30703],{"className":30696},[1075,1112],[57,30698,1733],{"className":30699},[1075,1148,1112],[57,30701,6487],{"className":30702},[1506,1112],[57,30704,574],{"className":30705},[1075,1112],[57,30707,30708,30711],{"style":17216},[57,30709],{"className":30710,"style":1493},[1104],[57,30712,30714],{"className":30713},[1109,1110,1111,1112],[57,30715,1625],{"className":30716,"style":1655},[1075,1148,1112],[57,30718,1183],{"className":30719},[1182],[57,30721,30723],{"className":30722},[1092],[57,30724,30726],{"className":30725,"style":27353},[1096],[57,30727],{},[57,30729],{"className":30730,"style":2595},[1452],[57,30732,24077],{"className":30733},[1506],[57,30735],{"className":30736,"style":2595},[1452],[57,30738,1754],{"className":30739},[1079],[57,30741,28563],{"className":30742},[1075,1148],[57,30744],{"className":30745,"style":2595},[1452],[57,30747,6487],{"className":30748},[1506],[57,30750],{"className":30751,"style":2595},[1452],[57,30753,30755,30758],{"className":30754},[1075],[57,30756,28563],{"className":30757},[1075,1148],[57,30759,30761],{"className":30760},[1152],[57,30762,30764],{"className":30763},[1087],[57,30765,30767],{"className":30766},[1092],[57,30768,30770],{"className":30769,"style":24338},[1096],[57,30771,30772,30775],{"style":17216},[57,30773],{"className":30774,"style":1493},[1104],[57,30776,30778],{"className":30777},[1109,1110,1111,1112],[57,30779,590],{"className":30780},[1075,1112],[57,30782],{"className":30783,"style":2595},[1452],[57,30785,6487],{"className":30786},[1506],[57,30788],{"className":30789,"style":2595},[1452],[57,30791,24272],{"className":30792},[1898],[57,30794],{"className":30795,"style":1853},[1452],[57,30797,1768],{"className":30798},[1269],[57,30800],{"className":30801,"style":2595},[1452],[57,30803,6487],{"className":30804},[1506],[57,30806],{"className":30807,"style":2595},[1452],[57,30809,24272],{"className":30810},[1898],[57,30812],{"className":30813,"style":1853},[1452],[57,30815,1768],{"className":30816},[1269],[57,30818,30819,30822],{"style":29350},[57,30820],{"className":30821,"style":29211},[1104],[57,30823,30825,30828,30831,30834,30837,30840,30843,30846,30849,30852,30855,30858,30861],{"className":30824},[1075],[57,30826],{"className":30827},[1075],[57,30829],{"className":30830,"style":1453},[1452],[57,30832,281],{"className":30833},[1457],[57,30835],{"className":30836,"style":1453},[1452],[57,30838,1754],{"className":30839},[1079],[57,30841,574],{"className":30842},[1075],[57,30844],{"className":30845,"style":2595},[1452],[57,30847,19155],{"className":30848},[1506],[57,30850],{"className":30851,"style":2595},[1452],[57,30853,28563],{"className":30854},[1075,1148],[57,30856,1768],{"className":30857},[1269],[57,30859],{"className":30860,"style":1853},[1452],[57,30862,30864,30870,30921,30996,30999,31002,31005,31008,31068,31143,31146,31149,31152,31155,31158],{"className":30863},[1898],[57,30865,30867],{"className":30866,"style":1903},[1079,1902],[57,30868,1754],{"className":30869},[1907,1111],[57,30871,30873,30876],{"className":30872},[1075],[57,30874,26504],{"className":30875,"style":26528},[1075,1148],[57,30877,30879],{"className":30878},[1152],[57,30880,30882,30913],{"className":30881},[1087,1088],[57,30883,30885,30910],{"className":30884},[1092],[57,30886,30888,30899],{"className":30887,"style":25824},[1096],[57,30889,30890,30893],{"style":26543},[57,30891],{"className":30892,"style":1493},[1104],[57,30894,30896],{"className":30895},[1109,1110,1111,1112],[57,30897,1733],{"className":30898},[1075,1148,1112],[57,30900,30901,30904],{"style":17216},[57,30902],{"className":30903,"style":1493},[1104],[57,30905,30907],{"className":30906},[1109,1110,1111,1112],[57,30908,1625],{"className":30909,"style":1655},[1075,1148,1112],[57,30911,1183],{"className":30912},[1182],[57,30914,30916],{"className":30915},[1092],[57,30917,30919],{"className":30918,"style":18131},[1096],[57,30920],{},[57,30922,30924,30927,30993],{"className":30923},[1075],[57,30925],{"className":30926},[1079,1080],[57,30928,30930],{"className":30929},[1035],[57,30931,30933,30984],{"className":30932},[1087,1088],[57,30934,30936,30981],{"className":30935},[1092],[57,30937,30939,30962,30970],{"className":30938,"style":17068},[1096],[57,30940,30941,30944],{"style":16608},[57,30942],{"className":30943,"style":1105},[1104],[57,30945,30947,30950,30953,30956,30959],{"className":30946},[1075],[57,30948,574],{"className":30949},[1075],[57,30951],{"className":30952,"style":2595},[1452],[57,30954,19155],{"className":30955},[1506],[57,30957],{"className":30958,"style":2595},[1452],[57,30960,28563],{"className":30961},[1075,1148],[57,30963,30964,30967],{"style":1230},[57,30965],{"className":30966,"style":1105},[1104],[57,30968],{"className":30969,"style":1238},[1237],[57,30971,30972,30975],{"style":16674},[57,30973],{"className":30974,"style":1105},[1104],[57,30976,30978],{"className":30977},[1075],[57,30979,574],{"className":30980},[1075],[57,30982,1183],{"className":30983},[1182],[57,30985,30987],{"className":30986},[1092],[57,30988,30991],{"className":30989,"style":30990},[1096],"height:0.7693em;",[57,30992],{},[57,30994],{"className":30995},[1269,1080],[57,30997],{"className":30998,"style":2595},[1452],[57,31000,6487],{"className":31001},[1506],[57,31003],{"className":31004,"style":2595},[1452],[57,31006,19786],{"className":31007,"style":20033},[1075,1148],[57,31009,31011,31014],{"className":31010},[1075],[57,31012,26504],{"className":31013,"style":26528},[1075,1148],[57,31015,31017],{"className":31016},[1152],[57,31018,31020,31060],{"className":31019},[1087,1088],[57,31021,31023,31057],{"className":31022},[1092],[57,31024,31026,31046],{"className":31025,"style":25824},[1096],[57,31027,31028,31031],{"style":26543},[57,31029],{"className":31030,"style":1493},[1104],[57,31032,31034],{"className":31033},[1109,1110,1111,1112],[57,31035,31037,31040,31043],{"className":31036},[1075,1112],[57,31038,1733],{"className":31039},[1075,1148,1112],[57,31041,6487],{"className":31042},[1506,1112],[57,31044,574],{"className":31045},[1075,1112],[57,31047,31048,31051],{"style":17216},[57,31049],{"className":31050,"style":1493},[1104],[57,31052,31054],{"className":31053},[1109,1110,1111,1112],[57,31055,1625],{"className":31056,"style":1655},[1075,1148,1112],[57,31058,1183],{"className":31059},[1182],[57,31061,31063],{"className":31062},[1092],[57,31064,31066],{"className":31065,"style":27353},[1096],[57,31067],{},[57,31069,31071,31074,31140],{"className":31070},[1075],[57,31072],{"className":31073},[1079,1080],[57,31075,31077],{"className":31076},[1035],[57,31078,31080,31132],{"className":31079},[1087,1088],[57,31081,31083,31129],{"className":31082},[1092],[57,31084,31087,31110,31118],{"className":31085,"style":31086},[1096],"height:1.3714em;",[57,31088,31089,31092],{"style":16608},[57,31090],{"className":31091,"style":1105},[1104],[57,31093,31095,31098,31101,31104,31107],{"className":31094},[1075],[57,31096,574],{"className":31097},[1075],[57,31099],{"className":31100,"style":2595},[1452],[57,31102,19155],{"className":31103},[1506],[57,31105],{"className":31106,"style":2595},[1452],[57,31108,28563],{"className":31109},[1075,1148],[57,31111,31112,31115],{"style":1230},[57,31113],{"className":31114,"style":1105},[1104],[57,31116],{"className":31117,"style":1238},[1237],[57,31119,31120,31123],{"style":16674},[57,31121],{"className":31122,"style":1105},[1104],[57,31124,31126],{"className":31125},[1075],[57,31127,28563],{"className":31128},[1075,1148],[57,31130,1183],{"className":31131},[1182],[57,31133,31135],{"className":31134},[1092],[57,31136,31138],{"className":31137,"style":30990},[1096],[57,31139],{},[57,31141],{"className":31142},[1269,1080],[57,31144],{"className":31145,"style":2595},[1452],[57,31147,6487],{"className":31148},[1506],[57,31150],{"className":31151,"style":2595},[1452],[57,31153,24272],{"className":31154},[1898],[57,31156],{"className":31157,"style":1853},[1452],[57,31159,31161],{"className":31160,"style":1903},[1269,1902],[57,31162,1768],{"className":31163},[1907,1111],[57,31165,31166,31169],{"style":29359},[57,31167],{"className":31168,"style":29211},[1104],[57,31170,31172,31175,31178,31181,31184,31251,31254,31257,31260,31290],{"className":31171},[1075],[57,31173],{"className":31174},[1075],[57,31176],{"className":31177,"style":1453},[1452],[57,31179,281],{"className":31180},[1457],[57,31182],{"className":31183,"style":1453},[1452],[57,31185,31187],{"className":31186},[1888,15027],[57,31188,31190,31242],{"className":31189},[1087,1088],[57,31191,31193,31239],{"className":31192},[1092],[57,31194,31197,31218,31228],{"className":31195,"style":31196},[1096],"height:1.6514em;",[57,31198,31200,31203],{"style":31199},"top:-1.8479em;margin-left:0em;",[57,31201],{"className":31202,"style":16221},[1104],[57,31204,31206],{"className":31205},[1109,1110,1111,1112],[57,31207,31209,31212,31215],{"className":31208},[1075,1112],[57,31210,8028],{"className":31211},[1075,1148,1112],[57,31213,281],{"className":31214},[1457,1112],[57,31216,560],{"className":31217},[1075,1112],[57,31219,31220,31223],{"style":16230},[57,31221],{"className":31222,"style":16221},[1104],[57,31224,31225],{},[57,31226,2549],{"className":31227},[1888,2633,16239],[57,31229,31230,31233],{"style":17164},[57,31231],{"className":31232,"style":16221},[1104],[57,31234,31236],{"className":31235},[1109,1110,1111,1112],[57,31237,29146],{"className":31238},[1075,1112],[57,31240,1183],{"className":31241},[1182],[57,31243,31245],{"className":31244},[1092],[57,31246,31249],{"className":31247,"style":31248},[1096],"height:1.3021em;",[57,31250],{},[57,31252,1754],{"className":31253},[1079],[57,31255,19786],{"className":31256,"style":20033},[1075,1148],[57,31258,28563],{"className":31259},[1075,1148],[57,31261,31263,31266],{"className":31262},[1269],[57,31264,1768],{"className":31265},[1269],[57,31267,31269],{"className":31268},[1152],[57,31270,31272],{"className":31271},[1087],[57,31273,31275],{"className":31274},[1092],[57,31276,31279],{"className":31277,"style":31278},[1096],"height:0.8991em;",[57,31280,31281,31284],{"style":17216},[57,31282],{"className":31283,"style":1493},[1104],[57,31285,31287],{"className":31286},[1109,1110,1111,1112],[57,31288,8028],{"className":31289},[1075,1148,1112],[57,31291,31293,31296],{"className":31292},[1075],[57,31294,26504],{"className":31295,"style":26528},[1075,1148],[57,31297,31299],{"className":31298},[1152],[57,31300,31302,31342],{"className":31301},[1087,1088],[57,31303,31305,31339],{"className":31304},[1092],[57,31306,31308,31328],{"className":31307,"style":25824},[1096],[57,31309,31310,31313],{"style":26543},[57,31311],{"className":31312,"style":1493},[1104],[57,31314,31316],{"className":31315},[1109,1110,1111,1112],[57,31317,31319,31322,31325],{"className":31318},[1075,1112],[57,31320,1733],{"className":31321},[1075,1148,1112],[57,31323,6487],{"className":31324},[1506,1112],[57,31326,8028],{"className":31327},[1075,1148,1112],[57,31329,31330,31333],{"style":17216},[57,31331],{"className":31332,"style":1493},[1104],[57,31334,31336],{"className":31335},[1109,1110,1111,1112],[57,31337,1625],{"className":31338,"style":1655},[1075,1148,1112],[57,31340,1183],{"className":31341},[1182],[57,31343,31345],{"className":31344},[1092],[57,31346,31348],{"className":31347,"style":27353},[1096],[57,31349],{},[57,31351,1183],{"className":31352},[1182],[57,31354,31356],{"className":31355},[1092],[57,31357,31359],{"className":31358,"style":29375},[1096],[57,31360],{},[14,31362,31363],{},"通过等比数列求和化简，最终得到 GAE 的紧凑形式。",[14,31365,31366,31367,31419,31420,31471],{},"GAE 优势函数本质上是在 ",[57,31368,31370,31389],{"className":31369},[1019],[57,31371,31373],{"className":31372},[1023],[1025,31374,31375],{"xmlns":1027},[1029,31376,31377,31386],{},[1032,31378,31379,31381,31384],{},[1047,31380,28563],{},[1414,31382,31383],{},"→",[1038,31385,560],{},[1054,31387,31388],{"encoding":1056},"\\lambda \\to 0",[57,31390,31392,31410],{"className":31391,"ariaHidden":1062},[1061],[57,31393,31395,31398,31401,31404,31407],{"className":31394},[1066],[57,31396],{"className":31397,"style":1360},[1070],[57,31399,28563],{"className":31400},[1075,1148],[57,31402],{"className":31403,"style":1453},[1452],[57,31405,31383],{"className":31406},[1457],[57,31408],{"className":31409,"style":1453},[1452],[57,31411,31413,31416],{"className":31412},[1066],[57,31414],{"className":31415,"style":2489},[1070],[57,31417,560],{"className":31418},[1075],"（高偏差、低方差）与 ",[57,31421,31423,31441],{"className":31422},[1019],[57,31424,31426],{"className":31425},[1023],[1025,31427,31428],{"xmlns":1027},[1029,31429,31430,31438],{},[1032,31431,31432,31434,31436],{},[1047,31433,28563],{},[1414,31435,31383],{},[1038,31437,574],{},[1054,31439,31440],{"encoding":1056},"\\lambda \\to 1",[57,31442,31444,31462],{"className":31443,"ariaHidden":1062},[1061],[57,31445,31447,31450,31453,31456,31459],{"className":31446},[1066],[57,31448],{"className":31449,"style":1360},[1070],[57,31451,28563],{"className":31452},[1075,1148],[57,31454],{"className":31455,"style":1453},[1452],[57,31457,31383],{"className":31458},[1457],[57,31460],{"className":31461,"style":1453},[1452],[57,31463,31465,31468],{"className":31464},[1066],[57,31466],{"className":31467,"style":2489},[1070],[57,31469,574],{"className":31470},[1075],"（低偏差、高方差）之间做插值，从而平衡 bias 与 variance。",[14,31473,31474],{},"整理一下，到这里我们得到了三个关键表达式：",[57,31476,31478],{"className":31477},[15139],[57,31479,31481,31737],{"className":31480},[1019],[57,31482,31484],{"className":31483},[1023],[1025,31485,31486],{"xmlns":1027,"display":10210},[1029,31487,31488,31734],{},[1032,31489,31490,31498,31500,31506,31508,31510,31516,31518,31530,31532,31534,31540,31542,31548,31550,31552,31566,31568,31574,31576,31578,31580,31582,31596,31598,31600,31602,31608,31622,31624,31630,31644,31662,31676,31678,31686,31688,31696,31698,31700,31702,31704,31706,31712,31714,31722,31724,31732],{},[2545,31491,31492,31494,31496],{},[1047,31493,26504],{},[1047,31495,1733],{},[1047,31497,1625],{},[1414,31499,281],{},[1044,31501,31502,31504],{},[1047,31503,8500],{},[1047,31505,1733],{},[1414,31507,6487],{},[1047,31509,19786],{},[1044,31511,31512,31514],{},[1047,31513,1625],{},[1047,31515,14745],{},[1414,31517,1754],{"stretchy":1753},[1044,31519,31520,31522],{},[1047,31521,13535],{},[1032,31523,31524,31526,31528],{},[1047,31525,1733],{},[1414,31527,6487],{},[1038,31529,574],{},[1414,31531,1768],{"stretchy":1753},[1414,31533,19155],{},[1044,31535,31536,31538],{},[1047,31537,1625],{},[1047,31539,14745],{},[1414,31541,1754],{"stretchy":1753},[1044,31543,31544,31546],{},[1047,31545,13535],{},[1047,31547,1733],{},[1414,31549,1768],{"stretchy":1753},[1452,31551],{"linebreak":22540},[2545,31553,31554,31556,31558],{},[1047,31555,1730],{},[1047,31557,14745],{},[1032,31559,31560,31562,31564],{},[1047,31561,28613],{},[1047,31563,1730],{},[1047,31565,6336],{},[1414,31567,1754],{"stretchy":1753},[1044,31569,31570,31572],{},[1047,31571,13535],{},[1047,31573,1733],{},[1414,31575,1759],{"separator":1062},[1047,31577,18],{},[1414,31579,1768],{"stretchy":1753},[1414,31581,281],{},[15472,31583,31584,31586,31594],{},[1414,31585,2549],{},[1032,31587,31588,31590,31592],{},[1047,31589,8028],{},[1414,31591,281],{},[1038,31593,560],{},[1047,31595,29146],{"mathvariant":1729},[1414,31597,1754],{"stretchy":1753},[1047,31599,19786],{},[1047,31601,28563],{},[1418,31603,31604,31606],{},[1414,31605,1768],{"stretchy":1753},[1047,31607,8028],{},[2545,31609,31610,31612,31620],{},[1047,31611,26504],{},[1032,31613,31614,31616,31618],{},[1047,31615,1733],{},[1414,31617,6487],{},[1047,31619,8028],{},[1047,31621,1625],{},[1452,31623],{"linebreak":22540},[1035,31625,31626,31628],{},[1038,31627,574],{},[1047,31629,7990],{},[15472,31631,31632,31634,31642],{},[1414,31633,2549],{},[1032,31635,31636,31638,31640],{},[1047,31637,1288],{},[1414,31639,281],{},[1038,31641,574],{},[1047,31643,7990],{},[15472,31645,31646,31648,31656],{},[1414,31647,2549],{},[1032,31649,31650,31652,31654],{},[1047,31651,1733],{},[1414,31653,281],{},[1038,31655,574],{},[1044,31657,31658,31660],{},[1047,31659,738],{},[1047,31661,1288],{},[2545,31663,31664,31666,31668],{},[1047,31665,1730],{},[1047,31667,14745],{},[1032,31669,31670,31672,31674],{},[1047,31671,28613],{},[1047,31673,1730],{},[1047,31675,6336],{},[1414,31677,1754],{"stretchy":1753},[2545,31679,31680,31682,31684],{},[1047,31681,13535],{},[1047,31683,1288],{},[1047,31685,1733],{},[1414,31687,1759],{"separator":1062},[2545,31689,31690,31692,31694],{},[1047,31691,18],{},[1047,31693,1288],{},[1047,31695,1733],{},[1414,31697,1768],{"stretchy":1753},[3999,31699,8013],{},[1047,31701,15169],{"mathvariant":1729},[1047,31703,15594],{},[1414,31705,1776],{},[1044,31707,31708,31710],{},[1047,31709,6333],{},[1047,31711,14745],{},[1414,31713,1754],{"stretchy":1753},[2545,31715,31716,31718,31720],{},[1047,31717,18],{},[1047,31719,1288],{},[1047,31721,1733],{},[1047,31723,13675],{"mathvariant":1729},[2545,31725,31726,31728,31730],{},[1047,31727,13535],{},[1047,31729,1288],{},[1047,31731,1733],{},[1414,31733,1768],{"stretchy":1753},[1054,31735,31736],{"encoding":1056},"\\delta_t^V = r_t + \\gamma V_\\theta(s_{t+1}) - V_\\theta(s_t) \\\\\nA_\\theta^{GAE}(s_t, a) = \\sum_{b=0}^{\\infty} (\\gamma\\lambda)^b \\delta_{t+b}^V \\\\\n\\frac{1}{N}\\sum_{n=1}^N\\sum_{t=1}^{T_n} A_\\theta^{GAE}(s_n^t, a_n^t)\\, \\nabla \\log P_\\theta(a_n^t|s_n^t)",[57,31738,31740,31806,31861,31974,32066,32069,32199,32371,32374],{"className":31739,"ariaHidden":1062},[1061],[57,31741,31743,31746,31797,31800,31803],{"className":31742},[1066],[57,31744],{"className":31745,"style":26949},[1070],[57,31747,31749,31752],{"className":31748},[1075],[57,31750,26504],{"className":31751,"style":26528},[1075,1148],[57,31753,31755],{"className":31754},[1152],[57,31756,31758,31789],{"className":31757},[1087,1088],[57,31759,31761,31786],{"className":31760},[1092],[57,31762,31764,31775],{"className":31763,"style":25824},[1096],[57,31765,31766,31769],{"style":26543},[57,31767],{"className":31768,"style":1493},[1104],[57,31770,31772],{"className":31771},[1109,1110,1111,1112],[57,31773,1733],{"className":31774},[1075,1148,1112],[57,31776,31777,31780],{"style":17216},[57,31778],{"className":31779,"style":1493},[1104],[57,31781,31783],{"className":31782},[1109,1110,1111,1112],[57,31784,1625],{"className":31785,"style":1655},[1075,1148,1112],[57,31787,1183],{"className":31788},[1182],[57,31790,31792],{"className":31791},[1092],[57,31793,31795],{"className":31794,"style":18131},[1096],[57,31796],{},[57,31798],{"className":31799,"style":1453},[1452],[57,31801,281],{"className":31802},[1457],[57,31804],{"className":31805,"style":1453},[1452],[57,31807,31809,31812,31852,31855,31858],{"className":31808},[1066],[57,31810],{"className":31811,"style":22799},[1070],[57,31813,31815,31818],{"className":31814},[1075],[57,31816,8500],{"className":31817,"style":3932},[1075,1148],[57,31819,31821],{"className":31820},[1152],[57,31822,31824,31844],{"className":31823},[1087,1088],[57,31825,31827,31841],{"className":31826},[1092],[57,31828,31830],{"className":31829,"style":13464},[1096],[57,31831,31832,31835],{"style":13853},[57,31833],{"className":31834,"style":1493},[1104],[57,31836,31838],{"className":31837},[1109,1110,1111,1112],[57,31839,1733],{"className":31840},[1075,1148,1112],[57,31842,1183],{"className":31843},[1182],[57,31845,31847],{"className":31846},[1092],[57,31848,31850],{"className":31849,"style":2230},[1096],[57,31851],{},[57,31853],{"className":31854,"style":2595},[1452],[57,31856,6487],{"className":31857},[1506],[57,31859],{"className":31860,"style":2595},[1452],[57,31862,31864,31867,31870,31910,31913,31962,31965,31968,31971],{"className":31863},[1066],[57,31865],{"className":31866,"style":1831},[1070],[57,31868,19786],{"className":31869,"style":20033},[1075,1148],[57,31871,31873,31876],{"className":31872},[1075],[57,31874,1625],{"className":31875,"style":1655},[1075,1148],[57,31877,31879],{"className":31878},[1152],[57,31880,31882,31902],{"className":31881},[1087,1088],[57,31883,31885,31899],{"className":31884},[1092],[57,31886,31888],{"className":31887,"style":2208},[1096],[57,31889,31890,31893],{"style":21382},[57,31891],{"className":31892,"style":1493},[1104],[57,31894,31896],{"className":31895},[1109,1110,1111,1112],[57,31897,14745],{"className":31898,"style":3932},[1075,1148,1112],[57,31900,1183],{"className":31901},[1182],[57,31903,31905],{"className":31904},[1092],[57,31906,31908],{"className":31907,"style":2230},[1096],[57,31909],{},[57,31911,1754],{"className":31912},[1079],[57,31914,31916,31919],{"className":31915},[1075],[57,31917,13535],{"className":31918},[1075,1148],[57,31920,31922],{"className":31921},[1152],[57,31923,31925,31954],{"className":31924},[1087,1088],[57,31926,31928,31951],{"className":31927},[1092],[57,31929,31931],{"className":31930,"style":3813},[1096],[57,31932,31933,31936],{"style":2211},[57,31934],{"className":31935,"style":1493},[1104],[57,31937,31939],{"className":31938},[1109,1110,1111,1112],[57,31940,31942,31945,31948],{"className":31941},[1075,1112],[57,31943,1733],{"className":31944},[1075,1148,1112],[57,31946,6487],{"className":31947},[1506,1112],[57,31949,574],{"className":31950},[1075,1112],[57,31952,1183],{"className":31953},[1182],[57,31955,31957],{"className":31956},[1092],[57,31958,31960],{"className":31959,"style":7608},[1096],[57,31961],{},[57,31963,1768],{"className":31964},[1269],[57,31966],{"className":31967,"style":2595},[1452],[57,31969,19155],{"className":31970},[1506],[57,31972],{"className":31973,"style":2595},[1452],[57,31975,31977,31980,32020,32023,32063],{"className":31976},[1066],[57,31978],{"className":31979,"style":1831},[1070],[57,31981,31983,31986],{"className":31982},[1075],[57,31984,1625],{"className":31985,"style":1655},[1075,1148],[57,31987,31989],{"className":31988},[1152],[57,31990,31992,32012],{"className":31991},[1087,1088],[57,31993,31995,32009],{"className":31994},[1092],[57,31996,31998],{"className":31997,"style":2208},[1096],[57,31999,32000,32003],{"style":21382},[57,32001],{"className":32002,"style":1493},[1104],[57,32004,32006],{"className":32005},[1109,1110,1111,1112],[57,32007,14745],{"className":32008,"style":3932},[1075,1148,1112],[57,32010,1183],{"className":32011},[1182],[57,32013,32015],{"className":32014},[1092],[57,32016,32018],{"className":32017,"style":2230},[1096],[57,32019],{},[57,32021,1754],{"className":32022},[1079],[57,32024,32026,32029],{"className":32025},[1075],[57,32027,13535],{"className":32028},[1075,1148],[57,32030,32032],{"className":32031},[1152],[57,32033,32035,32055],{"className":32034},[1087,1088],[57,32036,32038,32052],{"className":32037},[1092],[57,32039,32041],{"className":32040,"style":13464},[1096],[57,32042,32043,32046],{"style":2211},[57,32044],{"className":32045,"style":1493},[1104],[57,32047,32049],{"className":32048},[1109,1110,1111,1112],[57,32050,1733],{"className":32051},[1075,1148,1112],[57,32053,1183],{"className":32054},[1182],[57,32056,32058],{"className":32057},[1092],[57,32059,32061],{"className":32060,"style":2230},[1096],[57,32062],{},[57,32064,1768],{"className":32065},[1269],[57,32067],{"className":32068},[1452,22540],[57,32070,32072,32075,32135,32138,32178,32181,32184,32187,32190,32193,32196],{"className":32071},[1066],[57,32073],{"className":32074,"style":25805},[1070],[57,32076,32078,32081],{"className":32077},[1075],[57,32079,1730],{"className":32080},[1075,1148],[57,32082,32084],{"className":32083},[1152],[57,32085,32087,32127],{"className":32086},[1087,1088],[57,32088,32090,32124],{"className":32089},[1092],[57,32091,32093,32104],{"className":32092,"style":25824},[1096],[57,32094,32095,32098],{"style":18101},[57,32096],{"className":32097,"style":1493},[1104],[57,32099,32101],{"className":32100},[1109,1110,1111,1112],[57,32102,14745],{"className":32103,"style":3932},[1075,1148,1112],[57,32105,32106,32109],{"style":17216},[57,32107],{"className":32108,"style":1493},[1104],[57,32110,32112],{"className":32111},[1109,1110,1111,1112],[57,32113,32115,32118,32121],{"className":32114},[1075,1112],[57,32116,28613],{"className":32117},[1075,1148,1112],[57,32119,1730],{"className":32120},[1075,1148,1112],[57,32122,6336],{"className":32123,"style":6749},[1075,1148,1112],[57,32125,1183],{"className":32126},[1182],[57,32128,32130],{"className":32129},[1092],[57,32131,32133],{"className":32132,"style":18131},[1096],[57,32134],{},[57,32136,1754],{"className":32137},[1079],[57,32139,32141,32144],{"className":32140},[1075],[57,32142,13535],{"className":32143},[1075,1148],[57,32145,32147],{"className":32146},[1152],[57,32148,32150,32170],{"className":32149},[1087,1088],[57,32151,32153,32167],{"className":32152},[1092],[57,32154,32156],{"className":32155,"style":13464},[1096],[57,32157,32158,32161],{"style":2211},[57,32159],{"className":32160,"style":1493},[1104],[57,32162,32164],{"className":32163},[1109,1110,1111,1112],[57,32165,1733],{"className":32166},[1075,1148,1112],[57,32168,1183],{"className":32169},[1182],[57,32171,32173],{"className":32172},[1092],[57,32174,32176],{"className":32175,"style":2230},[1096],[57,32177],{},[57,32179,1759],{"className":32180},[1849],[57,32182],{"className":32183,"style":1853},[1452],[57,32185,18],{"className":32186},[1075,1148],[57,32188,1768],{"className":32189},[1269],[57,32191],{"className":32192,"style":1453},[1452],[57,32194,281],{"className":32195},[1457],[57,32197],{"className":32198,"style":1453},[1452],[57,32200,32202,32206,32273,32276,32279,32282,32311],{"className":32201},[1066],[57,32203],{"className":32204,"style":32205},[1070],"height:2.9535em;vertical-align:-1.3021em;",[57,32207,32209],{"className":32208},[1888,15027],[57,32210,32212,32265],{"className":32211},[1087,1088],[57,32213,32215,32262],{"className":32214},[1092],[57,32216,32218,32238,32248],{"className":32217,"style":31196},[1096],[57,32219,32220,32223],{"style":31199},[57,32221],{"className":32222,"style":16221},[1104],[57,32224,32226],{"className":32225},[1109,1110,1111,1112],[57,32227,32229,32232,32235],{"className":32228},[1075,1112],[57,32230,8028],{"className":32231},[1075,1148,1112],[57,32233,281],{"className":32234},[1457,1112],[57,32236,560],{"className":32237},[1075,1112],[57,32239,32240,32243],{"style":16230},[57,32241],{"className":32242,"style":16221},[1104],[57,32244,32245],{},[57,32246,2549],{"className":32247},[1888,2633,16239],[57,32249,32250,32253],{"style":17164},[57,32251],{"className":32252,"style":16221},[1104],[57,32254,32256],{"className":32255},[1109,1110,1111,1112],[57,32257,32259],{"className":32258},[1075,1112],[57,32260,29146],{"className":32261},[1075,1112],[57,32263,1183],{"className":32264},[1182],[57,32266,32268],{"className":32267},[1092],[57,32269,32271],{"className":32270,"style":31248},[1096],[57,32272],{},[57,32274,1754],{"className":32275},[1079],[57,32277,19786],{"className":32278,"style":20033},[1075,1148],[57,32280,28563],{"className":32281},[1075,1148],[57,32283,32285,32288],{"className":32284},[1269],[57,32286,1768],{"className":32287},[1269],[57,32289,32291],{"className":32290},[1152],[57,32292,32294],{"className":32293},[1087],[57,32295,32297],{"className":32296},[1092],[57,32298,32300],{"className":32299,"style":31278},[1096],[57,32301,32302,32305],{"style":17216},[57,32303],{"className":32304,"style":1493},[1104],[57,32306,32308],{"className":32307},[1109,1110,1111,1112],[57,32309,8028],{"className":32310},[1075,1148,1112],[57,32312,32314,32317],{"className":32313},[1075],[57,32315,26504],{"className":32316,"style":26528},[1075,1148],[57,32318,32320],{"className":32319},[1152],[57,32321,32323,32363],{"className":32322},[1087,1088],[57,32324,32326,32360],{"className":32325},[1092],[57,32327,32329,32349],{"className":32328,"style":25824},[1096],[57,32330,32331,32334],{"style":26543},[57,32332],{"className":32333,"style":1493},[1104],[57,32335,32337],{"className":32336},[1109,1110,1111,1112],[57,32338,32340,32343,32346],{"className":32339},[1075,1112],[57,32341,1733],{"className":32342},[1075,1148,1112],[57,32344,6487],{"className":32345},[1506,1112],[57,32347,8028],{"className":32348},[1075,1148,1112],[57,32350,32351,32354],{"style":17216},[57,32352],{"className":32353,"style":1493},[1104],[57,32355,32357],{"className":32356},[1109,1110,1111,1112],[57,32358,1625],{"className":32359,"style":1655},[1075,1148,1112],[57,32361,1183],{"className":32362},[1182],[57,32364,32366],{"className":32365},[1092],[57,32367,32369],{"className":32368,"style":27353},[1096],[57,32370],{},[57,32372],{"className":32373},[1452,22540],[57,32375,32377,32381,32443,32446,32510,32513,32617,32620,32680,32683,32734,32737,32740,32791,32794,32797,32800,32803,32808,32811,32851,32854,32905,32908,32959],{"className":32376},[1066],[57,32378],{"className":32379,"style":32380},[1070],"height:3.1065em;vertical-align:-1.2671em;",[57,32382,32384,32387,32440],{"className":32383},[1075],[57,32385],{"className":32386},[1079,1080],[57,32388,32390],{"className":32389},[1035],[57,32391,32393,32432],{"className":32392},[1087,1088],[57,32394,32396,32429],{"className":32395},[1092],[57,32397,32399,32410,32418],{"className":32398,"style":17068},[1096],[57,32400,32401,32404],{"style":16608},[57,32402],{"className":32403,"style":1105},[1104],[57,32405,32407],{"className":32406},[1075],[57,32408,7990],{"className":32409,"style":17080},[1075,1148],[57,32411,32412,32415],{"style":1230},[57,32413],{"className":32414,"style":1105},[1104],[57,32416],{"className":32417,"style":1238},[1237],[57,32419,32420,32423],{"style":16674},[57,32421],{"className":32422,"style":1105},[1104],[57,32424,32426],{"className":32425},[1075],[57,32427,574],{"className":32428},[1075],[57,32430,1183],{"className":32431},[1182],[57,32433,32435],{"className":32434},[1092],[57,32436,32438],{"className":32437,"style":17109},[1096],[57,32439],{},[57,32441],{"className":32442},[1269,1080],[57,32444],{"className":32445,"style":1853},[1452],[57,32447,32449],{"className":32448},[1888,15027],[57,32450,32452,32502],{"className":32451},[1087,1088],[57,32453,32455,32499],{"className":32454},[1092],[57,32456,32458,32478,32488],{"className":32457,"style":17130},[1096],[57,32459,32460,32463],{"style":17133},[57,32461],{"className":32462,"style":16221},[1104],[57,32464,32466],{"className":32465},[1109,1110,1111,1112],[57,32467,32469,32472,32475],{"className":32468},[1075,1112],[57,32470,1288],{"className":32471},[1075,1148,1112],[57,32473,281],{"className":32474},[1457,1112],[57,32476,574],{"className":32477},[1075,1112],[57,32479,32480,32483],{"style":16230},[57,32481],{"className":32482,"style":16221},[1104],[57,32484,32485],{},[57,32486,2549],{"className":32487},[1888,2633,16239],[57,32489,32490,32493],{"style":17164},[57,32491],{"className":32492,"style":16221},[1104],[57,32494,32496],{"className":32495},[1109,1110,1111,1112],[57,32497,7990],{"className":32498,"style":17080},[1075,1148,1112],[57,32500,1183],{"className":32501},[1182],[57,32503,32505],{"className":32504},[1092],[57,32506,32508],{"className":32507,"style":17183},[1096],[57,32509],{},[57,32511],{"className":32512,"style":1853},[1452],[57,32514,32516],{"className":32515},[1888,15027],[57,32517,32519,32609],{"className":32518},[1087,1088],[57,32520,32522,32606],{"className":32521},[1092],[57,32523,32525,32545,32555],{"className":32524,"style":17939},[1096],[57,32526,32527,32530],{"style":17133},[57,32528],{"className":32529,"style":16221},[1104],[57,32531,32533],{"className":32532},[1109,1110,1111,1112],[57,32534,32536,32539,32542],{"className":32535},[1075,1112],[57,32537,1733],{"className":32538},[1075,1148,1112],[57,32540,281],{"className":32541},[1457,1112],[57,32543,574],{"className":32544},[1075,1112],[57,32546,32547,32550],{"style":16230},[57,32548],{"className":32549,"style":16221},[1104],[57,32551,32552],{},[57,32553,2549],{"className":32554},[1888,2633,16239],[57,32556,32557,32560],{"style":17972},[57,32558],{"className":32559,"style":16221},[1104],[57,32561,32563],{"className":32562},[1109,1110,1111,1112],[57,32564,32566],{"className":32565},[1075,1112],[57,32567,32569,32572],{"className":32568},[1075,1112],[57,32570,738],{"className":32571,"style":3907},[1075,1148,1112],[57,32573,32575],{"className":32574},[1152],[57,32576,32578,32598],{"className":32577},[1087,1088],[57,32579,32581,32595],{"className":32580},[1092],[57,32582,32584],{"className":32583,"style":5022},[1096],[57,32585,32586,32589],{"style":18002},[57,32587],{"className":32588,"style":1169},[1104],[57,32590,32592],{"className":32591},[1109,1173,1174,1112],[57,32593,1288],{"className":32594},[1075,1148,1112],[57,32596,1183],{"className":32597},[1182],[57,32599,32601],{"className":32600},[1092],[57,32602,32604],{"className":32603,"style":5044},[1096],[57,32605],{},[57,32607,1183],{"className":32608},[1182],[57,32610,32612],{"className":32611},[1092],[57,32613,32615],{"className":32614,"style":17183},[1096],[57,32616],{},[57,32618],{"className":32619,"style":1853},[1452],[57,32621,32623,32626],{"className":32622},[1075],[57,32624,1730],{"className":32625},[1075,1148],[57,32627,32629],{"className":32628},[1152],[57,32630,32632,32672],{"className":32631},[1087,1088],[57,32633,32635,32669],{"className":32634},[1092],[57,32636,32638,32649],{"className":32637,"style":25824},[1096],[57,32639,32640,32643],{"style":18101},[57,32641],{"className":32642,"style":1493},[1104],[57,32644,32646],{"className":32645},[1109,1110,1111,1112],[57,32647,14745],{"className":32648,"style":3932},[1075,1148,1112],[57,32650,32651,32654],{"style":17216},[57,32652],{"className":32653,"style":1493},[1104],[57,32655,32657],{"className":32656},[1109,1110,1111,1112],[57,32658,32660,32663,32666],{"className":32659},[1075,1112],[57,32661,28613],{"className":32662},[1075,1148,1112],[57,32664,1730],{"className":32665},[1075,1148,1112],[57,32667,6336],{"className":32668,"style":6749},[1075,1148,1112],[57,32670,1183],{"className":32671},[1182],[57,32673,32675],{"className":32674},[1092],[57,32676,32678],{"className":32677,"style":18131},[1096],[57,32679],{},[57,32681,1754],{"className":32682},[1079],[57,32684,32686,32689],{"className":32685},[1075],[57,32687,13535],{"className":32688},[1075,1148],[57,32690,32692],{"className":32691},[1152],[57,32693,32695,32726],{"className":32694},[1087,1088],[57,32696,32698,32723],{"className":32697},[1092],[57,32699,32701,32712],{"className":32700,"style":18098},[1096],[57,32702,32703,32706],{"style":18101},[57,32704],{"className":32705,"style":1493},[1104],[57,32707,32709],{"className":32708},[1109,1110,1111,1112],[57,32710,1288],{"className":32711},[1075,1148,1112],[57,32713,32714,32717],{"style":17216},[57,32715],{"className":32716,"style":1493},[1104],[57,32718,32720],{"className":32719},[1109,1110,1111,1112],[57,32721,1733],{"className":32722},[1075,1148,1112],[57,32724,1183],{"className":32725},[1182],[57,32727,32729],{"className":32728},[1092],[57,32730,32732],{"className":32731,"style":18131},[1096],[57,32733],{},[57,32735,1759],{"className":32736},[1849],[57,32738],{"className":32739,"style":1853},[1452],[57,32741,32743,32746],{"className":32742},[1075],[57,32744,18],{"className":32745},[1075,1148],[57,32747,32749],{"className":32748},[1152],[57,32750,32752,32783],{"className":32751},[1087,1088],[57,32753,32755,32780],{"className":32754},[1092],[57,32756,32758,32769],{"className":32757,"style":18098},[1096],[57,32759,32760,32763],{"style":18101},[57,32761],{"className":32762,"style":1493},[1104],[57,32764,32766],{"className":32765},[1109,1110,1111,1112],[57,32767,1288],{"className":32768},[1075,1148,1112],[57,32770,32771,32774],{"style":17216},[57,32772],{"className":32773,"style":1493},[1104],[57,32775,32777],{"className":32776},[1109,1110,1111,1112],[57,32778,1733],{"className":32779},[1075,1148,1112],[57,32781,1183],{"className":32782},[1182],[57,32784,32786],{"className":32785},[1092],[57,32787,32789],{"className":32788,"style":18131},[1096],[57,32790],{},[57,32792,1768],{"className":32793},[1269],[57,32795],{"className":32796,"style":1853},[1452],[57,32798,15169],{"className":32799},[1075],[57,32801],{"className":32802,"style":1853},[1452],[57,32804,17640,32806],{"className":32805},[1888],[57,32807,17644],{"style":17643},[57,32809],{"className":32810,"style":1853},[1452],[57,32812,32814,32817],{"className":32813},[1075],[57,32815,6333],{"className":32816,"style":3907},[1075,1148],[57,32818,32820],{"className":32819},[1152],[57,32821,32823,32843],{"className":32822},[1087,1088],[57,32824,32826,32840],{"className":32825},[1092],[57,32827,32829],{"className":32828,"style":2208},[1096],[57,32830,32831,32834],{"style":8131},[57,32832],{"className":32833,"style":1493},[1104],[57,32835,32837],{"className":32836},[1109,1110,1111,1112],[57,32838,14745],{"className":32839,"style":3932},[1075,1148,1112],[57,32841,1183],{"className":32842},[1182],[57,32844,32846],{"className":32845},[1092],[57,32847,32849],{"className":32848,"style":2230},[1096],[57,32850],{},[57,32852,1754],{"className":32853},[1079],[57,32855,32857,32860],{"className":32856},[1075],[57,32858,18],{"className":32859},[1075,1148],[57,32861,32863],{"className":32862},[1152],[57,32864,32866,32897],{"className":32865},[1087,1088],[57,32867,32869,32894],{"className":32868},[1092],[57,32870,32872,32883],{"className":32871,"style":18098},[1096],[57,32873,32874,32877],{"style":18101},[57,32875],{"className":32876,"style":1493},[1104],[57,32878,32880],{"className":32879},[1109,1110,1111,1112],[57,32881,1288],{"className":32882},[1075,1148,1112],[57,32884,32885,32888],{"style":17216},[57,32886],{"className":32887,"style":1493},[1104],[57,32889,32891],{"className":32890},[1109,1110,1111,1112],[57,32892,1733],{"className":32893},[1075,1148,1112],[57,32895,1183],{"className":32896},[1182],[57,32898,32900],{"className":32899},[1092],[57,32901,32903],{"className":32902,"style":18131},[1096],[57,32904],{},[57,32906,13675],{"className":32907},[1075],[57,32909,32911,32914],{"className":32910},[1075],[57,32912,13535],{"className":32913},[1075,1148],[57,32915,32917],{"className":32916},[1152],[57,32918,32920,32951],{"className":32919},[1087,1088],[57,32921,32923,32948],{"className":32922},[1092],[57,32924,32926,32937],{"className":32925,"style":18098},[1096],[57,32927,32928,32931],{"style":18101},[57,32929],{"className":32930,"style":1493},[1104],[57,32932,32934],{"className":32933},[1109,1110,1111,1112],[57,32935,1288],{"className":32936},[1075,1148,1112],[57,32938,32939,32942],{"style":17216},[57,32940],{"className":32941,"style":1493},[1104],[57,32943,32945],{"className":32944},[1109,1110,1111,1112],[57,32946,1733],{"className":32947},[1075,1148,1112],[57,32949,1183],{"className":32950},[1182],[57,32952,32954],{"className":32953},[1092],[57,32955,32957],{"className":32956,"style":18131},[1096],[57,32958],{},[57,32960,1768],{"className":32961},[1269],[14,32963,32964,32965,33034],{},"这里的状态价值函数 ",[57,32966,32968,32985],{"className":32967},[1019],[57,32969,32971],{"className":32970},[1023],[1025,32972,32973],{"xmlns":1027},[1029,32974,32975,32983],{},[1032,32976,32977],{},[1044,32978,32979,32981],{},[1047,32980,1625],{},[1047,32982,14745],{},[1054,32984,23730],{"encoding":1056},[57,32986,32988],{"className":32987,"ariaHidden":1062},[1061],[57,32989,32991,32994],{"className":32990},[1066],[57,32992],{"className":32993,"style":7644},[1070],[57,32995,32997,33000],{"className":32996},[1075],[57,32998,1625],{"className":32999,"style":1655},[1075,1148],[57,33001,33003],{"className":33002},[1152],[57,33004,33006,33026],{"className":33005},[1087,1088],[57,33007,33009,33023],{"className":33008},[1092],[57,33010,33012],{"className":33011,"style":2208},[1096],[57,33013,33014,33017],{"style":21382},[57,33015],{"className":33016,"style":1493},[1104],[57,33018,33020],{"className":33019},[1109,1110,1111,1112],[57,33021,14745],{"className":33022,"style":3932},[1075,1148,1112],[57,33024,1183],{"className":33025},[1182],[57,33027,33029],{"className":33028},[1092],[57,33030,33032],{"className":33031,"style":2230},[1096],[57,33033],{}," 一般用一个神经网络来拟合（即 critic 网络），可以与策略网络共用主干参数，仅在最后一层分叉为两个 head。",[10,33036,33038],{"id":33037},"proximal-policy-optimization-ppo-近端策略优化算法","Proximal Policy Optimization (PPO) 近端策略优化算法",[14,33040,33041,33042,33045,33046,33115,33116,21450,33119,33225,33226,33229,33230,222],{},"在经典的强化学习训练范式里，我们通常一边采集数据、一边更新模型，采过的数据用一次就丢掉——这种做法被称为 ",[11865,33043,33044],{},"on-policy","。问题在于，强化学习的环境交互成本往往很高，这样\"用一次就扔\"显然非常浪费。如果我们能让当前策略 ",[57,33047,33049,33066],{"className":33048},[1019],[57,33050,33052],{"className":33051},[1023],[1025,33053,33054],{"xmlns":1027},[1029,33055,33056,33064],{},[1032,33057,33058],{},[1044,33059,33060,33062],{},[1047,33061,7412],{},[1047,33063,14745],{},[1054,33065,14748],{"encoding":1056},[57,33067,33069],{"className":33068,"ariaHidden":1062},[1061],[57,33070,33072,33075],{"className":33071},[1066],[57,33073],{"className":33074,"style":13445},[1070],[57,33076,33078,33081],{"className":33077},[1075],[57,33079,7412],{"className":33080,"style":1394},[1075,1148],[57,33082,33084],{"className":33083},[1152],[57,33085,33087,33107],{"className":33086},[1087,1088],[57,33088,33090,33104],{"className":33089},[1092],[57,33091,33093],{"className":33092,"style":2208},[1096],[57,33094,33095,33098],{"style":2758},[57,33096],{"className":33097,"style":1493},[1104],[57,33099,33101],{"className":33100},[1109,1110,1111,1112],[57,33102,14745],{"className":33103,"style":3932},[1075,1148,1112],[57,33105,1183],{"className":33106},[1182],[57,33108,33110],{"className":33109},[1092],[57,33111,33113],{"className":33112,"style":2230},[1096],[57,33114],{}," 复用 ",[11865,33117,33118],{},"旧策略",[57,33120,33122,33144],{"className":33121},[1019],[57,33123,33125],{"className":33124},[1023],[1025,33126,33127],{"xmlns":1027},[1029,33128,33129,33141],{},[1032,33130,33131],{},[1044,33132,33133,33135],{},[1047,33134,7412],{},[1418,33136,33137,33139],{},[1047,33138,14745],{},[1414,33140,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1054,33142,33143],{"encoding":1056},"\\pi_{\\theta'}",[57,33145,33147],{"className":33146,"ariaHidden":1062},[1061],[57,33148,33150,33153],{"className":33149},[1066],[57,33151],{"className":33152,"style":13445},[1070],[57,33154,33156,33159],{"className":33155},[1075],[57,33157,7412],{"className":33158,"style":1394},[1075,1148],[57,33160,33162],{"className":33161},[1152],[57,33163,33165,33217],{"className":33164},[1087,1088],[57,33166,33168,33214],{"className":33167},[1092],[57,33169,33171],{"className":33170,"style":2208},[1096],[57,33172,33173,33176],{"style":2758},[57,33174],{"className":33175,"style":1493},[1104],[57,33177,33179],{"className":33178},[1109,1110,1111,1112],[57,33180,33182],{"className":33181},[1075,1112],[57,33183,33185,33188],{"className":33184},[1075,1112],[57,33186,14745],{"className":33187,"style":3932},[1075,1148,1112],[57,33189,33191],{"className":33190},[1152],[57,33192,33194],{"className":33193},[1087],[57,33195,33197],{"className":33196},[1092],[57,33198,33200],{"className":33199,"style":19935},[1096],[57,33201,33202,33205],{"style":19938},[57,33203],{"className":33204,"style":1169},[1104],[57,33206,33208],{"className":33207},[1109,1173,1174,1112],[57,33209,33211],{"className":33210},[1075,1112],[57,33212,19775],{"className":33213},[1075,1112],[57,33215,1183],{"className":33216},[1182],[57,33218,33220],{"className":33219},[1092],[57,33221,33223],{"className":33222,"style":2230},[1096],[57,33224],{}," 采集的数据进行训练（即 ",[11865,33227,33228],{},"off-policy","），训练效率就能显著提升。要实现这一点，关键就是 ",[11865,33231,33232],{},"重要性采样（Importance Sampling）",[248,33234,33235],{"id":33235},"重要性采样",[14,33237,33238,33239,33283,33284,33312,33313,33486,33487,33515,33516,33544,33545,33573],{},"我们可以把\"",[57,33240,33242,33262],{"className":33241},[1019],[57,33243,33245],{"className":33244},[1023],[1025,33246,33247],{"xmlns":1027},[1029,33248,33249,33259],{},[1032,33250,33251,33253,33255,33257],{},[1047,33252,12389],{},[1414,33254,1754],{"stretchy":1753},[1047,33256,7995],{},[1414,33258,1768],{"stretchy":1753},[1054,33260,33261],{"encoding":1056},"f(x)",[57,33263,33265],{"className":33264,"ariaHidden":1062},[1061],[57,33266,33268,33271,33274,33277,33280],{"className":33267},[1066],[57,33269],{"className":33270,"style":1831},[1070],[57,33272,12389],{"className":33273,"style":14380},[1075,1148],[57,33275,1754],{"className":33276},[1079],[57,33278,7995],{"className":33279},[1075,1148],[57,33281,1768],{"className":33282},[1269]," 在分布 ",[57,33285,33287,33300],{"className":33286},[1019],[57,33288,33290],{"className":33289},[1023],[1025,33291,33292],{"xmlns":1027},[1029,33293,33294,33298],{},[1032,33295,33296],{},[1047,33297,14],{},[1054,33299,14],{"encoding":1056},[57,33301,33303],{"className":33302,"ariaHidden":1062},[1061],[57,33304,33306,33309],{"className":33305},[1066],[57,33307],{"className":33308,"style":2430},[1070],[57,33310,14],{"className":33311},[1075,1148]," 下的期望\"改写为\"",[57,33314,33316,33360],{"className":33315},[1019],[57,33317,33319],{"className":33318},[1023],[1025,33320,33321],{"xmlns":1027},[1029,33322,33323,33357],{},[1032,33324,33325,33327,33329,33331,33333,33335],{},[1047,33326,12389],{},[1414,33328,1754],{"stretchy":1753},[1047,33330,7995],{},[1414,33332,1768],{"stretchy":1753},[1414,33334,2539],{},[1035,33336,33337,33347],{},[1032,33338,33339,33341,33343,33345],{},[1047,33340,14],{},[1414,33342,1754],{"stretchy":1753},[1047,33344,7995],{},[1414,33346,1768],{"stretchy":1753},[1032,33348,33349,33351,33353,33355],{},[1047,33350,2418],{},[1414,33352,1754],{"stretchy":1753},[1047,33354,7995],{},[1414,33356,1768],{"stretchy":1753},[1054,33358,33359],{"encoding":1056},"f(x) \\cdot \\frac{p(x)}{q(x)}",[57,33361,33363,33390],{"className":33362,"ariaHidden":1062},[1061],[57,33364,33366,33369,33372,33375,33378,33381,33384,33387],{"className":33365},[1066],[57,33367],{"className":33368,"style":1831},[1070],[57,33370,12389],{"className":33371,"style":14380},[1075,1148],[57,33373,1754],{"className":33374},[1079],[57,33376,7995],{"className":33377},[1075,1148],[57,33379,1768],{"className":33380},[1269],[57,33382],{"className":33383,"style":2595},[1452],[57,33385,2539],{"className":33386},[1506],[57,33388],{"className":33389,"style":2595},[1452],[57,33391,33393,33397],{"className":33392},[1066],[57,33394],{"className":33395,"style":33396},[1070],"height:1.53em;vertical-align:-0.52em;",[57,33398,33400,33403,33483],{"className":33399},[1075],[57,33401],{"className":33402},[1079,1080],[57,33404,33406],{"className":33405},[1035],[57,33407,33409,33474],{"className":33408},[1087,1088],[57,33410,33412,33471],{"className":33411},[1092],[57,33413,33416,33439,33447],{"className":33414,"style":33415},[1096],"height:1.01em;",[57,33417,33418,33421],{"style":19297},[57,33419],{"className":33420,"style":1105},[1104],[57,33422,33424],{"className":33423},[1109,1110,1111,1112],[57,33425,33427,33430,33433,33436],{"className":33426},[1075,1112],[57,33428,2418],{"className":33429,"style":1394},[1075,1148,1112],[57,33431,1754],{"className":33432},[1079,1112],[57,33434,7995],{"className":33435},[1075,1148,1112],[57,33437,1768],{"className":33438},[1269,1112],[57,33440,33441,33444],{"style":1230},[57,33442],{"className":33443,"style":1105},[1104],[57,33445],{"className":33446,"style":1238},[1237],[57,33448,33450,33453],{"style":33449},"top:-3.485em;",[57,33451],{"className":33452,"style":1105},[1104],[57,33454,33456],{"className":33455},[1109,1110,1111,1112],[57,33457,33459,33462,33465,33468],{"className":33458},[1075,1112],[57,33460,14],{"className":33461},[1075,1148,1112],[57,33463,1754],{"className":33464},[1079,1112],[57,33466,7995],{"className":33467},[1075,1148,1112],[57,33469,1768],{"className":33470},[1269,1112],[57,33472,1183],{"className":33473},[1182],[57,33475,33477],{"className":33476},[1092],[57,33478,33481],{"className":33479,"style":33480},[1096],"height:0.52em;",[57,33482],{},[57,33484],{"className":33485},[1269,1080]," 在另一分布 ",[57,33488,33490,33503],{"className":33489},[1019],[57,33491,33493],{"className":33492},[1023],[1025,33494,33495],{"xmlns":1027},[1029,33496,33497,33501],{},[1032,33498,33499],{},[1047,33500,2418],{},[1054,33502,2418],{"encoding":1056},[57,33504,33506],{"className":33505,"ariaHidden":1062},[1061],[57,33507,33509,33512],{"className":33508},[1066],[57,33510],{"className":33511,"style":2430},[1070],[57,33513,2418],{"className":33514,"style":1394},[1075,1148],"（proposal distribution）下的期望\"，这样就能用从 ",[57,33517,33519,33532],{"className":33518},[1019],[57,33520,33522],{"className":33521},[1023],[1025,33523,33524],{"xmlns":1027},[1029,33525,33526,33530],{},[1032,33527,33528],{},[1047,33529,2418],{},[1054,33531,2418],{"encoding":1056},[57,33533,33535],{"className":33534,"ariaHidden":1062},[1061],[57,33536,33538,33541],{"className":33537},[1066],[57,33539],{"className":33540,"style":2430},[1070],[57,33542,2418],{"className":33543,"style":1394},[1075,1148]," 采样的数据来估计原本在 ",[57,33546,33548,33561],{"className":33547},[1019],[57,33549,33551],{"className":33550},[1023],[1025,33552,33553],{"xmlns":1027},[1029,33554,33555,33559],{},[1032,33556,33557],{},[1047,33558,14],{},[1054,33560,14],{"encoding":1056},[57,33562,33564],{"className":33563,"ariaHidden":1062},[1061],[57,33565,33567,33570],{"className":33566},[1066],[57,33568],{"className":33569,"style":2430},[1070],[57,33571,14],{"className":33572},[1075,1148]," 下的期望：",[57,33575,33577],{"className":33576},[15139],[57,33578,33580,33964],{"className":33579},[1019],[57,33581,33583],{"className":33582},[1023],[1025,33584,33585],{"xmlns":1027,"display":10210},[1029,33586,33587,33961],{},[15151,33588,33589,33661,33725,33789,33861],{"rowspacing":15153,"columnalign":15154,"columnspacing":15155},[15157,33590,33591,33627],{},[15160,33592,33593],{},[15163,33594,33595],{"scriptlevel":560,"displaystyle":1062},[1032,33596,33597,33599,33601,33603,33605,33607,33609],{},[1047,33598,6336],{"mathvariant":1422},[1414,33600,1754],{"stretchy":1753},[1047,33602,12389],{},[1414,33604,1754],{"stretchy":1753},[1047,33606,7995],{},[1414,33608,1768],{"stretchy":1753},[1044,33610,33611,33613],{},[1414,33612,1768],{"stretchy":1753},[1032,33614,33615,33617,33619,33621,33623,33625],{},[1047,33616,7995],{},[1414,33618,14500],{},[1047,33620,14],{},[1414,33622,1754],{"stretchy":1753},[1047,33624,7995],{},[1414,33626,1768],{"stretchy":1753},[15160,33628,33629],{},[15163,33630,33631],{"scriptlevel":560,"displaystyle":1062},[1032,33632,33633,33635,33637,33643,33645,33647,33649,33651,33653,33655,33657,33659],{},[1032,33634],{},[1414,33636,281],{},[14850,33638,33639,33641],{},[1414,33640,2549],{},[1047,33642,7995],{},[1047,33644,12389],{},[1414,33646,1754],{"stretchy":1753},[1047,33648,7995],{},[1414,33650,1768],{"stretchy":1753},[1414,33652,2539],{},[1047,33654,14],{},[1414,33656,1754],{"stretchy":1753},[1047,33658,7995],{},[1414,33660,1768],{"stretchy":1753},[15157,33662,33663,33669],{},[15160,33664,33665],{},[15163,33666,33667],{"scriptlevel":560,"displaystyle":1062},[1032,33668],{},[15160,33670,33671],{},[15163,33672,33673],{"scriptlevel":560,"displaystyle":1062},[1032,33674,33675,33677,33679,33685,33687,33689,33691,33693,33695,33697,33699,33701,33703],{},[1032,33676],{},[1414,33678,281],{},[14850,33680,33681,33683],{},[1414,33682,2549],{},[1047,33684,7995],{},[1047,33686,12389],{},[1414,33688,1754],{"stretchy":1753},[1047,33690,7995],{},[1414,33692,1768],{"stretchy":1753},[1414,33694,2539],{},[1047,33696,14],{},[1414,33698,1754],{"stretchy":1753},[1047,33700,7995],{},[1414,33702,1768],{"stretchy":1753},[1035,33704,33705,33715],{},[1032,33706,33707,33709,33711,33713],{},[1047,33708,2418],{},[1414,33710,1754],{"stretchy":1753},[1047,33712,7995],{},[1414,33714,1768],{"stretchy":1753},[1032,33716,33717,33719,33721,33723],{},[1047,33718,2418],{},[1414,33720,1754],{"stretchy":1753},[1047,33722,7995],{},[1414,33724,1768],{"stretchy":1753},[15157,33726,33727,33733],{},[15160,33728,33729],{},[15163,33730,33731],{"scriptlevel":560,"displaystyle":1062},[1032,33732],{},[15160,33734,33735],{},[15163,33736,33737],{"scriptlevel":560,"displaystyle":1062},[1032,33738,33739,33741,33743,33749,33751,33753,33755,33757,33779,33781,33783,33785,33787],{},[1032,33740],{},[1414,33742,281],{},[14850,33744,33745,33747],{},[1414,33746,2549],{},[1047,33748,7995],{},[1047,33750,12389],{},[1414,33752,1754],{"stretchy":1753},[1047,33754,7995],{},[1414,33756,1768],{"stretchy":1753},[1035,33758,33759,33769],{},[1032,33760,33761,33763,33765,33767],{},[1047,33762,14],{},[1414,33764,1754],{"stretchy":1753},[1047,33766,7995],{},[1414,33768,1768],{"stretchy":1753},[1032,33770,33771,33773,33775,33777],{},[1047,33772,2418],{},[1414,33774,1754],{"stretchy":1753},[1047,33776,7995],{},[1414,33778,1768],{"stretchy":1753},[1414,33780,2539],{},[1047,33782,2418],{},[1414,33784,1754],{"stretchy":1753},[1047,33786,7995],{},[1414,33788,1768],{"stretchy":1753},[15157,33790,33791,33797],{},[15160,33792,33793],{},[15163,33794,33795],{"scriptlevel":560,"displaystyle":1062},[1032,33796],{},[15160,33798,33799],{},[15163,33800,33801],{"scriptlevel":560,"displaystyle":1062},[1032,33802,33803,33805,33807,33809],{},[1032,33804],{},[1414,33806,281],{},[1047,33808,6336],{"mathvariant":1422},[1044,33810,33811,33847],{},[1032,33812,33813,33815,33817,33819,33821,33823,33845],{},[1414,33814,1754],{"fence":1062},[1047,33816,12389],{},[1414,33818,1754],{"stretchy":1753},[1047,33820,7995],{},[1414,33822,1768],{"stretchy":1753},[1035,33824,33825,33835],{},[1032,33826,33827,33829,33831,33833],{},[1047,33828,14],{},[1414,33830,1754],{"stretchy":1753},[1047,33832,7995],{},[1414,33834,1768],{"stretchy":1753},[1032,33836,33837,33839,33841,33843],{},[1047,33838,2418],{},[1414,33840,1754],{"stretchy":1753},[1047,33842,7995],{},[1414,33844,1768],{"stretchy":1753},[1414,33846,1768],{"fence":1062},[1032,33848,33849,33851,33853,33855,33857,33859],{},[1047,33850,7995],{},[1414,33852,14500],{},[1047,33854,2418],{},[1414,33856,1754],{"stretchy":1753},[1047,33858,7995],{},[1414,33860,1768],{"stretchy":1753},[15157,33862,33863,33869],{},[15160,33864,33865],{},[15163,33866,33867],{"scriptlevel":560,"displaystyle":1062},[1032,33868],{},[15160,33870,33871],{},[15163,33872,33873],{"scriptlevel":560,"displaystyle":1062},[1032,33874,33875,33877,33879,33885,33899,33901,33903,33909,33911,33941,33943,33945,33951,33953,33955,33957,33959],{},[1032,33876],{},[1414,33878,15464],{},[1035,33880,33881,33883],{},[1038,33882,574],{},[1047,33884,7990],{},[15472,33886,33887,33889,33897],{},[1414,33888,2549],{},[1032,33890,33891,33893,33895],{},[1047,33892,1288],{},[1414,33894,281],{},[1038,33896,574],{},[1047,33898,7990],{},[1047,33900,12389],{},[1414,33902,1754],{"stretchy":1753},[1044,33904,33905,33907],{},[1047,33906,7995],{},[1047,33908,1288],{},[1414,33910,1768],{"stretchy":1753},[1035,33912,33913,33927],{},[1032,33914,33915,33917,33919,33925],{},[1047,33916,14],{},[1414,33918,1754],{"stretchy":1753},[1044,33920,33921,33923],{},[1047,33922,7995],{},[1047,33924,1288],{},[1414,33926,1768],{"stretchy":1753},[1032,33928,33929,33931,33933,33939],{},[1047,33930,2418],{},[1414,33932,1754],{"stretchy":1753},[1044,33934,33935,33937],{},[1047,33936,7995],{},[1047,33938,1288],{},[1414,33940,1768],{"stretchy":1753},[1414,33942,1759],{"separator":1062},[1452,33944],{"width":28702},[1044,33946,33947,33949],{},[1047,33948,7995],{},[1047,33950,1288],{},[1414,33952,14500],{},[1047,33954,2418],{},[1414,33956,1754],{"stretchy":1753},[1047,33958,7995],{},[1414,33960,1768],{"stretchy":1753},[1054,33962,33963],{"encoding":1056},"\\begin{aligned}\n\\mathbb{E}(f(x))_{x \\sim p(x)} &= \\sum_{x} f(x) \\cdot p(x) \\\\\n&= \\sum_{x} f(x) \\cdot p(x) \\frac{q(x)}{q(x)} \\\\\n&= \\sum_{x} f(x) \\frac{p(x)}{q(x)} \\cdot q(x) \\\\\n&= \\mathbb{E}\\left(f(x) \\frac{p(x)}{q(x)}\\right)_{x \\sim q(x)} \\\\\n&\\approx \\frac{1}{N} \\sum_{n=1}^{N} f(x_n) \\frac{p(x_n)}{q(x_n)}, \\quad x_n \\sim q(x)\n\\end{aligned}",[57,33965,33967],{"className":33966,"ariaHidden":1062},[1061],[57,33968,33970,33974],{"className":33969},[1066],[57,33971],{"className":33972,"style":33973},[1070],"height:14.8742em;vertical-align:-7.1871em;",[57,33975,33977],{"className":33976},[1075],[57,33978,33980,34127],{"className":33979},[15151],[57,33981,33983],{"className":33982},[15942],[57,33984,33986,34118],{"className":33985},[1087,1088],[57,33987,33989,34115],{"className":33988},[1092],[57,33990,33993,34079,34088,34097,34106],{"className":33991,"style":33992},[1096],"height:7.6871em;",[57,33994,33996,34000],{"style":33995},"top:-10.4654em;",[57,33997],{"className":33998,"style":33999},[1104],"height:3.8283em;",[57,34001,34003,34006,34009,34012,34015,34018,34021],{"className":34002},[1075],[57,34004,6336],{"className":34005},[1075,1474],[57,34007,1754],{"className":34008},[1079],[57,34010,12389],{"className":34011,"style":14380},[1075,1148],[57,34013,1754],{"className":34014},[1079],[57,34016,7995],{"className":34017},[1075,1148],[57,34019,1768],{"className":34020},[1269],[57,34022,34024,34027],{"className":34023},[1269],[57,34025,1768],{"className":34026},[1269],[57,34028,34030],{"className":34029},[1152],[57,34031,34033,34071],{"className":34032},[1087,1088],[57,34034,34036,34068],{"className":34035},[1092],[57,34037,34039],{"className":34038,"style":1162},[1096],[57,34040,34041,34044],{"style":14928},[57,34042],{"className":34043,"style":1493},[1104],[57,34045,34047],{"className":34046},[1109,1110,1111,1112],[57,34048,34050,34053,34056,34059,34062,34065],{"className":34049},[1075,1112],[57,34051,7995],{"className":34052},[1075,1148,1112],[57,34054,14500],{"className":34055},[1457,1112],[57,34057,14],{"className":34058},[1075,1148,1112],[57,34060,1754],{"className":34061},[1079,1112],[57,34063,7995],{"className":34064},[1075,1148,1112],[57,34066,1768],{"className":34067},[1269,1112],[57,34069,1183],{"className":34070},[1182],[57,34072,34074],{"className":34073},[1092],[57,34075,34077],{"className":34076,"style":6801},[1096],[57,34078],{},[57,34080,34082,34085],{"style":34081},"top:-7.4884em;",[57,34083],{"className":34084,"style":33999},[1104],[57,34086],{"className":34087},[1075],[57,34089,34091,34094],{"style":34090},"top:-4.5114em;",[57,34092],{"className":34093,"style":33999},[1104],[57,34095],{"className":34096},[1075],[57,34098,34100,34103],{"style":34099},"top:-1.5114em;",[57,34101],{"className":34102,"style":33999},[1104],[57,34104],{"className":34105},[1075],[57,34107,34109,34112],{"style":34108},"top:1.7917em;",[57,34110],{"className":34111,"style":33999},[1104],[57,34113],{"className":34114},[1075],[57,34116,1183],{"className":34117},[1182],[57,34119,34121],{"className":34120},[1092],[57,34122,34125],{"className":34123,"style":34124},[1096],"height:7.1871em;",[57,34126],{},[57,34128,34130],{"className":34129},[16166],[57,34131,34133,35230],{"className":34132},[1087,1088],[57,34134,34136,35227],{"className":34135},[1092],[57,34137,34139,34242,34425,34608,34799],{"className":34138,"style":33992},[1096],[57,34140,34141,34144],{"style":33995},[57,34142],{"className":34143,"style":33999},[1104],[57,34145,34147,34150,34153,34156,34159,34206,34209,34212,34215,34218,34221,34224,34227,34230,34233,34236,34239],{"className":34146},[1075],[57,34148],{"className":34149},[1075],[57,34151],{"className":34152,"style":1453},[1452],[57,34154,281],{"className":34155},[1457],[57,34157],{"className":34158,"style":1453},[1452],[57,34160,34162],{"className":34161},[1888,15027],[57,34163,34165,34198],{"className":34164},[1087,1088],[57,34166,34168,34195],{"className":34167},[1092],[57,34169,34171,34185],{"className":34170,"style":16214},[1096],[57,34172,34173,34176],{"style":16217},[57,34174],{"className":34175,"style":16221},[1104],[57,34177,34179],{"className":34178},[1109,1110,1111,1112],[57,34180,34182],{"className":34181},[1075,1112],[57,34183,7995],{"className":34184},[1075,1148,1112],[57,34186,34187,34190],{"style":16230},[57,34188],{"className":34189,"style":16221},[1104],[57,34191,34192],{},[57,34193,2549],{"className":34194},[1888,2633,16239],[57,34196,1183],{"className":34197},[1182],[57,34199,34201],{"className":34200},[1092],[57,34202,34204],{"className":34203,"style":16249},[1096],[57,34205],{},[57,34207],{"className":34208,"style":1853},[1452],[57,34210,12389],{"className":34211,"style":14380},[1075,1148],[57,34213,1754],{"className":34214},[1079],[57,34216,7995],{"className":34217},[1075,1148],[57,34219,1768],{"className":34220},[1269],[57,34222],{"className":34223,"style":2595},[1452],[57,34225,2539],{"className":34226},[1506],[57,34228],{"className":34229,"style":2595},[1452],[57,34231,14],{"className":34232},[1075,1148],[57,34234,1754],{"className":34235},[1079],[57,34237,7995],{"className":34238},[1075,1148],[57,34240,1768],{"className":34241},[1269],[57,34243,34244,34247],{"style":34081},[57,34245],{"className":34246,"style":33999},[1104],[57,34248,34250,34253,34256,34259,34262,34309,34312,34315,34318,34321,34324,34327,34330,34333,34336,34339,34342,34345],{"className":34249},[1075],[57,34251],{"className":34252},[1075],[57,34254],{"className":34255,"style":1453},[1452],[57,34257,281],{"className":34258},[1457],[57,34260],{"className":34261,"style":1453},[1452],[57,34263,34265],{"className":34264},[1888,15027],[57,34266,34268,34301],{"className":34267},[1087,1088],[57,34269,34271,34298],{"className":34270},[1092],[57,34272,34274,34288],{"className":34273,"style":16214},[1096],[57,34275,34276,34279],{"style":16217},[57,34277],{"className":34278,"style":16221},[1104],[57,34280,34282],{"className":34281},[1109,1110,1111,1112],[57,34283,34285],{"className":34284},[1075,1112],[57,34286,7995],{"className":34287},[1075,1148,1112],[57,34289,34290,34293],{"style":16230},[57,34291],{"className":34292,"style":16221},[1104],[57,34294,34295],{},[57,34296,2549],{"className":34297},[1888,2633,16239],[57,34299,1183],{"className":34300},[1182],[57,34302,34304],{"className":34303},[1092],[57,34305,34307],{"className":34306,"style":16249},[1096],[57,34308],{},[57,34310],{"className":34311,"style":1853},[1452],[57,34313,12389],{"className":34314,"style":14380},[1075,1148],[57,34316,1754],{"className":34317},[1079],[57,34319,7995],{"className":34320},[1075,1148],[57,34322,1768],{"className":34323},[1269],[57,34325],{"className":34326,"style":2595},[1452],[57,34328,2539],{"className":34329},[1506],[57,34331],{"className":34332,"style":2595},[1452],[57,34334,14],{"className":34335},[1075,1148],[57,34337,1754],{"className":34338},[1079],[57,34340,7995],{"className":34341},[1075,1148],[57,34343,1768],{"className":34344},[1269],[57,34346,34348,34351,34422],{"className":34347},[1075],[57,34349],{"className":34350},[1079,1080],[57,34352,34354],{"className":34353},[1035],[57,34355,34357,34414],{"className":34356},[1087,1088],[57,34358,34360,34411],{"className":34359},[1092],[57,34361,34363,34383,34391],{"className":34362,"style":16605},[1096],[57,34364,34365,34368],{"style":16608},[57,34366],{"className":34367,"style":1105},[1104],[57,34369,34371,34374,34377,34380],{"className":34370},[1075],[57,34372,2418],{"className":34373,"style":1394},[1075,1148],[57,34375,1754],{"className":34376},[1079],[57,34378,7995],{"className":34379},[1075,1148],[57,34381,1768],{"className":34382},[1269],[57,34384,34385,34388],{"style":1230},[57,34386],{"className":34387,"style":1105},[1104],[57,34389],{"className":34390,"style":1238},[1237],[57,34392,34393,34396],{"style":16674},[57,34394],{"className":34395,"style":1105},[1104],[57,34397,34399,34402,34405,34408],{"className":34398},[1075],[57,34400,2418],{"className":34401,"style":1394},[1075,1148],[57,34403,1754],{"className":34404},[1079],[57,34406,7995],{"className":34407},[1075,1148],[57,34409,1768],{"className":34410},[1269],[57,34412,1183],{"className":34413},[1182],[57,34415,34417],{"className":34416},[1092],[57,34418,34420],{"className":34419,"style":16739},[1096],[57,34421],{},[57,34423],{"className":34424},[1269,1080],[57,34426,34427,34430],{"style":34090},[57,34428],{"className":34429,"style":33999},[1104],[57,34431,34433,34436,34439,34442,34445,34492,34495,34498,34501,34504,34507,34587,34590,34593,34596,34599,34602,34605],{"className":34432},[1075],[57,34434],{"className":34435},[1075],[57,34437],{"className":34438,"style":1453},[1452],[57,34440,281],{"className":34441},[1457],[57,34443],{"className":34444,"style":1453},[1452],[57,34446,34448],{"className":34447},[1888,15027],[57,34449,34451,34484],{"className":34450},[1087,1088],[57,34452,34454,34481],{"className":34453},[1092],[57,34455,34457,34471],{"className":34456,"style":16214},[1096],[57,34458,34459,34462],{"style":16217},[57,34460],{"className":34461,"style":16221},[1104],[57,34463,34465],{"className":34464},[1109,1110,1111,1112],[57,34466,34468],{"className":34467},[1075,1112],[57,34469,7995],{"className":34470},[1075,1148,1112],[57,34472,34473,34476],{"style":16230},[57,34474],{"className":34475,"style":16221},[1104],[57,34477,34478],{},[57,34479,2549],{"className":34480},[1888,2633,16239],[57,34482,1183],{"className":34483},[1182],[57,34485,34487],{"className":34486},[1092],[57,34488,34490],{"className":34489,"style":16249},[1096],[57,34491],{},[57,34493],{"className":34494,"style":1853},[1452],[57,34496,12389],{"className":34497,"style":14380},[1075,1148],[57,34499,1754],{"className":34500},[1079],[57,34502,7995],{"className":34503},[1075,1148],[57,34505,1768],{"className":34506},[1269],[57,34508,34510,34513,34584],{"className":34509},[1075],[57,34511],{"className":34512},[1079,1080],[57,34514,34516],{"className":34515},[1035],[57,34517,34519,34576],{"className":34518},[1087,1088],[57,34520,34522,34573],{"className":34521},[1092],[57,34523,34525,34545,34553],{"className":34524,"style":16605},[1096],[57,34526,34527,34530],{"style":16608},[57,34528],{"className":34529,"style":1105},[1104],[57,34531,34533,34536,34539,34542],{"className":34532},[1075],[57,34534,2418],{"className":34535,"style":1394},[1075,1148],[57,34537,1754],{"className":34538},[1079],[57,34540,7995],{"className":34541},[1075,1148],[57,34543,1768],{"className":34544},[1269],[57,34546,34547,34550],{"style":1230},[57,34548],{"className":34549,"style":1105},[1104],[57,34551],{"className":34552,"style":1238},[1237],[57,34554,34555,34558],{"style":16674},[57,34556],{"className":34557,"style":1105},[1104],[57,34559,34561,34564,34567,34570],{"className":34560},[1075],[57,34562,14],{"className":34563},[1075,1148],[57,34565,1754],{"className":34566},[1079],[57,34568,7995],{"className":34569},[1075,1148],[57,34571,1768],{"className":34572},[1269],[57,34574,1183],{"className":34575},[1182],[57,34577,34579],{"className":34578},[1092],[57,34580,34582],{"className":34581,"style":16739},[1096],[57,34583],{},[57,34585],{"className":34586},[1269,1080],[57,34588],{"className":34589,"style":2595},[1452],[57,34591,2539],{"className":34592},[1506],[57,34594],{"className":34595,"style":2595},[1452],[57,34597,2418],{"className":34598,"style":1394},[1075,1148],[57,34600,1754],{"className":34601},[1079],[57,34603,7995],{"className":34604},[1075,1148],[57,34606,1768],{"className":34607},[1269],[57,34609,34610,34613],{"style":34099},[57,34611],{"className":34612,"style":33999},[1104],[57,34614,34616,34619,34622,34625,34628,34631,34634],{"className":34615},[1075],[57,34617],{"className":34618},[1075],[57,34620],{"className":34621,"style":1453},[1452],[57,34623,281],{"className":34624},[1457],[57,34626],{"className":34627,"style":1453},[1452],[57,34629,6336],{"className":34630},[1075,1474],[57,34632],{"className":34633,"style":1853},[1452],[57,34635,34637,34744],{"className":34636},[1898],[57,34638,34640,34646,34649,34652,34655,34658,34738],{"className":34639},[1898],[57,34641,34643],{"className":34642,"style":1903},[1079,1902],[57,34644,1754],{"className":34645},[1907,1111],[57,34647,12389],{"className":34648,"style":14380},[1075,1148],[57,34650,1754],{"className":34651},[1079],[57,34653,7995],{"className":34654},[1075,1148],[57,34656,1768],{"className":34657},[1269],[57,34659,34661,34664,34735],{"className":34660},[1075],[57,34662],{"className":34663},[1079,1080],[57,34665,34667],{"className":34666},[1035],[57,34668,34670,34727],{"className":34669},[1087,1088],[57,34671,34673,34724],{"className":34672},[1092],[57,34674,34676,34696,34704],{"className":34675,"style":16605},[1096],[57,34677,34678,34681],{"style":16608},[57,34679],{"className":34680,"style":1105},[1104],[57,34682,34684,34687,34690,34693],{"className":34683},[1075],[57,34685,2418],{"className":34686,"style":1394},[1075,1148],[57,34688,1754],{"className":34689},[1079],[57,34691,7995],{"className":34692},[1075,1148],[57,34694,1768],{"className":34695},[1269],[57,34697,34698,34701],{"style":1230},[57,34699],{"className":34700,"style":1105},[1104],[57,34702],{"className":34703,"style":1238},[1237],[57,34705,34706,34709],{"style":16674},[57,34707],{"className":34708,"style":1105},[1104],[57,34710,34712,34715,34718,34721],{"className":34711},[1075],[57,34713,14],{"className":34714},[1075,1148],[57,34716,1754],{"className":34717},[1079],[57,34719,7995],{"className":34720},[1075,1148],[57,34722,1768],{"className":34723},[1269],[57,34725,1183],{"className":34726},[1182],[57,34728,34730],{"className":34729},[1092],[57,34731,34733],{"className":34732,"style":16739},[1096],[57,34734],{},[57,34736],{"className":34737},[1269,1080],[57,34739,34741],{"className":34740,"style":1903},[1269,1902],[57,34742,1768],{"className":34743},[1907,1111],[57,34745,34747],{"className":34746},[1152],[57,34748,34750,34790],{"className":34749},[1087,1088],[57,34751,34753,34787],{"className":34752},[1092],[57,34754,34757],{"className":34755,"style":34756},[1096],"height:-0.4747em;",[57,34758,34760,34763],{"style":34759},"top:-1.7003em;margin-right:0.05em;",[57,34761],{"className":34762,"style":1493},[1104],[57,34764,34766],{"className":34765},[1109,1110,1111,1112],[57,34767,34769,34772,34775,34778,34781,34784],{"className":34768},[1075,1112],[57,34770,7995],{"className":34771},[1075,1148,1112],[57,34773,14500],{"className":34774},[1457,1112],[57,34776,2418],{"className":34777,"style":1394},[1075,1148,1112],[57,34779,1754],{"className":34780},[1079,1112],[57,34782,7995],{"className":34783},[1075,1148,1112],[57,34785,1768],{"className":34786},[1269,1112],[57,34788,1183],{"className":34789},[1182],[57,34791,34793],{"className":34792},[1092],[57,34794,34797],{"className":34795,"style":34796},[1096],"height:1.1747em;",[57,34798],{},[57,34800,34801,34804],{"style":34108},[57,34802],{"className":34803,"style":33999},[1104],[57,34805,34807,34810,34813,34816,34819,34881,34884,34951,34954,34957,34960,35000,35003,35157,35160,35163,35166,35206,35209,35212,35215,35218,35221,35224],{"className":34806},[1075],[57,34808],{"className":34809},[1075],[57,34811],{"className":34812,"style":1453},[1452],[57,34814,15464],{"className":34815},[1457],[57,34817],{"className":34818,"style":1453},[1452],[57,34820,34822,34825,34878],{"className":34821},[1075],[57,34823],{"className":34824},[1079,1080],[57,34826,34828],{"className":34827},[1035],[57,34829,34831,34870],{"className":34830},[1087,1088],[57,34832,34834,34867],{"className":34833},[1092],[57,34835,34837,34848,34856],{"className":34836,"style":17068},[1096],[57,34838,34839,34842],{"style":16608},[57,34840],{"className":34841,"style":1105},[1104],[57,34843,34845],{"className":34844},[1075],[57,34846,7990],{"className":34847,"style":17080},[1075,1148],[57,34849,34850,34853],{"style":1230},[57,34851],{"className":34852,"style":1105},[1104],[57,34854],{"className":34855,"style":1238},[1237],[57,34857,34858,34861],{"style":16674},[57,34859],{"className":34860,"style":1105},[1104],[57,34862,34864],{"className":34863},[1075],[57,34865,574],{"className":34866},[1075],[57,34868,1183],{"className":34869},[1182],[57,34871,34873],{"className":34872},[1092],[57,34874,34876],{"className":34875,"style":17109},[1096],[57,34877],{},[57,34879],{"className":34880},[1269,1080],[57,34882],{"className":34883,"style":1853},[1452],[57,34885,34887],{"className":34886},[1888,15027],[57,34888,34890,34943],{"className":34889},[1087,1088],[57,34891,34893,34940],{"className":34892},[1092],[57,34894,34896,34916,34926],{"className":34895,"style":17130},[1096],[57,34897,34898,34901],{"style":17133},[57,34899],{"className":34900,"style":16221},[1104],[57,34902,34904],{"className":34903},[1109,1110,1111,1112],[57,34905,34907,34910,34913],{"className":34906},[1075,1112],[57,34908,1288],{"className":34909},[1075,1148,1112],[57,34911,281],{"className":34912},[1457,1112],[57,34914,574],{"className":34915},[1075,1112],[57,34917,34918,34921],{"style":16230},[57,34919],{"className":34920,"style":16221},[1104],[57,34922,34923],{},[57,34924,2549],{"className":34925},[1888,2633,16239],[57,34927,34928,34931],{"style":17164},[57,34929],{"className":34930,"style":16221},[1104],[57,34932,34934],{"className":34933},[1109,1110,1111,1112],[57,34935,34937],{"className":34936},[1075,1112],[57,34938,7990],{"className":34939,"style":17080},[1075,1148,1112],[57,34941,1183],{"className":34942},[1182],[57,34944,34946],{"className":34945},[1092],[57,34947,34949],{"className":34948,"style":17183},[1096],[57,34950],{},[57,34952],{"className":34953,"style":1853},[1452],[57,34955,12389],{"className":34956,"style":14380},[1075,1148],[57,34958,1754],{"className":34959},[1079],[57,34961,34963,34966],{"className":34962},[1075],[57,34964,7995],{"className":34965},[1075,1148],[57,34967,34969],{"className":34968},[1152],[57,34970,34972,34992],{"className":34971},[1087,1088],[57,34973,34975,34989],{"className":34974},[1092],[57,34976,34978],{"className":34977,"style":8990},[1096],[57,34979,34980,34983],{"style":2211},[57,34981],{"className":34982,"style":1493},[1104],[57,34984,34986],{"className":34985},[1109,1110,1111,1112],[57,34987,1288],{"className":34988},[1075,1148,1112],[57,34990,1183],{"className":34991},[1182],[57,34993,34995],{"className":34994},[1092],[57,34996,34998],{"className":34997,"style":2230},[1096],[57,34999],{},[57,35001,1768],{"className":35002},[1269],[57,35004,35006,35009,35154],{"className":35005},[1075],[57,35007],{"className":35008},[1079,1080],[57,35010,35012],{"className":35011},[1035],[57,35013,35015,35146],{"className":35014},[1087,1088],[57,35016,35018,35143],{"className":35017},[1092],[57,35019,35021,35078,35086],{"className":35020,"style":16605},[1096],[57,35022,35023,35026],{"style":16608},[57,35024],{"className":35025,"style":1105},[1104],[57,35027,35029,35032,35035,35075],{"className":35028},[1075],[57,35030,2418],{"className":35031,"style":1394},[1075,1148],[57,35033,1754],{"className":35034},[1079],[57,35036,35038,35041],{"className":35037},[1075],[57,35039,7995],{"className":35040},[1075,1148],[57,35042,35044],{"className":35043},[1152],[57,35045,35047,35067],{"className":35046},[1087,1088],[57,35048,35050,35064],{"className":35049},[1092],[57,35051,35053],{"className":35052,"style":8990},[1096],[57,35054,35055,35058],{"style":2211},[57,35056],{"className":35057,"style":1493},[1104],[57,35059,35061],{"className":35060},[1109,1110,1111,1112],[57,35062,1288],{"className":35063},[1075,1148,1112],[57,35065,1183],{"className":35066},[1182],[57,35068,35070],{"className":35069},[1092],[57,35071,35073],{"className":35072,"style":2230},[1096],[57,35074],{},[57,35076,1768],{"className":35077},[1269],[57,35079,35080,35083],{"style":1230},[57,35081],{"className":35082,"style":1105},[1104],[57,35084],{"className":35085,"style":1238},[1237],[57,35087,35088,35091],{"style":16674},[57,35089],{"className":35090,"style":1105},[1104],[57,35092,35094,35097,35100,35140],{"className":35093},[1075],[57,35095,14],{"className":35096},[1075,1148],[57,35098,1754],{"className":35099},[1079],[57,35101,35103,35106],{"className":35102},[1075],[57,35104,7995],{"className":35105},[1075,1148],[57,35107,35109],{"className":35108},[1152],[57,35110,35112,35132],{"className":35111},[1087,1088],[57,35113,35115,35129],{"className":35114},[1092],[57,35116,35118],{"className":35117,"style":8990},[1096],[57,35119,35120,35123],{"style":2211},[57,35121],{"className":35122,"style":1493},[1104],[57,35124,35126],{"className":35125},[1109,1110,1111,1112],[57,35127,1288],{"className":35128},[1075,1148,1112],[57,35130,1183],{"className":35131},[1182],[57,35133,35135],{"className":35134},[1092],[57,35136,35138],{"className":35137,"style":2230},[1096],[57,35139],{},[57,35141,1768],{"className":35142},[1269],[57,35144,1183],{"className":35145},[1182],[57,35147,35149],{"className":35148},[1092],[57,35150,35152],{"className":35151,"style":16739},[1096],[57,35153],{},[57,35155],{"className":35156},[1269,1080],[57,35158,1759],{"className":35159},[1849],[57,35161],{"className":35162,"style":29667},[1452],[57,35164],{"className":35165,"style":1853},[1452],[57,35167,35169,35172],{"className":35168},[1075],[57,35170,7995],{"className":35171},[1075,1148],[57,35173,35175],{"className":35174},[1152],[57,35176,35178,35198],{"className":35177},[1087,1088],[57,35179,35181,35195],{"className":35180},[1092],[57,35182,35184],{"className":35183,"style":8990},[1096],[57,35185,35186,35189],{"style":2211},[57,35187],{"className":35188,"style":1493},[1104],[57,35190,35192],{"className":35191},[1109,1110,1111,1112],[57,35193,1288],{"className":35194},[1075,1148,1112],[57,35196,1183],{"className":35197},[1182],[57,35199,35201],{"className":35200},[1092],[57,35202,35204],{"className":35203,"style":2230},[1096],[57,35205],{},[57,35207],{"className":35208,"style":1453},[1452],[57,35210,14500],{"className":35211},[1457],[57,35213],{"className":35214,"style":1453},[1452],[57,35216,2418],{"className":35217,"style":1394},[1075,1148],[57,35219,1754],{"className":35220},[1079],[57,35222,7995],{"className":35223},[1075,1148],[57,35225,1768],{"className":35226},[1269],[57,35228,1183],{"className":35229},[1182],[57,35231,35233],{"className":35232},[1092],[57,35234,35236],{"className":35235,"style":34124},[1096],[57,35237],{},[14,35239,35240],{},"利用重要性采样，我们就可以把 on-policy 的梯度公式改写为可以复用旧数据的 off-policy 形式。",[248,35242,35243],{"id":33228},"Off-policy",[14,35245,35246,35247,35310,35311,7517,35314,35310,35343,35346,35347,35484,35485,35676,35677,21450,35680,35895],{},"记 ",[57,35248,35250,35268],{"className":35249},[1019],[57,35251,35253],{"className":35252},[1023],[1025,35254,35255],{"xmlns":1027},[1029,35256,35257,35265],{},[1032,35258,35259],{},[1418,35260,35261,35263],{},[1047,35262,14745],{},[1414,35264,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1054,35266,35267],{"encoding":1056},"\\theta'",[57,35269,35271],{"className":35270,"ariaHidden":1062},[1061],[57,35272,35274,35278],{"className":35273},[1066],[57,35275],{"className":35276,"style":35277},[1070],"height:0.7519em;",[57,35279,35281,35284],{"className":35280},[1075],[57,35282,14745],{"className":35283,"style":3932},[1075,1148],[57,35285,35287],{"className":35286},[1152],[57,35288,35290],{"className":35289},[1087],[57,35291,35293],{"className":35292},[1092],[57,35294,35296],{"className":35295,"style":35277},[1096],[57,35297,35298,35301],{"style":1489},[57,35299],{"className":35300,"style":1493},[1104],[57,35302,35304],{"className":35303},[1109,1110,1111,1112],[57,35305,35307],{"className":35306},[1075,1112],[57,35308,19775],{"className":35309},[1075,1112]," 为 ",[11865,35312,35313],{},"采集数据时使用的旧策略",[57,35315,35317,35331],{"className":35316},[1019],[57,35318,35320],{"className":35319},[1023],[1025,35321,35322],{"xmlns":1027},[1029,35323,35324,35328],{},[1032,35325,35326],{},[1047,35327,14745],{},[1054,35329,35330],{"encoding":1056},"\\theta",[57,35332,35334],{"className":35333,"ariaHidden":1062},[1061],[57,35335,35337,35340],{"className":35336},[1066],[57,35338],{"className":35339,"style":1360},[1070],[57,35341,14745],{"className":35342,"style":3932},[1075,1148],[11865,35344,35345],{},"当前要优化的策略","；优势 ",[57,35348,35350,35380],{"className":35349},[1019],[57,35351,35353],{"className":35352},[1023],[1025,35354,35355],{"xmlns":1027},[1029,35356,35357,35377],{},[1032,35358,35359],{},[2545,35360,35361,35363,35369],{},[1047,35362,1730],{},[1418,35364,35365,35367],{},[1047,35366,14745],{},[1414,35368,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,35370,35371,35373,35375],{},[1047,35372,28613],{},[1047,35374,1730],{},[1047,35376,6336],{},[1054,35378,35379],{"encoding":1056},"A_{\\theta'}^{GAE}",[57,35381,35383],{"className":35382,"ariaHidden":1062},[1061],[57,35384,35386,35390],{"className":35385},[1066],[57,35387],{"className":35388,"style":35389},[1070],"height:1.1244em;vertical-align:-0.2831em;",[57,35391,35393,35396],{"className":35392},[1075],[57,35394,1730],{"className":35395},[1075,1148],[57,35397,35399],{"className":35398},[1152],[57,35400,35402,35475],{"className":35401},[1087,1088],[57,35403,35405,35472],{"className":35404},[1092],[57,35406,35408,35452],{"className":35407,"style":3920},[1096],[57,35409,35411,35414],{"style":35410},"top:-2.4169em;margin-left:0em;margin-right:0.05em;",[57,35412],{"className":35413,"style":1493},[1104],[57,35415,35417],{"className":35416},[1109,1110,1111,1112],[57,35418,35420],{"className":35419},[1075,1112],[57,35421,35423,35426],{"className":35422},[1075,1112],[57,35424,14745],{"className":35425,"style":3932},[1075,1148,1112],[57,35427,35429],{"className":35428},[1152],[57,35430,35432],{"className":35431},[1087],[57,35433,35435],{"className":35434},[1092],[57,35436,35438],{"className":35437,"style":19935},[1096],[57,35439,35440,35443],{"style":19938},[57,35441],{"className":35442,"style":1169},[1104],[57,35444,35446],{"className":35445},[1109,1173,1174,1112],[57,35447,35449],{"className":35448},[1075,1112],[57,35450,19775],{"className":35451},[1075,1112],[57,35453,35454,35457],{"style":1489},[57,35455],{"className":35456,"style":1493},[1104],[57,35458,35460],{"className":35459},[1109,1110,1111,1112],[57,35461,35463,35466,35469],{"className":35462},[1075,1112],[57,35464,28613],{"className":35465},[1075,1148,1112],[57,35467,1730],{"className":35468},[1075,1148,1112],[57,35470,6336],{"className":35471,"style":6749},[1075,1148,1112],[57,35473,1183],{"className":35474},[1182],[57,35476,35478],{"className":35477},[1092],[57,35479,35482],{"className":35480,"style":35481},[1096],"height:0.2831em;",[57,35483],{}," 由旧策略下的价值网络估计而来。结合恒等式 ",[57,35486,35488,35542],{"className":35487},[1019],[57,35489,35491],{"className":35490},[1023],[1025,35492,35493],{"xmlns":1027},[1029,35494,35495,35539],{},[1032,35496,35497,35499,35501,35503,35505,35507,35509,35511,35513],{},[1047,35498,15169],{"mathvariant":1729},[1047,35500,15594],{},[1414,35502,1776],{},[1047,35504,12389],{},[1414,35506,1754],{"stretchy":1753},[1047,35508,7995],{},[1414,35510,1768],{"stretchy":1753},[1414,35512,281],{},[15163,35514,35515],{"scriptlevel":560,"displaystyle":1062},[1035,35516,35517,35529],{},[1032,35518,35519,35521,35523,35525,35527],{},[1047,35520,15169],{"mathvariant":1729},[1047,35522,12389],{},[1414,35524,1754],{"stretchy":1753},[1047,35526,7995],{},[1414,35528,1768],{"stretchy":1753},[1032,35530,35531,35533,35535,35537],{},[1047,35532,12389],{},[1414,35534,1754],{"stretchy":1753},[1047,35536,7995],{},[1414,35538,1768],{"stretchy":1753},[1054,35540,35541],{"encoding":1056},"\\nabla \\log f(x) = \\dfrac{\\nabla f(x)}{f(x)}",[57,35543,35545,35586],{"className":35544,"ariaHidden":1062},[1061],[57,35546,35548,35551,35554,35557,35562,35565,35568,35571,35574,35577,35580,35583],{"className":35547},[1066],[57,35549],{"className":35550,"style":1831},[1070],[57,35552,15169],{"className":35553},[1075],[57,35555],{"className":35556,"style":1853},[1452],[57,35558,17640,35560],{"className":35559},[1888],[57,35561,17644],{"style":17643},[57,35563],{"className":35564,"style":1853},[1452],[57,35566,12389],{"className":35567,"style":14380},[1075,1148],[57,35569,1754],{"className":35570},[1079],[57,35572,7995],{"className":35573},[1075,1148],[57,35575,1768],{"className":35576},[1269],[57,35578],{"className":35579,"style":1453},[1452],[57,35581,281],{"className":35582},[1457],[57,35584],{"className":35585,"style":1453},[1452],[57,35587,35589,35593],{"className":35588},[1066],[57,35590],{"className":35591,"style":35592},[1070],"height:2.363em;vertical-align:-0.936em;",[57,35594,35596,35599,35673],{"className":35595},[1075],[57,35597],{"className":35598},[1079,1080],[57,35600,35602],{"className":35601},[1035],[57,35603,35605,35665],{"className":35604},[1087,1088],[57,35606,35608,35662],{"className":35607},[1092],[57,35609,35611,35631,35639],{"className":35610,"style":16605},[1096],[57,35612,35613,35616],{"style":16608},[57,35614],{"className":35615,"style":1105},[1104],[57,35617,35619,35622,35625,35628],{"className":35618},[1075],[57,35620,12389],{"className":35621,"style":14380},[1075,1148],[57,35623,1754],{"className":35624},[1079],[57,35626,7995],{"className":35627},[1075,1148],[57,35629,1768],{"className":35630},[1269],[57,35632,35633,35636],{"style":1230},[57,35634],{"className":35635,"style":1105},[1104],[57,35637],{"className":35638,"style":1238},[1237],[57,35640,35641,35644],{"style":16674},[57,35642],{"className":35643,"style":1105},[1104],[57,35645,35647,35650,35653,35656,35659],{"className":35646},[1075],[57,35648,15169],{"className":35649},[1075],[57,35651,12389],{"className":35652,"style":14380},[1075,1148],[57,35654,1754],{"className":35655},[1079],[57,35657,7995],{"className":35658},[1075,1148],[57,35660,1768],{"className":35661},[1269],[57,35663,1183],{"className":35664},[1182],[57,35666,35668],{"className":35667},[1092],[57,35669,35671],{"className":35670,"style":16739},[1096],[57,35672],{},[57,35674],{"className":35675},[1269,1080],"，可以把策略梯度写成包含 ",[11865,35678,35679],{},"重要性采样比",[57,35681,35683,35715],{"className":35682},[1019],[57,35684,35686],{"className":35685},[1023],[1025,35687,35688],{"xmlns":1027},[1029,35689,35690,35712],{},[1032,35691,35692],{},[15163,35693,35694],{"scriptlevel":560,"displaystyle":1062},[1035,35695,35696,35702],{},[1044,35697,35698,35700],{},[1047,35699,6333],{},[1047,35701,14745],{},[1044,35703,35704,35706],{},[1047,35705,6333],{},[1418,35707,35708,35710],{},[1047,35709,14745],{},[1414,35711,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1054,35713,35714],{"encoding":1056},"\\dfrac{P_\\theta}{P_{\\theta'}}",[57,35716,35718],{"className":35717,"ariaHidden":1062},[1061],[57,35719,35721,35725],{"className":35720},[1066],[57,35722],{"className":35723,"style":35724},[1070],"height:2.1963em;vertical-align:-0.836em;",[57,35726,35728,35731,35892],{"className":35727},[1075],[57,35729],{"className":35730},[1079,1080],[57,35732,35734],{"className":35733},[1035],[57,35735,35737,35883],{"className":35736},[1087,1088],[57,35738,35740,35880],{"className":35739},[1092],[57,35741,35744,35824,35832],{"className":35742,"style":35743},[1096],"height:1.3603em;",[57,35745,35746,35749],{"style":16608},[57,35747],{"className":35748,"style":1105},[1104],[57,35750,35752],{"className":35751},[1075],[57,35753,35755,35758],{"className":35754},[1075],[57,35756,6333],{"className":35757,"style":3907},[1075,1148],[57,35759,35761],{"className":35760},[1152],[57,35762,35764,35816],{"className":35763},[1087,1088],[57,35765,35767,35813],{"className":35766},[1092],[57,35768,35770],{"className":35769,"style":2208},[1096],[57,35771,35772,35775],{"style":8131},[57,35773],{"className":35774,"style":1493},[1104],[57,35776,35778],{"className":35777},[1109,1110,1111,1112],[57,35779,35781],{"className":35780},[1075,1112],[57,35782,35784,35787],{"className":35783},[1075,1112],[57,35785,14745],{"className":35786,"style":3932},[1075,1148,1112],[57,35788,35790],{"className":35789},[1152],[57,35791,35793],{"className":35792},[1087],[57,35794,35796],{"className":35795},[1092],[57,35797,35799],{"className":35798,"style":19935},[1096],[57,35800,35801,35804],{"style":19938},[57,35802],{"className":35803,"style":1169},[1104],[57,35805,35807],{"className":35806},[1109,1173,1174,1112],[57,35808,35810],{"className":35809},[1075,1112],[57,35811,19775],{"className":35812},[1075,1112],[57,35814,1183],{"className":35815},[1182],[57,35817,35819],{"className":35818},[1092],[57,35820,35822],{"className":35821,"style":2230},[1096],[57,35823],{},[57,35825,35826,35829],{"style":1230},[57,35827],{"className":35828,"style":1105},[1104],[57,35830],{"className":35831,"style":1238},[1237],[57,35833,35834,35837],{"style":16674},[57,35835],{"className":35836,"style":1105},[1104],[57,35838,35840],{"className":35839},[1075],[57,35841,35843,35846],{"className":35842},[1075],[57,35844,6333],{"className":35845,"style":3907},[1075,1148],[57,35847,35849],{"className":35848},[1152],[57,35850,35852,35872],{"className":35851},[1087,1088],[57,35853,35855,35869],{"className":35854},[1092],[57,35856,35858],{"className":35857,"style":2208},[1096],[57,35859,35860,35863],{"style":8131},[57,35861],{"className":35862,"style":1493},[1104],[57,35864,35866],{"className":35865},[1109,1110,1111,1112],[57,35867,14745],{"className":35868,"style":3932},[1075,1148,1112],[57,35870,1183],{"className":35871},[1182],[57,35873,35875],{"className":35874},[1092],[57,35876,35878],{"className":35877,"style":2230},[1096],[57,35879],{},[57,35881,1183],{"className":35882},[1182],[57,35884,35886],{"className":35885},[1092],[57,35887,35890],{"className":35888,"style":35889},[1096],"height:0.836em;",[57,35891],{},[57,35893],{"className":35894},[1269,1080]," 的形式：",[57,35897,35899],{"className":35898},[15139],[57,35900,35902,35954],{"className":35901},[1019],[57,35903,35905],{"className":35904},[1023],[1025,35906,35907],{"xmlns":1027,"display":10210},[1029,35908,35909,35951],{},[1032,35910,35911,35913,35915,35917,35919,35921,35923,35925,35927],{},[1047,35912,15169],{"mathvariant":1729},[1047,35914,15594],{},[1414,35916,1776],{},[1047,35918,12389],{},[1414,35920,1754],{"stretchy":1753},[1047,35922,7995],{},[1414,35924,1768],{"stretchy":1753},[1414,35926,281],{},[1035,35928,35929,35941],{},[1032,35930,35931,35933,35935,35937,35939],{},[1047,35932,15169],{"mathvariant":1729},[1047,35934,12389],{},[1414,35936,1754],{"stretchy":1753},[1047,35938,7995],{},[1414,35940,1768],{"stretchy":1753},[1032,35942,35943,35945,35947,35949],{},[1047,35944,12389],{},[1414,35946,1754],{"stretchy":1753},[1047,35948,7995],{},[1414,35950,1768],{"stretchy":1753},[1054,35952,35953],{"encoding":1056},"\\nabla \\log f(x) = \\frac{\\nabla f(x)}{f(x)}",[57,35955,35957,35998],{"className":35956,"ariaHidden":1062},[1061],[57,35958,35960,35963,35966,35969,35974,35977,35980,35983,35986,35989,35992,35995],{"className":35959},[1066],[57,35961],{"className":35962,"style":1831},[1070],[57,35964,15169],{"className":35965},[1075],[57,35967],{"className":35968,"style":1853},[1452],[57,35970,17640,35972],{"className":35971},[1888],[57,35973,17644],{"style":17643},[57,35975],{"className":35976,"style":1853},[1452],[57,35978,12389],{"className":35979,"style":14380},[1075,1148],[57,35981,1754],{"className":35982},[1079],[57,35984,7995],{"className":35985},[1075,1148],[57,35987,1768],{"className":35988},[1269],[57,35990],{"className":35991,"style":1453},[1452],[57,35993,281],{"className":35994},[1457],[57,35996],{"className":35997,"style":1453},[1452],[57,35999,36001,36004],{"className":36000},[1066],[57,36002],{"className":36003,"style":35592},[1070],[57,36005,36007,36010,36084],{"className":36006},[1075],[57,36008],{"className":36009},[1079,1080],[57,36011,36013],{"className":36012},[1035],[57,36014,36016,36076],{"className":36015},[1087,1088],[57,36017,36019,36073],{"className":36018},[1092],[57,36020,36022,36042,36050],{"className":36021,"style":16605},[1096],[57,36023,36024,36027],{"style":16608},[57,36025],{"className":36026,"style":1105},[1104],[57,36028,36030,36033,36036,36039],{"className":36029},[1075],[57,36031,12389],{"className":36032,"style":14380},[1075,1148],[57,36034,1754],{"className":36035},[1079],[57,36037,7995],{"className":36038},[1075,1148],[57,36040,1768],{"className":36041},[1269],[57,36043,36044,36047],{"style":1230},[57,36045],{"className":36046,"style":1105},[1104],[57,36048],{"className":36049,"style":1238},[1237],[57,36051,36052,36055],{"style":16674},[57,36053],{"className":36054,"style":1105},[1104],[57,36056,36058,36061,36064,36067,36070],{"className":36057},[1075],[57,36059,15169],{"className":36060},[1075],[57,36062,12389],{"className":36063,"style":14380},[1075,1148],[57,36065,1754],{"className":36066},[1079],[57,36068,7995],{"className":36069},[1075,1148],[57,36071,1768],{"className":36072},[1269],[57,36074,1183],{"className":36075},[1182],[57,36077,36079],{"className":36078},[1092],[57,36080,36082],{"className":36081,"style":16739},[1096],[57,36083],{},[57,36085],{"className":36086},[1269,1080],[57,36088,36090],{"className":36089},[15139],[57,36091,36093,36815],{"className":36092},[1019],[57,36094,36096],{"className":36095},[1023],[1025,36097,36098],{"xmlns":1027,"display":10210},[1029,36099,36100,36812],{},[15151,36101,36102,36226,36422,36648],{"rowspacing":15153,"columnalign":15154,"columnspacing":15155},[15157,36103,36104,36110],{},[15160,36105,36106],{},[15163,36107,36108],{"scriptlevel":560,"displaystyle":1062},[1032,36109],{},[15160,36111,36112],{},[15163,36113,36114],{"scriptlevel":560,"displaystyle":1062},[1032,36115,36116,36118,36124,36138,36156,36170,36172,36180,36182,36190,36192,36194,36196,36198,36204,36206,36214,36216,36224],{},[1032,36117],{},[1035,36119,36120,36122],{},[1038,36121,574],{},[1047,36123,7990],{},[15472,36125,36126,36128,36136],{},[1414,36127,2549],{},[1032,36129,36130,36132,36134],{},[1047,36131,1288],{},[1414,36133,281],{},[1038,36135,574],{},[1047,36137,7990],{},[15472,36139,36140,36142,36150],{},[1414,36141,2549],{},[1032,36143,36144,36146,36148],{},[1047,36145,1733],{},[1414,36147,281],{},[1038,36149,574],{},[1044,36151,36152,36154],{},[1047,36153,738],{},[1047,36155,1288],{},[2545,36157,36158,36160,36162],{},[1047,36159,1730],{},[1047,36161,14745],{},[1032,36163,36164,36166,36168],{},[1047,36165,28613],{},[1047,36167,1730],{},[1047,36169,6336],{},[1414,36171,1754],{"stretchy":1753},[2545,36173,36174,36176,36178],{},[1047,36175,13535],{},[1047,36177,1288],{},[1047,36179,1733],{},[1414,36181,1759],{"separator":1062},[2545,36183,36184,36186,36188],{},[1047,36185,18],{},[1047,36187,1288],{},[1047,36189,1733],{},[1414,36191,1768],{"stretchy":1753},[1047,36193,15169],{"mathvariant":1729},[1047,36195,15594],{},[1414,36197,1776],{},[1044,36199,36200,36202],{},[1047,36201,6333],{},[1047,36203,14745],{},[1414,36205,1754],{"stretchy":1753},[2545,36207,36208,36210,36212],{},[1047,36209,18],{},[1047,36211,1288],{},[1047,36213,1733],{},[1047,36215,13675],{"mathvariant":1729},[2545,36217,36218,36220,36222],{},[1047,36219,13535],{},[1047,36221,1288],{},[1047,36223,1733],{},[1414,36225,1768],{"stretchy":1753},[15157,36227,36228,36234],{},[15160,36229,36230],{},[15163,36231,36232],{"scriptlevel":560,"displaystyle":1062},[1032,36233],{},[15160,36235,36236],{},[15163,36237,36238],{"scriptlevel":560,"displaystyle":1062},[1032,36239,36240,36242,36244,36250,36264,36282,36300,36302,36310,36312,36320,36322,36388,36390,36392,36394,36400,36402,36410,36412,36420],{},[1032,36241],{},[1414,36243,281],{},[1035,36245,36246,36248],{},[1038,36247,574],{},[1047,36249,7990],{},[15472,36251,36252,36254,36262],{},[1414,36253,2549],{},[1032,36255,36256,36258,36260],{},[1047,36257,1288],{},[1414,36259,281],{},[1038,36261,574],{},[1047,36263,7990],{},[15472,36265,36266,36268,36276],{},[1414,36267,2549],{},[1032,36269,36270,36272,36274],{},[1047,36271,1733],{},[1414,36273,281],{},[1038,36275,574],{},[1044,36277,36278,36280],{},[1047,36279,738],{},[1047,36281,1288],{},[2545,36283,36284,36286,36292],{},[1047,36285,1730],{},[1418,36287,36288,36290],{},[1047,36289,14745],{},[1414,36291,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,36293,36294,36296,36298],{},[1047,36295,28613],{},[1047,36297,1730],{},[1047,36299,6336],{},[1414,36301,1754],{"stretchy":1753},[2545,36303,36304,36306,36308],{},[1047,36305,13535],{},[1047,36307,1288],{},[1047,36309,1733],{},[1414,36311,1759],{"separator":1062},[2545,36313,36314,36316,36318],{},[1047,36315,18],{},[1047,36317,1288],{},[1047,36319,1733],{},[1414,36321,1768],{"stretchy":1753},[1035,36323,36324,36354],{},[1032,36325,36326,36332,36334,36342,36344,36352],{},[1044,36327,36328,36330],{},[1047,36329,6333],{},[1047,36331,14745],{},[1414,36333,1754],{"stretchy":1753},[2545,36335,36336,36338,36340],{},[1047,36337,18],{},[1047,36339,1288],{},[1047,36341,1733],{},[1047,36343,13675],{"mathvariant":1729},[2545,36345,36346,36348,36350],{},[1047,36347,13535],{},[1047,36349,1288],{},[1047,36351,1733],{},[1414,36353,1768],{"stretchy":1753},[1032,36355,36356,36366,36368,36376,36378,36386],{},[1044,36357,36358,36360],{},[1047,36359,6333],{},[1418,36361,36362,36364],{},[1047,36363,14745],{},[1414,36365,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,36367,1754],{"stretchy":1753},[2545,36369,36370,36372,36374],{},[1047,36371,18],{},[1047,36373,1288],{},[1047,36375,1733],{},[1047,36377,13675],{"mathvariant":1729},[2545,36379,36380,36382,36384],{},[1047,36381,13535],{},[1047,36383,1288],{},[1047,36385,1733],{},[1414,36387,1768],{"stretchy":1753},[1047,36389,15169],{"mathvariant":1729},[1047,36391,15594],{},[1414,36393,1776],{},[1044,36395,36396,36398],{},[1047,36397,6333],{},[1047,36399,14745],{},[1414,36401,1754],{"stretchy":1753},[2545,36403,36404,36406,36408],{},[1047,36405,18],{},[1047,36407,1288],{},[1047,36409,1733],{},[1047,36411,13675],{"mathvariant":1729},[2545,36413,36414,36416,36418],{},[1047,36415,13535],{},[1047,36417,1288],{},[1047,36419,1733],{},[1414,36421,1768],{"stretchy":1753},[15157,36423,36424,36430],{},[15160,36425,36426],{},[15163,36427,36428],{"scriptlevel":560,"displaystyle":1062},[1032,36429],{},[15160,36431,36432],{},[15163,36433,36434],{"scriptlevel":560,"displaystyle":1062},[1032,36435,36436,36438,36440,36446,36460,36478,36496,36498,36506,36508,36516,36518,36584],{},[1032,36437],{},[1414,36439,281],{},[1035,36441,36442,36444],{},[1038,36443,574],{},[1047,36445,7990],{},[15472,36447,36448,36450,36458],{},[1414,36449,2549],{},[1032,36451,36452,36454,36456],{},[1047,36453,1288],{},[1414,36455,281],{},[1038,36457,574],{},[1047,36459,7990],{},[15472,36461,36462,36464,36472],{},[1414,36463,2549],{},[1032,36465,36466,36468,36470],{},[1047,36467,1733],{},[1414,36469,281],{},[1038,36471,574],{},[1044,36473,36474,36476],{},[1047,36475,738],{},[1047,36477,1288],{},[2545,36479,36480,36482,36488],{},[1047,36481,1730],{},[1418,36483,36484,36486],{},[1047,36485,14745],{},[1414,36487,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,36489,36490,36492,36494],{},[1047,36491,28613],{},[1047,36493,1730],{},[1047,36495,6336],{},[1414,36497,1754],{"stretchy":1753},[2545,36499,36500,36502,36504],{},[1047,36501,13535],{},[1047,36503,1288],{},[1047,36505,1733],{},[1414,36507,1759],{"separator":1062},[2545,36509,36510,36512,36514],{},[1047,36511,18],{},[1047,36513,1288],{},[1047,36515,1733],{},[1414,36517,1768],{"stretchy":1753},[1035,36519,36520,36550],{},[1032,36521,36522,36528,36530,36538,36540,36548],{},[1044,36523,36524,36526],{},[1047,36525,6333],{},[1047,36527,14745],{},[1414,36529,1754],{"stretchy":1753},[2545,36531,36532,36534,36536],{},[1047,36533,18],{},[1047,36535,1288],{},[1047,36537,1733],{},[1047,36539,13675],{"mathvariant":1729},[2545,36541,36542,36544,36546],{},[1047,36543,13535],{},[1047,36545,1288],{},[1047,36547,1733],{},[1414,36549,1768],{"stretchy":1753},[1032,36551,36552,36562,36564,36572,36574,36582],{},[1044,36553,36554,36556],{},[1047,36555,6333],{},[1418,36557,36558,36560],{},[1047,36559,14745],{},[1414,36561,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,36563,1754],{"stretchy":1753},[2545,36565,36566,36568,36570],{},[1047,36567,18],{},[1047,36569,1288],{},[1047,36571,1733],{},[1047,36573,13675],{"mathvariant":1729},[2545,36575,36576,36578,36580],{},[1047,36577,13535],{},[1047,36579,1288],{},[1047,36581,1733],{},[1414,36583,1768],{"stretchy":1753},[1035,36585,36586,36618],{},[1032,36587,36588,36590,36596,36598,36606,36608,36616],{},[1047,36589,15169],{"mathvariant":1729},[1044,36591,36592,36594],{},[1047,36593,6333],{},[1047,36595,14745],{},[1414,36597,1754],{"stretchy":1753},[2545,36599,36600,36602,36604],{},[1047,36601,18],{},[1047,36603,1288],{},[1047,36605,1733],{},[1047,36607,13675],{"mathvariant":1729},[2545,36609,36610,36612,36614],{},[1047,36611,13535],{},[1047,36613,1288],{},[1047,36615,1733],{},[1414,36617,1768],{"stretchy":1753},[1032,36619,36620,36626,36628,36636,36638,36646],{},[1044,36621,36622,36624],{},[1047,36623,6333],{},[1047,36625,14745],{},[1414,36627,1754],{"stretchy":1753},[2545,36629,36630,36632,36634],{},[1047,36631,18],{},[1047,36633,1288],{},[1047,36635,1733],{},[1047,36637,13675],{"mathvariant":1729},[2545,36639,36640,36642,36644],{},[1047,36641,13535],{},[1047,36643,1288],{},[1047,36645,1733],{},[1414,36647,1768],{"stretchy":1753},[15157,36649,36650,36656],{},[15160,36651,36652],{},[15163,36653,36654],{"scriptlevel":560,"displaystyle":1062},[1032,36655],{},[15160,36657,36658],{},[15163,36659,36660],{"scriptlevel":560,"displaystyle":1062},[1032,36661,36662,36664,36666,36672,36686,36704,36722,36724,36732,36734,36742,36744],{},[1032,36663],{},[1414,36665,281],{},[1035,36667,36668,36670],{},[1038,36669,574],{},[1047,36671,7990],{},[15472,36673,36674,36676,36684],{},[1414,36675,2549],{},[1032,36677,36678,36680,36682],{},[1047,36679,1288],{},[1414,36681,281],{},[1038,36683,574],{},[1047,36685,7990],{},[15472,36687,36688,36690,36698],{},[1414,36689,2549],{},[1032,36691,36692,36694,36696],{},[1047,36693,1733],{},[1414,36695,281],{},[1038,36697,574],{},[1044,36699,36700,36702],{},[1047,36701,738],{},[1047,36703,1288],{},[2545,36705,36706,36708,36714],{},[1047,36707,1730],{},[1418,36709,36710,36712],{},[1047,36711,14745],{},[1414,36713,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,36715,36716,36718,36720],{},[1047,36717,28613],{},[1047,36719,1730],{},[1047,36721,6336],{},[1414,36723,1754],{"stretchy":1753},[2545,36725,36726,36728,36730],{},[1047,36727,13535],{},[1047,36729,1288],{},[1047,36731,1733],{},[1414,36733,1759],{"separator":1062},[2545,36735,36736,36738,36740],{},[1047,36737,18],{},[1047,36739,1288],{},[1047,36741,1733],{},[1414,36743,1768],{"stretchy":1753},[1035,36745,36746,36778],{},[1032,36747,36748,36750,36756,36758,36766,36768,36776],{},[1047,36749,15169],{"mathvariant":1729},[1044,36751,36752,36754],{},[1047,36753,6333],{},[1047,36755,14745],{},[1414,36757,1754],{"stretchy":1753},[2545,36759,36760,36762,36764],{},[1047,36761,18],{},[1047,36763,1288],{},[1047,36765,1733],{},[1047,36767,13675],{"mathvariant":1729},[2545,36769,36770,36772,36774],{},[1047,36771,13535],{},[1047,36773,1288],{},[1047,36775,1733],{},[1414,36777,1768],{"stretchy":1753},[1032,36779,36780,36790,36792,36800,36802,36810],{},[1044,36781,36782,36784],{},[1047,36783,6333],{},[1418,36785,36786,36788],{},[1047,36787,14745],{},[1414,36789,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,36791,1754],{"stretchy":1753},[2545,36793,36794,36796,36798],{},[1047,36795,18],{},[1047,36797,1288],{},[1047,36799,1733],{},[1047,36801,13675],{"mathvariant":1729},[2545,36803,36804,36806,36808],{},[1047,36805,13535],{},[1047,36807,1288],{},[1047,36809,1733],{},[1414,36811,1768],{"stretchy":1753},[1054,36813,36814],{"encoding":1056},"\\begin{aligned}\n&\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_\\theta^{GAE}(s_n^t, a_n^t) \\nabla \\log P_\\theta(a_n^t | s_n^t) \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)} \\nabla \\log P_\\theta(a_n^t | s_n^t) \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)} \\frac{\\nabla P_\\theta(a_n^t | s_n^t)}{P_\\theta(a_n^t | s_n^t)} \\\\\n&= \\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{\\nabla P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)}\n\\end{aligned}",[57,36816,36818],{"className":36817,"ariaHidden":1062},[1061],[57,36819,36821,36825],{"className":36820},[1066],[57,36822],{"className":36823,"style":36824},[1070],"height:13.6262em;vertical-align:-6.5631em;",[57,36826,36828],{"className":36827},[1075],[57,36829,36831,36892],{"className":36830},[15151],[57,36832,36834],{"className":36833},[15942],[57,36835,36837,36883],{"className":36836},[1087,1088],[57,36838,36840,36880],{"className":36839},[1092],[57,36841,36844,36853,36862,36871],{"className":36842,"style":36843},[1096],"height:7.0631em;",[57,36845,36847,36850],{"style":36846},"top:-9.0631em;",[57,36848],{"className":36849,"style":15959},[1104],[57,36851],{"className":36852},[1075],[57,36854,36856,36859],{"style":36855},"top:-5.6565em;",[57,36857],{"className":36858,"style":15959},[1104],[57,36860],{"className":36861},[1075],[57,36863,36865,36868],{"style":36864},"top:-2.25em;",[57,36866],{"className":36867,"style":15959},[1104],[57,36869],{"className":36870},[1075],[57,36872,36874,36877],{"style":36873},"top:1.1565em;",[57,36875],{"className":36876,"style":15959},[1104],[57,36878],{"className":36879},[1075],[57,36881,1183],{"className":36882},[1182],[57,36884,36886],{"className":36885},[1092],[57,36887,36890],{"className":36888,"style":36889},[1096],"height:6.5631em;",[57,36891],{},[57,36893,36895],{"className":36894},[16166],[57,36896,36898,40604],{"className":36897},[1087,1088],[57,36899,36901,40601],{"className":36900},[1092],[57,36902,36904,37496,38521,39740],{"className":36903,"style":36843},[1096],[57,36905,36906,36909],{"style":36846},[57,36907],{"className":36908,"style":15959},[1104],[57,36910,36912,36915,36977,36980,37047,37050,37154,37157,37217,37220,37271,37274,37277,37328,37331,37334,37337,37342,37345,37385,37388,37439,37442,37493],{"className":36911},[1075],[57,36913],{"className":36914},[1075],[57,36916,36918,36921,36974],{"className":36917},[1075],[57,36919],{"className":36920},[1079,1080],[57,36922,36924],{"className":36923},[1035],[57,36925,36927,36966],{"className":36926},[1087,1088],[57,36928,36930,36963],{"className":36929},[1092],[57,36931,36933,36944,36952],{"className":36932,"style":17068},[1096],[57,36934,36935,36938],{"style":16608},[57,36936],{"className":36937,"style":1105},[1104],[57,36939,36941],{"className":36940},[1075],[57,36942,7990],{"className":36943,"style":17080},[1075,1148],[57,36945,36946,36949],{"style":1230},[57,36947],{"className":36948,"style":1105},[1104],[57,36950],{"className":36951,"style":1238},[1237],[57,36953,36954,36957],{"style":16674},[57,36955],{"className":36956,"style":1105},[1104],[57,36958,36960],{"className":36959},[1075],[57,36961,574],{"className":36962},[1075],[57,36964,1183],{"className":36965},[1182],[57,36967,36969],{"className":36968},[1092],[57,36970,36972],{"className":36971,"style":17109},[1096],[57,36973],{},[57,36975],{"className":36976},[1269,1080],[57,36978],{"className":36979,"style":1853},[1452],[57,36981,36983],{"className":36982},[1888,15027],[57,36984,36986,37039],{"className":36985},[1087,1088],[57,36987,36989,37036],{"className":36988},[1092],[57,36990,36992,37012,37022],{"className":36991,"style":17130},[1096],[57,36993,36994,36997],{"style":17133},[57,36995],{"className":36996,"style":16221},[1104],[57,36998,37000],{"className":36999},[1109,1110,1111,1112],[57,37001,37003,37006,37009],{"className":37002},[1075,1112],[57,37004,1288],{"className":37005},[1075,1148,1112],[57,37007,281],{"className":37008},[1457,1112],[57,37010,574],{"className":37011},[1075,1112],[57,37013,37014,37017],{"style":16230},[57,37015],{"className":37016,"style":16221},[1104],[57,37018,37019],{},[57,37020,2549],{"className":37021},[1888,2633,16239],[57,37023,37024,37027],{"style":17164},[57,37025],{"className":37026,"style":16221},[1104],[57,37028,37030],{"className":37029},[1109,1110,1111,1112],[57,37031,37033],{"className":37032},[1075,1112],[57,37034,7990],{"className":37035,"style":17080},[1075,1148,1112],[57,37037,1183],{"className":37038},[1182],[57,37040,37042],{"className":37041},[1092],[57,37043,37045],{"className":37044,"style":17183},[1096],[57,37046],{},[57,37048],{"className":37049,"style":1853},[1452],[57,37051,37053],{"className":37052},[1888,15027],[57,37054,37056,37146],{"className":37055},[1087,1088],[57,37057,37059,37143],{"className":37058},[1092],[57,37060,37062,37082,37092],{"className":37061,"style":17939},[1096],[57,37063,37064,37067],{"style":17133},[57,37065],{"className":37066,"style":16221},[1104],[57,37068,37070],{"className":37069},[1109,1110,1111,1112],[57,37071,37073,37076,37079],{"className":37072},[1075,1112],[57,37074,1733],{"className":37075},[1075,1148,1112],[57,37077,281],{"className":37078},[1457,1112],[57,37080,574],{"className":37081},[1075,1112],[57,37083,37084,37087],{"style":16230},[57,37085],{"className":37086,"style":16221},[1104],[57,37088,37089],{},[57,37090,2549],{"className":37091},[1888,2633,16239],[57,37093,37094,37097],{"style":17972},[57,37095],{"className":37096,"style":16221},[1104],[57,37098,37100],{"className":37099},[1109,1110,1111,1112],[57,37101,37103],{"className":37102},[1075,1112],[57,37104,37106,37109],{"className":37105},[1075,1112],[57,37107,738],{"className":37108,"style":3907},[1075,1148,1112],[57,37110,37112],{"className":37111},[1152],[57,37113,37115,37135],{"className":37114},[1087,1088],[57,37116,37118,37132],{"className":37117},[1092],[57,37119,37121],{"className":37120,"style":5022},[1096],[57,37122,37123,37126],{"style":18002},[57,37124],{"className":37125,"style":1169},[1104],[57,37127,37129],{"className":37128},[1109,1173,1174,1112],[57,37130,1288],{"className":37131},[1075,1148,1112],[57,37133,1183],{"className":37134},[1182],[57,37136,37138],{"className":37137},[1092],[57,37139,37141],{"className":37140,"style":5044},[1096],[57,37142],{},[57,37144,1183],{"className":37145},[1182],[57,37147,37149],{"className":37148},[1092],[57,37150,37152],{"className":37151,"style":17183},[1096],[57,37153],{},[57,37155],{"className":37156,"style":1853},[1452],[57,37158,37160,37163],{"className":37159},[1075],[57,37161,1730],{"className":37162},[1075,1148],[57,37164,37166],{"className":37165},[1152],[57,37167,37169,37209],{"className":37168},[1087,1088],[57,37170,37172,37206],{"className":37171},[1092],[57,37173,37175,37186],{"className":37174,"style":25824},[1096],[57,37176,37177,37180],{"style":18101},[57,37178],{"className":37179,"style":1493},[1104],[57,37181,37183],{"className":37182},[1109,1110,1111,1112],[57,37184,14745],{"className":37185,"style":3932},[1075,1148,1112],[57,37187,37188,37191],{"style":17216},[57,37189],{"className":37190,"style":1493},[1104],[57,37192,37194],{"className":37193},[1109,1110,1111,1112],[57,37195,37197,37200,37203],{"className":37196},[1075,1112],[57,37198,28613],{"className":37199},[1075,1148,1112],[57,37201,1730],{"className":37202},[1075,1148,1112],[57,37204,6336],{"className":37205,"style":6749},[1075,1148,1112],[57,37207,1183],{"className":37208},[1182],[57,37210,37212],{"className":37211},[1092],[57,37213,37215],{"className":37214,"style":18131},[1096],[57,37216],{},[57,37218,1754],{"className":37219},[1079],[57,37221,37223,37226],{"className":37222},[1075],[57,37224,13535],{"className":37225},[1075,1148],[57,37227,37229],{"className":37228},[1152],[57,37230,37232,37263],{"className":37231},[1087,1088],[57,37233,37235,37260],{"className":37234},[1092],[57,37236,37238,37249],{"className":37237,"style":18098},[1096],[57,37239,37240,37243],{"style":18101},[57,37241],{"className":37242,"style":1493},[1104],[57,37244,37246],{"className":37245},[1109,1110,1111,1112],[57,37247,1288],{"className":37248},[1075,1148,1112],[57,37250,37251,37254],{"style":17216},[57,37252],{"className":37253,"style":1493},[1104],[57,37255,37257],{"className":37256},[1109,1110,1111,1112],[57,37258,1733],{"className":37259},[1075,1148,1112],[57,37261,1183],{"className":37262},[1182],[57,37264,37266],{"className":37265},[1092],[57,37267,37269],{"className":37268,"style":18131},[1096],[57,37270],{},[57,37272,1759],{"className":37273},[1849],[57,37275],{"className":37276,"style":1853},[1452],[57,37278,37280,37283],{"className":37279},[1075],[57,37281,18],{"className":37282},[1075,1148],[57,37284,37286],{"className":37285},[1152],[57,37287,37289,37320],{"className":37288},[1087,1088],[57,37290,37292,37317],{"className":37291},[1092],[57,37293,37295,37306],{"className":37294,"style":18098},[1096],[57,37296,37297,37300],{"style":18101},[57,37298],{"className":37299,"style":1493},[1104],[57,37301,37303],{"className":37302},[1109,1110,1111,1112],[57,37304,1288],{"className":37305},[1075,1148,1112],[57,37307,37308,37311],{"style":17216},[57,37309],{"className":37310,"style":1493},[1104],[57,37312,37314],{"className":37313},[1109,1110,1111,1112],[57,37315,1733],{"className":37316},[1075,1148,1112],[57,37318,1183],{"className":37319},[1182],[57,37321,37323],{"className":37322},[1092],[57,37324,37326],{"className":37325,"style":18131},[1096],[57,37327],{},[57,37329,1768],{"className":37330},[1269],[57,37332,15169],{"className":37333},[1075],[57,37335],{"className":37336,"style":1853},[1452],[57,37338,17640,37340],{"className":37339},[1888],[57,37341,17644],{"style":17643},[57,37343],{"className":37344,"style":1853},[1452],[57,37346,37348,37351],{"className":37347},[1075],[57,37349,6333],{"className":37350,"style":3907},[1075,1148],[57,37352,37354],{"className":37353},[1152],[57,37355,37357,37377],{"className":37356},[1087,1088],[57,37358,37360,37374],{"className":37359},[1092],[57,37361,37363],{"className":37362,"style":2208},[1096],[57,37364,37365,37368],{"style":8131},[57,37366],{"className":37367,"style":1493},[1104],[57,37369,37371],{"className":37370},[1109,1110,1111,1112],[57,37372,14745],{"className":37373,"style":3932},[1075,1148,1112],[57,37375,1183],{"className":37376},[1182],[57,37378,37380],{"className":37379},[1092],[57,37381,37383],{"className":37382,"style":2230},[1096],[57,37384],{},[57,37386,1754],{"className":37387},[1079],[57,37389,37391,37394],{"className":37390},[1075],[57,37392,18],{"className":37393},[1075,1148],[57,37395,37397],{"className":37396},[1152],[57,37398,37400,37431],{"className":37399},[1087,1088],[57,37401,37403,37428],{"className":37402},[1092],[57,37404,37406,37417],{"className":37405,"style":18098},[1096],[57,37407,37408,37411],{"style":18101},[57,37409],{"className":37410,"style":1493},[1104],[57,37412,37414],{"className":37413},[1109,1110,1111,1112],[57,37415,1288],{"className":37416},[1075,1148,1112],[57,37418,37419,37422],{"style":17216},[57,37420],{"className":37421,"style":1493},[1104],[57,37423,37425],{"className":37424},[1109,1110,1111,1112],[57,37426,1733],{"className":37427},[1075,1148,1112],[57,37429,1183],{"className":37430},[1182],[57,37432,37434],{"className":37433},[1092],[57,37435,37437],{"className":37436,"style":18131},[1096],[57,37438],{},[57,37440,13675],{"className":37441},[1075],[57,37443,37445,37448],{"className":37444},[1075],[57,37446,13535],{"className":37447},[1075,1148],[57,37449,37451],{"className":37450},[1152],[57,37452,37454,37485],{"className":37453},[1087,1088],[57,37455,37457,37482],{"className":37456},[1092],[57,37458,37460,37471],{"className":37459,"style":18098},[1096],[57,37461,37462,37465],{"style":18101},[57,37463],{"className":37464,"style":1493},[1104],[57,37466,37468],{"className":37467},[1109,1110,1111,1112],[57,37469,1288],{"className":37470},[1075,1148,1112],[57,37472,37473,37476],{"style":17216},[57,37474],{"className":37475,"style":1493},[1104],[57,37477,37479],{"className":37478},[1109,1110,1111,1112],[57,37480,1733],{"className":37481},[1075,1148,1112],[57,37483,1183],{"className":37484},[1182],[57,37486,37488],{"className":37487},[1092],[57,37489,37491],{"className":37490,"style":18131},[1096],[57,37492],{},[57,37494,1768],{"className":37495},[1269],[57,37497,37498,37501],{"style":36855},[57,37499],{"className":37500,"style":15959},[1104],[57,37502,37504,37507,37510,37513,37516,37578,37581,37648,37651,37755,37758,37850,37853,37904,37907,37910,37961,37964,38356,38359,38362,38367,38370,38410,38413,38464,38467,38518],{"className":37503},[1075],[57,37505],{"className":37506},[1075],[57,37508],{"className":37509,"style":1453},[1452],[57,37511,281],{"className":37512},[1457],[57,37514],{"className":37515,"style":1453},[1452],[57,37517,37519,37522,37575],{"className":37518},[1075],[57,37520],{"className":37521},[1079,1080],[57,37523,37525],{"className":37524},[1035],[57,37526,37528,37567],{"className":37527},[1087,1088],[57,37529,37531,37564],{"className":37530},[1092],[57,37532,37534,37545,37553],{"className":37533,"style":17068},[1096],[57,37535,37536,37539],{"style":16608},[57,37537],{"className":37538,"style":1105},[1104],[57,37540,37542],{"className":37541},[1075],[57,37543,7990],{"className":37544,"style":17080},[1075,1148],[57,37546,37547,37550],{"style":1230},[57,37548],{"className":37549,"style":1105},[1104],[57,37551],{"className":37552,"style":1238},[1237],[57,37554,37555,37558],{"style":16674},[57,37556],{"className":37557,"style":1105},[1104],[57,37559,37561],{"className":37560},[1075],[57,37562,574],{"className":37563},[1075],[57,37565,1183],{"className":37566},[1182],[57,37568,37570],{"className":37569},[1092],[57,37571,37573],{"className":37572,"style":17109},[1096],[57,37574],{},[57,37576],{"className":37577},[1269,1080],[57,37579],{"className":37580,"style":1853},[1452],[57,37582,37584],{"className":37583},[1888,15027],[57,37585,37587,37640],{"className":37586},[1087,1088],[57,37588,37590,37637],{"className":37589},[1092],[57,37591,37593,37613,37623],{"className":37592,"style":17130},[1096],[57,37594,37595,37598],{"style":17133},[57,37596],{"className":37597,"style":16221},[1104],[57,37599,37601],{"className":37600},[1109,1110,1111,1112],[57,37602,37604,37607,37610],{"className":37603},[1075,1112],[57,37605,1288],{"className":37606},[1075,1148,1112],[57,37608,281],{"className":37609},[1457,1112],[57,37611,574],{"className":37612},[1075,1112],[57,37614,37615,37618],{"style":16230},[57,37616],{"className":37617,"style":16221},[1104],[57,37619,37620],{},[57,37621,2549],{"className":37622},[1888,2633,16239],[57,37624,37625,37628],{"style":17164},[57,37626],{"className":37627,"style":16221},[1104],[57,37629,37631],{"className":37630},[1109,1110,1111,1112],[57,37632,37634],{"className":37633},[1075,1112],[57,37635,7990],{"className":37636,"style":17080},[1075,1148,1112],[57,37638,1183],{"className":37639},[1182],[57,37641,37643],{"className":37642},[1092],[57,37644,37646],{"className":37645,"style":17183},[1096],[57,37647],{},[57,37649],{"className":37650,"style":1853},[1452],[57,37652,37654],{"className":37653},[1888,15027],[57,37655,37657,37747],{"className":37656},[1087,1088],[57,37658,37660,37744],{"className":37659},[1092],[57,37661,37663,37683,37693],{"className":37662,"style":17939},[1096],[57,37664,37665,37668],{"style":17133},[57,37666],{"className":37667,"style":16221},[1104],[57,37669,37671],{"className":37670},[1109,1110,1111,1112],[57,37672,37674,37677,37680],{"className":37673},[1075,1112],[57,37675,1733],{"className":37676},[1075,1148,1112],[57,37678,281],{"className":37679},[1457,1112],[57,37681,574],{"className":37682},[1075,1112],[57,37684,37685,37688],{"style":16230},[57,37686],{"className":37687,"style":16221},[1104],[57,37689,37690],{},[57,37691,2549],{"className":37692},[1888,2633,16239],[57,37694,37695,37698],{"style":17972},[57,37696],{"className":37697,"style":16221},[1104],[57,37699,37701],{"className":37700},[1109,1110,1111,1112],[57,37702,37704],{"className":37703},[1075,1112],[57,37705,37707,37710],{"className":37706},[1075,1112],[57,37708,738],{"className":37709,"style":3907},[1075,1148,1112],[57,37711,37713],{"className":37712},[1152],[57,37714,37716,37736],{"className":37715},[1087,1088],[57,37717,37719,37733],{"className":37718},[1092],[57,37720,37722],{"className":37721,"style":5022},[1096],[57,37723,37724,37727],{"style":18002},[57,37725],{"className":37726,"style":1169},[1104],[57,37728,37730],{"className":37729},[1109,1173,1174,1112],[57,37731,1288],{"className":37732},[1075,1148,1112],[57,37734,1183],{"className":37735},[1182],[57,37737,37739],{"className":37738},[1092],[57,37740,37742],{"className":37741,"style":5044},[1096],[57,37743],{},[57,37745,1183],{"className":37746},[1182],[57,37748,37750],{"className":37749},[1092],[57,37751,37753],{"className":37752,"style":17183},[1096],[57,37754],{},[57,37756],{"className":37757,"style":1853},[1452],[57,37759,37761,37764],{"className":37760},[1075],[57,37762,1730],{"className":37763},[1075,1148],[57,37765,37767],{"className":37766},[1152],[57,37768,37770,37842],{"className":37769},[1087,1088],[57,37771,37773,37839],{"className":37772},[1092],[57,37774,37776,37819],{"className":37775,"style":25824},[1096],[57,37777,37778,37781],{"style":18101},[57,37779],{"className":37780,"style":1493},[1104],[57,37782,37784],{"className":37783},[1109,1110,1111,1112],[57,37785,37787],{"className":37786},[1075,1112],[57,37788,37790,37793],{"className":37789},[1075,1112],[57,37791,14745],{"className":37792,"style":3932},[1075,1148,1112],[57,37794,37796],{"className":37795},[1152],[57,37797,37799],{"className":37798},[1087],[57,37800,37802],{"className":37801},[1092],[57,37803,37805],{"className":37804,"style":19935},[1096],[57,37806,37807,37810],{"style":19938},[57,37808],{"className":37809,"style":1169},[1104],[57,37811,37813],{"className":37812},[1109,1173,1174,1112],[57,37814,37816],{"className":37815},[1075,1112],[57,37817,19775],{"className":37818},[1075,1112],[57,37820,37821,37824],{"style":17216},[57,37822],{"className":37823,"style":1493},[1104],[57,37825,37827],{"className":37826},[1109,1110,1111,1112],[57,37828,37830,37833,37836],{"className":37829},[1075,1112],[57,37831,28613],{"className":37832},[1075,1148,1112],[57,37834,1730],{"className":37835},[1075,1148,1112],[57,37837,6336],{"className":37838,"style":6749},[1075,1148,1112],[57,37840,1183],{"className":37841},[1182],[57,37843,37845],{"className":37844},[1092],[57,37846,37848],{"className":37847,"style":18131},[1096],[57,37849],{},[57,37851,1754],{"className":37852},[1079],[57,37854,37856,37859],{"className":37855},[1075],[57,37857,13535],{"className":37858},[1075,1148],[57,37860,37862],{"className":37861},[1152],[57,37863,37865,37896],{"className":37864},[1087,1088],[57,37866,37868,37893],{"className":37867},[1092],[57,37869,37871,37882],{"className":37870,"style":18098},[1096],[57,37872,37873,37876],{"style":18101},[57,37874],{"className":37875,"style":1493},[1104],[57,37877,37879],{"className":37878},[1109,1110,1111,1112],[57,37880,1288],{"className":37881},[1075,1148,1112],[57,37883,37884,37887],{"style":17216},[57,37885],{"className":37886,"style":1493},[1104],[57,37888,37890],{"className":37889},[1109,1110,1111,1112],[57,37891,1733],{"className":37892},[1075,1148,1112],[57,37894,1183],{"className":37895},[1182],[57,37897,37899],{"className":37898},[1092],[57,37900,37902],{"className":37901,"style":18131},[1096],[57,37903],{},[57,37905,1759],{"className":37906},[1849],[57,37908],{"className":37909,"style":1853},[1452],[57,37911,37913,37916],{"className":37912},[1075],[57,37914,18],{"className":37915},[1075,1148],[57,37917,37919],{"className":37918},[1152],[57,37920,37922,37953],{"className":37921},[1087,1088],[57,37923,37925,37950],{"className":37924},[1092],[57,37926,37928,37939],{"className":37927,"style":18098},[1096],[57,37929,37930,37933],{"style":18101},[57,37931],{"className":37932,"style":1493},[1104],[57,37934,37936],{"className":37935},[1109,1110,1111,1112],[57,37937,1288],{"className":37938},[1075,1148,1112],[57,37940,37941,37944],{"style":17216},[57,37942],{"className":37943,"style":1493},[1104],[57,37945,37947],{"className":37946},[1109,1110,1111,1112],[57,37948,1733],{"className":37949},[1075,1148,1112],[57,37951,1183],{"className":37952},[1182],[57,37954,37956],{"className":37955},[1092],[57,37957,37959],{"className":37958,"style":18131},[1096],[57,37960],{},[57,37962,1768],{"className":37963},[1269],[57,37965,37967,37970,38353],{"className":37966},[1075],[57,37968],{"className":37969},[1079,1080],[57,37971,37973],{"className":37972},[1035],[57,37974,37976,38345],{"className":37975},[1087,1088],[57,37977,37979,38342],{"className":37978},[1092],[57,37980,37983,38175,38183],{"className":37981,"style":37982},[1096],"height:1.4706em;",[57,37984,37985,37988],{"style":16608},[57,37986],{"className":37987,"style":1105},[1104],[57,37989,37991,38063,38066,38118,38121,38172],{"className":37990},[1075],[57,37992,37994,37997],{"className":37993},[1075],[57,37995,6333],{"className":37996,"style":3907},[1075,1148],[57,37998,38000],{"className":37999},[1152],[57,38001,38003,38055],{"className":38002},[1087,1088],[57,38004,38006,38052],{"className":38005},[1092],[57,38007,38009],{"className":38008,"style":2208},[1096],[57,38010,38011,38014],{"style":8131},[57,38012],{"className":38013,"style":1493},[1104],[57,38015,38017],{"className":38016},[1109,1110,1111,1112],[57,38018,38020],{"className":38019},[1075,1112],[57,38021,38023,38026],{"className":38022},[1075,1112],[57,38024,14745],{"className":38025,"style":3932},[1075,1148,1112],[57,38027,38029],{"className":38028},[1152],[57,38030,38032],{"className":38031},[1087],[57,38033,38035],{"className":38034},[1092],[57,38036,38038],{"className":38037,"style":19935},[1096],[57,38039,38040,38043],{"style":19938},[57,38041],{"className":38042,"style":1169},[1104],[57,38044,38046],{"className":38045},[1109,1173,1174,1112],[57,38047,38049],{"className":38048},[1075,1112],[57,38050,19775],{"className":38051},[1075,1112],[57,38053,1183],{"className":38054},[1182],[57,38056,38058],{"className":38057},[1092],[57,38059,38061],{"className":38060,"style":2230},[1096],[57,38062],{},[57,38064,1754],{"className":38065},[1079],[57,38067,38069,38072],{"className":38068},[1075],[57,38070,18],{"className":38071},[1075,1148],[57,38073,38075],{"className":38074},[1152],[57,38076,38078,38110],{"className":38077},[1087,1088],[57,38079,38081,38107],{"className":38080},[1092],[57,38082,38085,38096],{"className":38083,"style":38084},[1096],"height:0.7196em;",[57,38086,38087,38090],{"style":18101},[57,38088],{"className":38089,"style":1493},[1104],[57,38091,38093],{"className":38092},[1109,1110,1111,1112],[57,38094,1288],{"className":38095},[1075,1148,1112],[57,38097,38098,38101],{"style":17319},[57,38099],{"className":38100,"style":1493},[1104],[57,38102,38104],{"className":38103},[1109,1110,1111,1112],[57,38105,1733],{"className":38106},[1075,1148,1112],[57,38108,1183],{"className":38109},[1182],[57,38111,38113],{"className":38112},[1092],[57,38114,38116],{"className":38115,"style":18131},[1096],[57,38117],{},[57,38119,13675],{"className":38120},[1075],[57,38122,38124,38127],{"className":38123},[1075],[57,38125,13535],{"className":38126},[1075,1148],[57,38128,38130],{"className":38129},[1152],[57,38131,38133,38164],{"className":38132},[1087,1088],[57,38134,38136,38161],{"className":38135},[1092],[57,38137,38139,38150],{"className":38138,"style":38084},[1096],[57,38140,38141,38144],{"style":18101},[57,38142],{"className":38143,"style":1493},[1104],[57,38145,38147],{"className":38146},[1109,1110,1111,1112],[57,38148,1288],{"className":38149},[1075,1148,1112],[57,38151,38152,38155],{"style":17319},[57,38153],{"className":38154,"style":1493},[1104],[57,38156,38158],{"className":38157},[1109,1110,1111,1112],[57,38159,1733],{"className":38160},[1075,1148,1112],[57,38162,1183],{"className":38163},[1182],[57,38165,38167],{"className":38166},[1092],[57,38168,38170],{"className":38169,"style":18131},[1096],[57,38171],{},[57,38173,1768],{"className":38174},[1269],[57,38176,38177,38180],{"style":1230},[57,38178],{"className":38179,"style":1105},[1104],[57,38181],{"className":38182,"style":1238},[1237],[57,38184,38185,38188],{"style":16674},[57,38186],{"className":38187,"style":1105},[1104],[57,38189,38191,38231,38234,38285,38288,38339],{"className":38190},[1075],[57,38192,38194,38197],{"className":38193},[1075],[57,38195,6333],{"className":38196,"style":3907},[1075,1148],[57,38198,38200],{"className":38199},[1152],[57,38201,38203,38223],{"className":38202},[1087,1088],[57,38204,38206,38220],{"className":38205},[1092],[57,38207,38209],{"className":38208,"style":2208},[1096],[57,38210,38211,38214],{"style":8131},[57,38212],{"className":38213,"style":1493},[1104],[57,38215,38217],{"className":38216},[1109,1110,1111,1112],[57,38218,14745],{"className":38219,"style":3932},[1075,1148,1112],[57,38221,1183],{"className":38222},[1182],[57,38224,38226],{"className":38225},[1092],[57,38227,38229],{"className":38228,"style":2230},[1096],[57,38230],{},[57,38232,1754],{"className":38233},[1079],[57,38235,38237,38240],{"className":38236},[1075],[57,38238,18],{"className":38239},[1075,1148],[57,38241,38243],{"className":38242},[1152],[57,38244,38246,38277],{"className":38245},[1087,1088],[57,38247,38249,38274],{"className":38248},[1092],[57,38250,38252,38263],{"className":38251,"style":19640},[1096],[57,38253,38254,38257],{"style":18101},[57,38255],{"className":38256,"style":1493},[1104],[57,38258,38260],{"className":38259},[1109,1110,1111,1112],[57,38261,1288],{"className":38262},[1075,1148,1112],[57,38264,38265,38268],{"style":1489},[57,38266],{"className":38267,"style":1493},[1104],[57,38269,38271],{"className":38270},[1109,1110,1111,1112],[57,38272,1733],{"className":38273},[1075,1148,1112],[57,38275,1183],{"className":38276},[1182],[57,38278,38280],{"className":38279},[1092],[57,38281,38283],{"className":38282,"style":18131},[1096],[57,38284],{},[57,38286,13675],{"className":38287},[1075],[57,38289,38291,38294],{"className":38290},[1075],[57,38292,13535],{"className":38293},[1075,1148],[57,38295,38297],{"className":38296},[1152],[57,38298,38300,38331],{"className":38299},[1087,1088],[57,38301,38303,38328],{"className":38302},[1092],[57,38304,38306,38317],{"className":38305,"style":19640},[1096],[57,38307,38308,38311],{"style":18101},[57,38309],{"className":38310,"style":1493},[1104],[57,38312,38314],{"className":38313},[1109,1110,1111,1112],[57,38315,1288],{"className":38316},[1075,1148,1112],[57,38318,38319,38322],{"style":1489},[57,38320],{"className":38321,"style":1493},[1104],[57,38323,38325],{"className":38324},[1109,1110,1111,1112],[57,38326,1733],{"className":38327},[1075,1148,1112],[57,38329,1183],{"className":38330},[1182],[57,38332,38334],{"className":38333},[1092],[57,38335,38337],{"className":38336,"style":18131},[1096],[57,38338],{},[57,38340,1768],{"className":38341},[1269],[57,38343,1183],{"className":38344},[1182],[57,38346,38348],{"className":38347},[1092],[57,38349,38351],{"className":38350,"style":16739},[1096],[57,38352],{},[57,38354],{"className":38355},[1269,1080],[57,38357,15169],{"className":38358},[1075],[57,38360],{"className":38361,"style":1853},[1452],[57,38363,17640,38365],{"className":38364},[1888],[57,38366,17644],{"style":17643},[57,38368],{"className":38369,"style":1853},[1452],[57,38371,38373,38376],{"className":38372},[1075],[57,38374,6333],{"className":38375,"style":3907},[1075,1148],[57,38377,38379],{"className":38378},[1152],[57,38380,38382,38402],{"className":38381},[1087,1088],[57,38383,38385,38399],{"className":38384},[1092],[57,38386,38388],{"className":38387,"style":2208},[1096],[57,38389,38390,38393],{"style":8131},[57,38391],{"className":38392,"style":1493},[1104],[57,38394,38396],{"className":38395},[1109,1110,1111,1112],[57,38397,14745],{"className":38398,"style":3932},[1075,1148,1112],[57,38400,1183],{"className":38401},[1182],[57,38403,38405],{"className":38404},[1092],[57,38406,38408],{"className":38407,"style":2230},[1096],[57,38409],{},[57,38411,1754],{"className":38412},[1079],[57,38414,38416,38419],{"className":38415},[1075],[57,38417,18],{"className":38418},[1075,1148],[57,38420,38422],{"className":38421},[1152],[57,38423,38425,38456],{"className":38424},[1087,1088],[57,38426,38428,38453],{"className":38427},[1092],[57,38429,38431,38442],{"className":38430,"style":18098},[1096],[57,38432,38433,38436],{"style":18101},[57,38434],{"className":38435,"style":1493},[1104],[57,38437,38439],{"className":38438},[1109,1110,1111,1112],[57,38440,1288],{"className":38441},[1075,1148,1112],[57,38443,38444,38447],{"style":17216},[57,38445],{"className":38446,"style":1493},[1104],[57,38448,38450],{"className":38449},[1109,1110,1111,1112],[57,38451,1733],{"className":38452},[1075,1148,1112],[57,38454,1183],{"className":38455},[1182],[57,38457,38459],{"className":38458},[1092],[57,38460,38462],{"className":38461,"style":18131},[1096],[57,38463],{},[57,38465,13675],{"className":38466},[1075],[57,38468,38470,38473],{"className":38469},[1075],[57,38471,13535],{"className":38472},[1075,1148],[57,38474,38476],{"className":38475},[1152],[57,38477,38479,38510],{"className":38478},[1087,1088],[57,38480,38482,38507],{"className":38481},[1092],[57,38483,38485,38496],{"className":38484,"style":18098},[1096],[57,38486,38487,38490],{"style":18101},[57,38488],{"className":38489,"style":1493},[1104],[57,38491,38493],{"className":38492},[1109,1110,1111,1112],[57,38494,1288],{"className":38495},[1075,1148,1112],[57,38497,38498,38501],{"style":17216},[57,38499],{"className":38500,"style":1493},[1104],[57,38502,38504],{"className":38503},[1109,1110,1111,1112],[57,38505,1733],{"className":38506},[1075,1148,1112],[57,38508,1183],{"className":38509},[1182],[57,38511,38513],{"className":38512},[1092],[57,38514,38516],{"className":38515,"style":18131},[1096],[57,38517],{},[57,38519,1768],{"className":38520},[1269],[57,38522,38523,38526],{"style":36864},[57,38524],{"className":38525,"style":15959},[1104],[57,38527,38529,38532,38535,38538,38541,38603,38606,38673,38676,38780,38783,38875,38878,38929,38932,38935,38986,38989,39379],{"className":38528},[1075],[57,38530],{"className":38531},[1075],[57,38533],{"className":38534,"style":1453},[1452],[57,38536,281],{"className":38537},[1457],[57,38539],{"className":38540,"style":1453},[1452],[57,38542,38544,38547,38600],{"className":38543},[1075],[57,38545],{"className":38546},[1079,1080],[57,38548,38550],{"className":38549},[1035],[57,38551,38553,38592],{"className":38552},[1087,1088],[57,38554,38556,38589],{"className":38555},[1092],[57,38557,38559,38570,38578],{"className":38558,"style":17068},[1096],[57,38560,38561,38564],{"style":16608},[57,38562],{"className":38563,"style":1105},[1104],[57,38565,38567],{"className":38566},[1075],[57,38568,7990],{"className":38569,"style":17080},[1075,1148],[57,38571,38572,38575],{"style":1230},[57,38573],{"className":38574,"style":1105},[1104],[57,38576],{"className":38577,"style":1238},[1237],[57,38579,38580,38583],{"style":16674},[57,38581],{"className":38582,"style":1105},[1104],[57,38584,38586],{"className":38585},[1075],[57,38587,574],{"className":38588},[1075],[57,38590,1183],{"className":38591},[1182],[57,38593,38595],{"className":38594},[1092],[57,38596,38598],{"className":38597,"style":17109},[1096],[57,38599],{},[57,38601],{"className":38602},[1269,1080],[57,38604],{"className":38605,"style":1853},[1452],[57,38607,38609],{"className":38608},[1888,15027],[57,38610,38612,38665],{"className":38611},[1087,1088],[57,38613,38615,38662],{"className":38614},[1092],[57,38616,38618,38638,38648],{"className":38617,"style":17130},[1096],[57,38619,38620,38623],{"style":17133},[57,38621],{"className":38622,"style":16221},[1104],[57,38624,38626],{"className":38625},[1109,1110,1111,1112],[57,38627,38629,38632,38635],{"className":38628},[1075,1112],[57,38630,1288],{"className":38631},[1075,1148,1112],[57,38633,281],{"className":38634},[1457,1112],[57,38636,574],{"className":38637},[1075,1112],[57,38639,38640,38643],{"style":16230},[57,38641],{"className":38642,"style":16221},[1104],[57,38644,38645],{},[57,38646,2549],{"className":38647},[1888,2633,16239],[57,38649,38650,38653],{"style":17164},[57,38651],{"className":38652,"style":16221},[1104],[57,38654,38656],{"className":38655},[1109,1110,1111,1112],[57,38657,38659],{"className":38658},[1075,1112],[57,38660,7990],{"className":38661,"style":17080},[1075,1148,1112],[57,38663,1183],{"className":38664},[1182],[57,38666,38668],{"className":38667},[1092],[57,38669,38671],{"className":38670,"style":17183},[1096],[57,38672],{},[57,38674],{"className":38675,"style":1853},[1452],[57,38677,38679],{"className":38678},[1888,15027],[57,38680,38682,38772],{"className":38681},[1087,1088],[57,38683,38685,38769],{"className":38684},[1092],[57,38686,38688,38708,38718],{"className":38687,"style":17939},[1096],[57,38689,38690,38693],{"style":17133},[57,38691],{"className":38692,"style":16221},[1104],[57,38694,38696],{"className":38695},[1109,1110,1111,1112],[57,38697,38699,38702,38705],{"className":38698},[1075,1112],[57,38700,1733],{"className":38701},[1075,1148,1112],[57,38703,281],{"className":38704},[1457,1112],[57,38706,574],{"className":38707},[1075,1112],[57,38709,38710,38713],{"style":16230},[57,38711],{"className":38712,"style":16221},[1104],[57,38714,38715],{},[57,38716,2549],{"className":38717},[1888,2633,16239],[57,38719,38720,38723],{"style":17972},[57,38721],{"className":38722,"style":16221},[1104],[57,38724,38726],{"className":38725},[1109,1110,1111,1112],[57,38727,38729],{"className":38728},[1075,1112],[57,38730,38732,38735],{"className":38731},[1075,1112],[57,38733,738],{"className":38734,"style":3907},[1075,1148,1112],[57,38736,38738],{"className":38737},[1152],[57,38739,38741,38761],{"className":38740},[1087,1088],[57,38742,38744,38758],{"className":38743},[1092],[57,38745,38747],{"className":38746,"style":5022},[1096],[57,38748,38749,38752],{"style":18002},[57,38750],{"className":38751,"style":1169},[1104],[57,38753,38755],{"className":38754},[1109,1173,1174,1112],[57,38756,1288],{"className":38757},[1075,1148,1112],[57,38759,1183],{"className":38760},[1182],[57,38762,38764],{"className":38763},[1092],[57,38765,38767],{"className":38766,"style":5044},[1096],[57,38768],{},[57,38770,1183],{"className":38771},[1182],[57,38773,38775],{"className":38774},[1092],[57,38776,38778],{"className":38777,"style":17183},[1096],[57,38779],{},[57,38781],{"className":38782,"style":1853},[1452],[57,38784,38786,38789],{"className":38785},[1075],[57,38787,1730],{"className":38788},[1075,1148],[57,38790,38792],{"className":38791},[1152],[57,38793,38795,38867],{"className":38794},[1087,1088],[57,38796,38798,38864],{"className":38797},[1092],[57,38799,38801,38844],{"className":38800,"style":25824},[1096],[57,38802,38803,38806],{"style":18101},[57,38804],{"className":38805,"style":1493},[1104],[57,38807,38809],{"className":38808},[1109,1110,1111,1112],[57,38810,38812],{"className":38811},[1075,1112],[57,38813,38815,38818],{"className":38814},[1075,1112],[57,38816,14745],{"className":38817,"style":3932},[1075,1148,1112],[57,38819,38821],{"className":38820},[1152],[57,38822,38824],{"className":38823},[1087],[57,38825,38827],{"className":38826},[1092],[57,38828,38830],{"className":38829,"style":19935},[1096],[57,38831,38832,38835],{"style":19938},[57,38833],{"className":38834,"style":1169},[1104],[57,38836,38838],{"className":38837},[1109,1173,1174,1112],[57,38839,38841],{"className":38840},[1075,1112],[57,38842,19775],{"className":38843},[1075,1112],[57,38845,38846,38849],{"style":17216},[57,38847],{"className":38848,"style":1493},[1104],[57,38850,38852],{"className":38851},[1109,1110,1111,1112],[57,38853,38855,38858,38861],{"className":38854},[1075,1112],[57,38856,28613],{"className":38857},[1075,1148,1112],[57,38859,1730],{"className":38860},[1075,1148,1112],[57,38862,6336],{"className":38863,"style":6749},[1075,1148,1112],[57,38865,1183],{"className":38866},[1182],[57,38868,38870],{"className":38869},[1092],[57,38871,38873],{"className":38872,"style":18131},[1096],[57,38874],{},[57,38876,1754],{"className":38877},[1079],[57,38879,38881,38884],{"className":38880},[1075],[57,38882,13535],{"className":38883},[1075,1148],[57,38885,38887],{"className":38886},[1152],[57,38888,38890,38921],{"className":38889},[1087,1088],[57,38891,38893,38918],{"className":38892},[1092],[57,38894,38896,38907],{"className":38895,"style":18098},[1096],[57,38897,38898,38901],{"style":18101},[57,38899],{"className":38900,"style":1493},[1104],[57,38902,38904],{"className":38903},[1109,1110,1111,1112],[57,38905,1288],{"className":38906},[1075,1148,1112],[57,38908,38909,38912],{"style":17216},[57,38910],{"className":38911,"style":1493},[1104],[57,38913,38915],{"className":38914},[1109,1110,1111,1112],[57,38916,1733],{"className":38917},[1075,1148,1112],[57,38919,1183],{"className":38920},[1182],[57,38922,38924],{"className":38923},[1092],[57,38925,38927],{"className":38926,"style":18131},[1096],[57,38928],{},[57,38930,1759],{"className":38931},[1849],[57,38933],{"className":38934,"style":1853},[1452],[57,38936,38938,38941],{"className":38937},[1075],[57,38939,18],{"className":38940},[1075,1148],[57,38942,38944],{"className":38943},[1152],[57,38945,38947,38978],{"className":38946},[1087,1088],[57,38948,38950,38975],{"className":38949},[1092],[57,38951,38953,38964],{"className":38952,"style":18098},[1096],[57,38954,38955,38958],{"style":18101},[57,38956],{"className":38957,"style":1493},[1104],[57,38959,38961],{"className":38960},[1109,1110,1111,1112],[57,38962,1288],{"className":38963},[1075,1148,1112],[57,38965,38966,38969],{"style":17216},[57,38967],{"className":38968,"style":1493},[1104],[57,38970,38972],{"className":38971},[1109,1110,1111,1112],[57,38973,1733],{"className":38974},[1075,1148,1112],[57,38976,1183],{"className":38977},[1182],[57,38979,38981],{"className":38980},[1092],[57,38982,38984],{"className":38983,"style":18131},[1096],[57,38985],{},[57,38987,1768],{"className":38988},[1269],[57,38990,38992,38995,39376],{"className":38991},[1075],[57,38993],{"className":38994},[1079,1080],[57,38996,38998],{"className":38997},[1035],[57,38999,39001,39368],{"className":39000},[1087,1088],[57,39002,39004,39365],{"className":39003},[1092],[57,39005,39007,39198,39206],{"className":39006,"style":37982},[1096],[57,39008,39009,39012],{"style":16608},[57,39010],{"className":39011,"style":1105},[1104],[57,39013,39015,39087,39090,39141,39144,39195],{"className":39014},[1075],[57,39016,39018,39021],{"className":39017},[1075],[57,39019,6333],{"className":39020,"style":3907},[1075,1148],[57,39022,39024],{"className":39023},[1152],[57,39025,39027,39079],{"className":39026},[1087,1088],[57,39028,39030,39076],{"className":39029},[1092],[57,39031,39033],{"className":39032,"style":2208},[1096],[57,39034,39035,39038],{"style":8131},[57,39036],{"className":39037,"style":1493},[1104],[57,39039,39041],{"className":39040},[1109,1110,1111,1112],[57,39042,39044],{"className":39043},[1075,1112],[57,39045,39047,39050],{"className":39046},[1075,1112],[57,39048,14745],{"className":39049,"style":3932},[1075,1148,1112],[57,39051,39053],{"className":39052},[1152],[57,39054,39056],{"className":39055},[1087],[57,39057,39059],{"className":39058},[1092],[57,39060,39062],{"className":39061,"style":19935},[1096],[57,39063,39064,39067],{"style":19938},[57,39065],{"className":39066,"style":1169},[1104],[57,39068,39070],{"className":39069},[1109,1173,1174,1112],[57,39071,39073],{"className":39072},[1075,1112],[57,39074,19775],{"className":39075},[1075,1112],[57,39077,1183],{"className":39078},[1182],[57,39080,39082],{"className":39081},[1092],[57,39083,39085],{"className":39084,"style":2230},[1096],[57,39086],{},[57,39088,1754],{"className":39089},[1079],[57,39091,39093,39096],{"className":39092},[1075],[57,39094,18],{"className":39095},[1075,1148],[57,39097,39099],{"className":39098},[1152],[57,39100,39102,39133],{"className":39101},[1087,1088],[57,39103,39105,39130],{"className":39104},[1092],[57,39106,39108,39119],{"className":39107,"style":38084},[1096],[57,39109,39110,39113],{"style":18101},[57,39111],{"className":39112,"style":1493},[1104],[57,39114,39116],{"className":39115},[1109,1110,1111,1112],[57,39117,1288],{"className":39118},[1075,1148,1112],[57,39120,39121,39124],{"style":17319},[57,39122],{"className":39123,"style":1493},[1104],[57,39125,39127],{"className":39126},[1109,1110,1111,1112],[57,39128,1733],{"className":39129},[1075,1148,1112],[57,39131,1183],{"className":39132},[1182],[57,39134,39136],{"className":39135},[1092],[57,39137,39139],{"className":39138,"style":18131},[1096],[57,39140],{},[57,39142,13675],{"className":39143},[1075],[57,39145,39147,39150],{"className":39146},[1075],[57,39148,13535],{"className":39149},[1075,1148],[57,39151,39153],{"className":39152},[1152],[57,39154,39156,39187],{"className":39155},[1087,1088],[57,39157,39159,39184],{"className":39158},[1092],[57,39160,39162,39173],{"className":39161,"style":38084},[1096],[57,39163,39164,39167],{"style":18101},[57,39165],{"className":39166,"style":1493},[1104],[57,39168,39170],{"className":39169},[1109,1110,1111,1112],[57,39171,1288],{"className":39172},[1075,1148,1112],[57,39174,39175,39178],{"style":17319},[57,39176],{"className":39177,"style":1493},[1104],[57,39179,39181],{"className":39180},[1109,1110,1111,1112],[57,39182,1733],{"className":39183},[1075,1148,1112],[57,39185,1183],{"className":39186},[1182],[57,39188,39190],{"className":39189},[1092],[57,39191,39193],{"className":39192,"style":18131},[1096],[57,39194],{},[57,39196,1768],{"className":39197},[1269],[57,39199,39200,39203],{"style":1230},[57,39201],{"className":39202,"style":1105},[1104],[57,39204],{"className":39205,"style":1238},[1237],[57,39207,39208,39211],{"style":16674},[57,39209],{"className":39210,"style":1105},[1104],[57,39212,39214,39254,39257,39308,39311,39362],{"className":39213},[1075],[57,39215,39217,39220],{"className":39216},[1075],[57,39218,6333],{"className":39219,"style":3907},[1075,1148],[57,39221,39223],{"className":39222},[1152],[57,39224,39226,39246],{"className":39225},[1087,1088],[57,39227,39229,39243],{"className":39228},[1092],[57,39230,39232],{"className":39231,"style":2208},[1096],[57,39233,39234,39237],{"style":8131},[57,39235],{"className":39236,"style":1493},[1104],[57,39238,39240],{"className":39239},[1109,1110,1111,1112],[57,39241,14745],{"className":39242,"style":3932},[1075,1148,1112],[57,39244,1183],{"className":39245},[1182],[57,39247,39249],{"className":39248},[1092],[57,39250,39252],{"className":39251,"style":2230},[1096],[57,39253],{},[57,39255,1754],{"className":39256},[1079],[57,39258,39260,39263],{"className":39259},[1075],[57,39261,18],{"className":39262},[1075,1148],[57,39264,39266],{"className":39265},[1152],[57,39267,39269,39300],{"className":39268},[1087,1088],[57,39270,39272,39297],{"className":39271},[1092],[57,39273,39275,39286],{"className":39274,"style":19640},[1096],[57,39276,39277,39280],{"style":18101},[57,39278],{"className":39279,"style":1493},[1104],[57,39281,39283],{"className":39282},[1109,1110,1111,1112],[57,39284,1288],{"className":39285},[1075,1148,1112],[57,39287,39288,39291],{"style":1489},[57,39289],{"className":39290,"style":1493},[1104],[57,39292,39294],{"className":39293},[1109,1110,1111,1112],[57,39295,1733],{"className":39296},[1075,1148,1112],[57,39298,1183],{"className":39299},[1182],[57,39301,39303],{"className":39302},[1092],[57,39304,39306],{"className":39305,"style":18131},[1096],[57,39307],{},[57,39309,13675],{"className":39310},[1075],[57,39312,39314,39317],{"className":39313},[1075],[57,39315,13535],{"className":39316},[1075,1148],[57,39318,39320],{"className":39319},[1152],[57,39321,39323,39354],{"className":39322},[1087,1088],[57,39324,39326,39351],{"className":39325},[1092],[57,39327,39329,39340],{"className":39328,"style":19640},[1096],[57,39330,39331,39334],{"style":18101},[57,39332],{"className":39333,"style":1493},[1104],[57,39335,39337],{"className":39336},[1109,1110,1111,1112],[57,39338,1288],{"className":39339},[1075,1148,1112],[57,39341,39342,39345],{"style":1489},[57,39343],{"className":39344,"style":1493},[1104],[57,39346,39348],{"className":39347},[1109,1110,1111,1112],[57,39349,1733],{"className":39350},[1075,1148,1112],[57,39352,1183],{"className":39353},[1182],[57,39355,39357],{"className":39356},[1092],[57,39358,39360],{"className":39359,"style":18131},[1096],[57,39361],{},[57,39363,1768],{"className":39364},[1269],[57,39366,1183],{"className":39367},[1182],[57,39369,39371],{"className":39370},[1092],[57,39372,39374],{"className":39373,"style":16739},[1096],[57,39375],{},[57,39377],{"className":39378},[1269,1080],[57,39380,39382,39385,39737],{"className":39381},[1075],[57,39383],{"className":39384},[1079,1080],[57,39386,39388],{"className":39387},[1035],[57,39389,39391,39729],{"className":39390},[1087,1088],[57,39392,39394,39726],{"className":39393},[1092],[57,39395,39397,39556,39564],{"className":39396,"style":37982},[1096],[57,39398,39399,39402],{"style":16608},[57,39400],{"className":39401,"style":1105},[1104],[57,39403,39405,39445,39448,39499,39502,39553],{"className":39404},[1075],[57,39406,39408,39411],{"className":39407},[1075],[57,39409,6333],{"className":39410,"style":3907},[1075,1148],[57,39412,39414],{"className":39413},[1152],[57,39415,39417,39437],{"className":39416},[1087,1088],[57,39418,39420,39434],{"className":39419},[1092],[57,39421,39423],{"className":39422,"style":2208},[1096],[57,39424,39425,39428],{"style":8131},[57,39426],{"className":39427,"style":1493},[1104],[57,39429,39431],{"className":39430},[1109,1110,1111,1112],[57,39432,14745],{"className":39433,"style":3932},[1075,1148,1112],[57,39435,1183],{"className":39436},[1182],[57,39438,39440],{"className":39439},[1092],[57,39441,39443],{"className":39442,"style":2230},[1096],[57,39444],{},[57,39446,1754],{"className":39447},[1079],[57,39449,39451,39454],{"className":39450},[1075],[57,39452,18],{"className":39453},[1075,1148],[57,39455,39457],{"className":39456},[1152],[57,39458,39460,39491],{"className":39459},[1087,1088],[57,39461,39463,39488],{"className":39462},[1092],[57,39464,39466,39477],{"className":39465,"style":38084},[1096],[57,39467,39468,39471],{"style":18101},[57,39469],{"className":39470,"style":1493},[1104],[57,39472,39474],{"className":39473},[1109,1110,1111,1112],[57,39475,1288],{"className":39476},[1075,1148,1112],[57,39478,39479,39482],{"style":17319},[57,39480],{"className":39481,"style":1493},[1104],[57,39483,39485],{"className":39484},[1109,1110,1111,1112],[57,39486,1733],{"className":39487},[1075,1148,1112],[57,39489,1183],{"className":39490},[1182],[57,39492,39494],{"className":39493},[1092],[57,39495,39497],{"className":39496,"style":18131},[1096],[57,39498],{},[57,39500,13675],{"className":39501},[1075],[57,39503,39505,39508],{"className":39504},[1075],[57,39506,13535],{"className":39507},[1075,1148],[57,39509,39511],{"className":39510},[1152],[57,39512,39514,39545],{"className":39513},[1087,1088],[57,39515,39517,39542],{"className":39516},[1092],[57,39518,39520,39531],{"className":39519,"style":38084},[1096],[57,39521,39522,39525],{"style":18101},[57,39523],{"className":39524,"style":1493},[1104],[57,39526,39528],{"className":39527},[1109,1110,1111,1112],[57,39529,1288],{"className":39530},[1075,1148,1112],[57,39532,39533,39536],{"style":17319},[57,39534],{"className":39535,"style":1493},[1104],[57,39537,39539],{"className":39538},[1109,1110,1111,1112],[57,39540,1733],{"className":39541},[1075,1148,1112],[57,39543,1183],{"className":39544},[1182],[57,39546,39548],{"className":39547},[1092],[57,39549,39551],{"className":39550,"style":18131},[1096],[57,39552],{},[57,39554,1768],{"className":39555},[1269],[57,39557,39558,39561],{"style":1230},[57,39559],{"className":39560,"style":1105},[1104],[57,39562],{"className":39563,"style":1238},[1237],[57,39565,39566,39569],{"style":16674},[57,39567],{"className":39568,"style":1105},[1104],[57,39570,39572,39575,39615,39618,39669,39672,39723],{"className":39571},[1075],[57,39573,15169],{"className":39574},[1075],[57,39576,39578,39581],{"className":39577},[1075],[57,39579,6333],{"className":39580,"style":3907},[1075,1148],[57,39582,39584],{"className":39583},[1152],[57,39585,39587,39607],{"className":39586},[1087,1088],[57,39588,39590,39604],{"className":39589},[1092],[57,39591,39593],{"className":39592,"style":2208},[1096],[57,39594,39595,39598],{"style":8131},[57,39596],{"className":39597,"style":1493},[1104],[57,39599,39601],{"className":39600},[1109,1110,1111,1112],[57,39602,14745],{"className":39603,"style":3932},[1075,1148,1112],[57,39605,1183],{"className":39606},[1182],[57,39608,39610],{"className":39609},[1092],[57,39611,39613],{"className":39612,"style":2230},[1096],[57,39614],{},[57,39616,1754],{"className":39617},[1079],[57,39619,39621,39624],{"className":39620},[1075],[57,39622,18],{"className":39623},[1075,1148],[57,39625,39627],{"className":39626},[1152],[57,39628,39630,39661],{"className":39629},[1087,1088],[57,39631,39633,39658],{"className":39632},[1092],[57,39634,39636,39647],{"className":39635,"style":19640},[1096],[57,39637,39638,39641],{"style":18101},[57,39639],{"className":39640,"style":1493},[1104],[57,39642,39644],{"className":39643},[1109,1110,1111,1112],[57,39645,1288],{"className":39646},[1075,1148,1112],[57,39648,39649,39652],{"style":1489},[57,39650],{"className":39651,"style":1493},[1104],[57,39653,39655],{"className":39654},[1109,1110,1111,1112],[57,39656,1733],{"className":39657},[1075,1148,1112],[57,39659,1183],{"className":39660},[1182],[57,39662,39664],{"className":39663},[1092],[57,39665,39667],{"className":39666,"style":18131},[1096],[57,39668],{},[57,39670,13675],{"className":39671},[1075],[57,39673,39675,39678],{"className":39674},[1075],[57,39676,13535],{"className":39677},[1075,1148],[57,39679,39681],{"className":39680},[1152],[57,39682,39684,39715],{"className":39683},[1087,1088],[57,39685,39687,39712],{"className":39686},[1092],[57,39688,39690,39701],{"className":39689,"style":19640},[1096],[57,39691,39692,39695],{"style":18101},[57,39693],{"className":39694,"style":1493},[1104],[57,39696,39698],{"className":39697},[1109,1110,1111,1112],[57,39699,1288],{"className":39700},[1075,1148,1112],[57,39702,39703,39706],{"style":1489},[57,39704],{"className":39705,"style":1493},[1104],[57,39707,39709],{"className":39708},[1109,1110,1111,1112],[57,39710,1733],{"className":39711},[1075,1148,1112],[57,39713,1183],{"className":39714},[1182],[57,39716,39718],{"className":39717},[1092],[57,39719,39721],{"className":39720,"style":18131},[1096],[57,39722],{},[57,39724,1768],{"className":39725},[1269],[57,39727,1183],{"className":39728},[1182],[57,39730,39732],{"className":39731},[1092],[57,39733,39735],{"className":39734,"style":16739},[1096],[57,39736],{},[57,39738],{"className":39739},[1269,1080],[57,39741,39742,39745],{"style":36873},[57,39743],{"className":39744,"style":15959},[1104],[57,39746,39748,39751,39754,39757,39760,39822,39825,39892,39895,39999,40002,40094,40097,40148,40151,40154,40205,40208],{"className":39747},[1075],[57,39749],{"className":39750},[1075],[57,39752],{"className":39753,"style":1453},[1452],[57,39755,281],{"className":39756},[1457],[57,39758],{"className":39759,"style":1453},[1452],[57,39761,39763,39766,39819],{"className":39762},[1075],[57,39764],{"className":39765},[1079,1080],[57,39767,39769],{"className":39768},[1035],[57,39770,39772,39811],{"className":39771},[1087,1088],[57,39773,39775,39808],{"className":39774},[1092],[57,39776,39778,39789,39797],{"className":39777,"style":17068},[1096],[57,39779,39780,39783],{"style":16608},[57,39781],{"className":39782,"style":1105},[1104],[57,39784,39786],{"className":39785},[1075],[57,39787,7990],{"className":39788,"style":17080},[1075,1148],[57,39790,39791,39794],{"style":1230},[57,39792],{"className":39793,"style":1105},[1104],[57,39795],{"className":39796,"style":1238},[1237],[57,39798,39799,39802],{"style":16674},[57,39800],{"className":39801,"style":1105},[1104],[57,39803,39805],{"className":39804},[1075],[57,39806,574],{"className":39807},[1075],[57,39809,1183],{"className":39810},[1182],[57,39812,39814],{"className":39813},[1092],[57,39815,39817],{"className":39816,"style":17109},[1096],[57,39818],{},[57,39820],{"className":39821},[1269,1080],[57,39823],{"className":39824,"style":1853},[1452],[57,39826,39828],{"className":39827},[1888,15027],[57,39829,39831,39884],{"className":39830},[1087,1088],[57,39832,39834,39881],{"className":39833},[1092],[57,39835,39837,39857,39867],{"className":39836,"style":17130},[1096],[57,39838,39839,39842],{"style":17133},[57,39840],{"className":39841,"style":16221},[1104],[57,39843,39845],{"className":39844},[1109,1110,1111,1112],[57,39846,39848,39851,39854],{"className":39847},[1075,1112],[57,39849,1288],{"className":39850},[1075,1148,1112],[57,39852,281],{"className":39853},[1457,1112],[57,39855,574],{"className":39856},[1075,1112],[57,39858,39859,39862],{"style":16230},[57,39860],{"className":39861,"style":16221},[1104],[57,39863,39864],{},[57,39865,2549],{"className":39866},[1888,2633,16239],[57,39868,39869,39872],{"style":17164},[57,39870],{"className":39871,"style":16221},[1104],[57,39873,39875],{"className":39874},[1109,1110,1111,1112],[57,39876,39878],{"className":39877},[1075,1112],[57,39879,7990],{"className":39880,"style":17080},[1075,1148,1112],[57,39882,1183],{"className":39883},[1182],[57,39885,39887],{"className":39886},[1092],[57,39888,39890],{"className":39889,"style":17183},[1096],[57,39891],{},[57,39893],{"className":39894,"style":1853},[1452],[57,39896,39898],{"className":39897},[1888,15027],[57,39899,39901,39991],{"className":39900},[1087,1088],[57,39902,39904,39988],{"className":39903},[1092],[57,39905,39907,39927,39937],{"className":39906,"style":17939},[1096],[57,39908,39909,39912],{"style":17133},[57,39910],{"className":39911,"style":16221},[1104],[57,39913,39915],{"className":39914},[1109,1110,1111,1112],[57,39916,39918,39921,39924],{"className":39917},[1075,1112],[57,39919,1733],{"className":39920},[1075,1148,1112],[57,39922,281],{"className":39923},[1457,1112],[57,39925,574],{"className":39926},[1075,1112],[57,39928,39929,39932],{"style":16230},[57,39930],{"className":39931,"style":16221},[1104],[57,39933,39934],{},[57,39935,2549],{"className":39936},[1888,2633,16239],[57,39938,39939,39942],{"style":17972},[57,39940],{"className":39941,"style":16221},[1104],[57,39943,39945],{"className":39944},[1109,1110,1111,1112],[57,39946,39948],{"className":39947},[1075,1112],[57,39949,39951,39954],{"className":39950},[1075,1112],[57,39952,738],{"className":39953,"style":3907},[1075,1148,1112],[57,39955,39957],{"className":39956},[1152],[57,39958,39960,39980],{"className":39959},[1087,1088],[57,39961,39963,39977],{"className":39962},[1092],[57,39964,39966],{"className":39965,"style":5022},[1096],[57,39967,39968,39971],{"style":18002},[57,39969],{"className":39970,"style":1169},[1104],[57,39972,39974],{"className":39973},[1109,1173,1174,1112],[57,39975,1288],{"className":39976},[1075,1148,1112],[57,39978,1183],{"className":39979},[1182],[57,39981,39983],{"className":39982},[1092],[57,39984,39986],{"className":39985,"style":5044},[1096],[57,39987],{},[57,39989,1183],{"className":39990},[1182],[57,39992,39994],{"className":39993},[1092],[57,39995,39997],{"className":39996,"style":17183},[1096],[57,39998],{},[57,40000],{"className":40001,"style":1853},[1452],[57,40003,40005,40008],{"className":40004},[1075],[57,40006,1730],{"className":40007},[1075,1148],[57,40009,40011],{"className":40010},[1152],[57,40012,40014,40086],{"className":40013},[1087,1088],[57,40015,40017,40083],{"className":40016},[1092],[57,40018,40020,40063],{"className":40019,"style":25824},[1096],[57,40021,40022,40025],{"style":18101},[57,40023],{"className":40024,"style":1493},[1104],[57,40026,40028],{"className":40027},[1109,1110,1111,1112],[57,40029,40031],{"className":40030},[1075,1112],[57,40032,40034,40037],{"className":40033},[1075,1112],[57,40035,14745],{"className":40036,"style":3932},[1075,1148,1112],[57,40038,40040],{"className":40039},[1152],[57,40041,40043],{"className":40042},[1087],[57,40044,40046],{"className":40045},[1092],[57,40047,40049],{"className":40048,"style":19935},[1096],[57,40050,40051,40054],{"style":19938},[57,40052],{"className":40053,"style":1169},[1104],[57,40055,40057],{"className":40056},[1109,1173,1174,1112],[57,40058,40060],{"className":40059},[1075,1112],[57,40061,19775],{"className":40062},[1075,1112],[57,40064,40065,40068],{"style":17216},[57,40066],{"className":40067,"style":1493},[1104],[57,40069,40071],{"className":40070},[1109,1110,1111,1112],[57,40072,40074,40077,40080],{"className":40073},[1075,1112],[57,40075,28613],{"className":40076},[1075,1148,1112],[57,40078,1730],{"className":40079},[1075,1148,1112],[57,40081,6336],{"className":40082,"style":6749},[1075,1148,1112],[57,40084,1183],{"className":40085},[1182],[57,40087,40089],{"className":40088},[1092],[57,40090,40092],{"className":40091,"style":18131},[1096],[57,40093],{},[57,40095,1754],{"className":40096},[1079],[57,40098,40100,40103],{"className":40099},[1075],[57,40101,13535],{"className":40102},[1075,1148],[57,40104,40106],{"className":40105},[1152],[57,40107,40109,40140],{"className":40108},[1087,1088],[57,40110,40112,40137],{"className":40111},[1092],[57,40113,40115,40126],{"className":40114,"style":18098},[1096],[57,40116,40117,40120],{"style":18101},[57,40118],{"className":40119,"style":1493},[1104],[57,40121,40123],{"className":40122},[1109,1110,1111,1112],[57,40124,1288],{"className":40125},[1075,1148,1112],[57,40127,40128,40131],{"style":17216},[57,40129],{"className":40130,"style":1493},[1104],[57,40132,40134],{"className":40133},[1109,1110,1111,1112],[57,40135,1733],{"className":40136},[1075,1148,1112],[57,40138,1183],{"className":40139},[1182],[57,40141,40143],{"className":40142},[1092],[57,40144,40146],{"className":40145,"style":18131},[1096],[57,40147],{},[57,40149,1759],{"className":40150},[1849],[57,40152],{"className":40153,"style":1853},[1452],[57,40155,40157,40160],{"className":40156},[1075],[57,40158,18],{"className":40159},[1075,1148],[57,40161,40163],{"className":40162},[1152],[57,40164,40166,40197],{"className":40165},[1087,1088],[57,40167,40169,40194],{"className":40168},[1092],[57,40170,40172,40183],{"className":40171,"style":18098},[1096],[57,40173,40174,40177],{"style":18101},[57,40175],{"className":40176,"style":1493},[1104],[57,40178,40180],{"className":40179},[1109,1110,1111,1112],[57,40181,1288],{"className":40182},[1075,1148,1112],[57,40184,40185,40188],{"style":17216},[57,40186],{"className":40187,"style":1493},[1104],[57,40189,40191],{"className":40190},[1109,1110,1111,1112],[57,40192,1733],{"className":40193},[1075,1148,1112],[57,40195,1183],{"className":40196},[1182],[57,40198,40200],{"className":40199},[1092],[57,40201,40203],{"className":40202,"style":18131},[1096],[57,40204],{},[57,40206,1768],{"className":40207},[1269],[57,40209,40211,40214,40598],{"className":40210},[1075],[57,40212],{"className":40213},[1079,1080],[57,40215,40217],{"className":40216},[1035],[57,40218,40220,40590],{"className":40219},[1087,1088],[57,40221,40223,40587],{"className":40222},[1092],[57,40224,40226,40417,40425],{"className":40225,"style":37982},[1096],[57,40227,40228,40231],{"style":16608},[57,40229],{"className":40230,"style":1105},[1104],[57,40232,40234,40306,40309,40360,40363,40414],{"className":40233},[1075],[57,40235,40237,40240],{"className":40236},[1075],[57,40238,6333],{"className":40239,"style":3907},[1075,1148],[57,40241,40243],{"className":40242},[1152],[57,40244,40246,40298],{"className":40245},[1087,1088],[57,40247,40249,40295],{"className":40248},[1092],[57,40250,40252],{"className":40251,"style":2208},[1096],[57,40253,40254,40257],{"style":8131},[57,40255],{"className":40256,"style":1493},[1104],[57,40258,40260],{"className":40259},[1109,1110,1111,1112],[57,40261,40263],{"className":40262},[1075,1112],[57,40264,40266,40269],{"className":40265},[1075,1112],[57,40267,14745],{"className":40268,"style":3932},[1075,1148,1112],[57,40270,40272],{"className":40271},[1152],[57,40273,40275],{"className":40274},[1087],[57,40276,40278],{"className":40277},[1092],[57,40279,40281],{"className":40280,"style":19935},[1096],[57,40282,40283,40286],{"style":19938},[57,40284],{"className":40285,"style":1169},[1104],[57,40287,40289],{"className":40288},[1109,1173,1174,1112],[57,40290,40292],{"className":40291},[1075,1112],[57,40293,19775],{"className":40294},[1075,1112],[57,40296,1183],{"className":40297},[1182],[57,40299,40301],{"className":40300},[1092],[57,40302,40304],{"className":40303,"style":2230},[1096],[57,40305],{},[57,40307,1754],{"className":40308},[1079],[57,40310,40312,40315],{"className":40311},[1075],[57,40313,18],{"className":40314},[1075,1148],[57,40316,40318],{"className":40317},[1152],[57,40319,40321,40352],{"className":40320},[1087,1088],[57,40322,40324,40349],{"className":40323},[1092],[57,40325,40327,40338],{"className":40326,"style":38084},[1096],[57,40328,40329,40332],{"style":18101},[57,40330],{"className":40331,"style":1493},[1104],[57,40333,40335],{"className":40334},[1109,1110,1111,1112],[57,40336,1288],{"className":40337},[1075,1148,1112],[57,40339,40340,40343],{"style":17319},[57,40341],{"className":40342,"style":1493},[1104],[57,40344,40346],{"className":40345},[1109,1110,1111,1112],[57,40347,1733],{"className":40348},[1075,1148,1112],[57,40350,1183],{"className":40351},[1182],[57,40353,40355],{"className":40354},[1092],[57,40356,40358],{"className":40357,"style":18131},[1096],[57,40359],{},[57,40361,13675],{"className":40362},[1075],[57,40364,40366,40369],{"className":40365},[1075],[57,40367,13535],{"className":40368},[1075,1148],[57,40370,40372],{"className":40371},[1152],[57,40373,40375,40406],{"className":40374},[1087,1088],[57,40376,40378,40403],{"className":40377},[1092],[57,40379,40381,40392],{"className":40380,"style":38084},[1096],[57,40382,40383,40386],{"style":18101},[57,40384],{"className":40385,"style":1493},[1104],[57,40387,40389],{"className":40388},[1109,1110,1111,1112],[57,40390,1288],{"className":40391},[1075,1148,1112],[57,40393,40394,40397],{"style":17319},[57,40395],{"className":40396,"style":1493},[1104],[57,40398,40400],{"className":40399},[1109,1110,1111,1112],[57,40401,1733],{"className":40402},[1075,1148,1112],[57,40404,1183],{"className":40405},[1182],[57,40407,40409],{"className":40408},[1092],[57,40410,40412],{"className":40411,"style":18131},[1096],[57,40413],{},[57,40415,1768],{"className":40416},[1269],[57,40418,40419,40422],{"style":1230},[57,40420],{"className":40421,"style":1105},[1104],[57,40423],{"className":40424,"style":1238},[1237],[57,40426,40427,40430],{"style":16674},[57,40428],{"className":40429,"style":1105},[1104],[57,40431,40433,40436,40476,40479,40530,40533,40584],{"className":40432},[1075],[57,40434,15169],{"className":40435},[1075],[57,40437,40439,40442],{"className":40438},[1075],[57,40440,6333],{"className":40441,"style":3907},[1075,1148],[57,40443,40445],{"className":40444},[1152],[57,40446,40448,40468],{"className":40447},[1087,1088],[57,40449,40451,40465],{"className":40450},[1092],[57,40452,40454],{"className":40453,"style":2208},[1096],[57,40455,40456,40459],{"style":8131},[57,40457],{"className":40458,"style":1493},[1104],[57,40460,40462],{"className":40461},[1109,1110,1111,1112],[57,40463,14745],{"className":40464,"style":3932},[1075,1148,1112],[57,40466,1183],{"className":40467},[1182],[57,40469,40471],{"className":40470},[1092],[57,40472,40474],{"className":40473,"style":2230},[1096],[57,40475],{},[57,40477,1754],{"className":40478},[1079],[57,40480,40482,40485],{"className":40481},[1075],[57,40483,18],{"className":40484},[1075,1148],[57,40486,40488],{"className":40487},[1152],[57,40489,40491,40522],{"className":40490},[1087,1088],[57,40492,40494,40519],{"className":40493},[1092],[57,40495,40497,40508],{"className":40496,"style":19640},[1096],[57,40498,40499,40502],{"style":18101},[57,40500],{"className":40501,"style":1493},[1104],[57,40503,40505],{"className":40504},[1109,1110,1111,1112],[57,40506,1288],{"className":40507},[1075,1148,1112],[57,40509,40510,40513],{"style":1489},[57,40511],{"className":40512,"style":1493},[1104],[57,40514,40516],{"className":40515},[1109,1110,1111,1112],[57,40517,1733],{"className":40518},[1075,1148,1112],[57,40520,1183],{"className":40521},[1182],[57,40523,40525],{"className":40524},[1092],[57,40526,40528],{"className":40527,"style":18131},[1096],[57,40529],{},[57,40531,13675],{"className":40532},[1075],[57,40534,40536,40539],{"className":40535},[1075],[57,40537,13535],{"className":40538},[1075,1148],[57,40540,40542],{"className":40541},[1152],[57,40543,40545,40576],{"className":40544},[1087,1088],[57,40546,40548,40573],{"className":40547},[1092],[57,40549,40551,40562],{"className":40550,"style":19640},[1096],[57,40552,40553,40556],{"style":18101},[57,40554],{"className":40555,"style":1493},[1104],[57,40557,40559],{"className":40558},[1109,1110,1111,1112],[57,40560,1288],{"className":40561},[1075,1148,1112],[57,40563,40564,40567],{"style":1489},[57,40565],{"className":40566,"style":1493},[1104],[57,40568,40570],{"className":40569},[1109,1110,1111,1112],[57,40571,1733],{"className":40572},[1075,1148,1112],[57,40574,1183],{"className":40575},[1182],[57,40577,40579],{"className":40578},[1092],[57,40580,40582],{"className":40581,"style":18131},[1096],[57,40583],{},[57,40585,1768],{"className":40586},[1269],[57,40588,1183],{"className":40589},[1182],[57,40591,40593],{"className":40592},[1092],[57,40594,40596],{"className":40595,"style":16739},[1096],[57,40597],{},[57,40599],{"className":40600},[1269,1080],[57,40602,1183],{"className":40603},[1182],[57,40605,40607],{"className":40606},[1092],[57,40608,40610],{"className":40609,"style":36889},[1096],[57,40611],{},[14,40613,40614],{},"对应地，将期望最大化转换为损失最小化（取负号）：",[57,40616,40618],{"className":40617},[15139],[57,40619,40621,40789],{"className":40620},[1019],[57,40622,40624],{"className":40623},[1023],[1025,40625,40626],{"xmlns":1027,"display":10210},[1029,40627,40628,40786],{},[1032,40629,40630,40632,40634,40636,40638,40640,40642,40648,40662,40680,40698,40700,40708,40710,40718,40720],{},[1047,40631,8490],{},[1047,40633,1748],{},[1047,40635,13535],{},[1047,40637,13535],{},[1414,40639,281],{},[1414,40641,19155],{},[1035,40643,40644,40646],{},[1038,40645,574],{},[1047,40647,7990],{},[15472,40649,40650,40652,40660],{},[1414,40651,2549],{},[1032,40653,40654,40656,40658],{},[1047,40655,1288],{},[1414,40657,281],{},[1038,40659,574],{},[1047,40661,7990],{},[15472,40663,40664,40666,40674],{},[1414,40665,2549],{},[1032,40667,40668,40670,40672],{},[1047,40669,1733],{},[1414,40671,281],{},[1038,40673,574],{},[1044,40675,40676,40678],{},[1047,40677,738],{},[1047,40679,1288],{},[2545,40681,40682,40684,40690],{},[1047,40683,1730],{},[1418,40685,40686,40688],{},[1047,40687,14745],{},[1414,40689,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,40691,40692,40694,40696],{},[1047,40693,28613],{},[1047,40695,1730],{},[1047,40697,6336],{},[1414,40699,1754],{"stretchy":1753},[2545,40701,40702,40704,40706],{},[1047,40703,13535],{},[1047,40705,1288],{},[1047,40707,1733],{},[1414,40709,1759],{"separator":1062},[2545,40711,40712,40714,40716],{},[1047,40713,18],{},[1047,40715,1288],{},[1047,40717,1733],{},[1414,40719,1768],{"stretchy":1753},[1035,40721,40722,40752],{},[1032,40723,40724,40730,40732,40740,40742,40750],{},[1044,40725,40726,40728],{},[1047,40727,6333],{},[1047,40729,14745],{},[1414,40731,1754],{"stretchy":1753},[2545,40733,40734,40736,40738],{},[1047,40735,18],{},[1047,40737,1288],{},[1047,40739,1733],{},[1047,40741,13675],{"mathvariant":1729},[2545,40743,40744,40746,40748],{},[1047,40745,13535],{},[1047,40747,1288],{},[1047,40749,1733],{},[1414,40751,1768],{"stretchy":1753},[1032,40753,40754,40764,40766,40774,40776,40784],{},[1044,40755,40756,40758],{},[1047,40757,6333],{},[1418,40759,40760,40762],{},[1047,40761,14745],{},[1414,40763,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,40765,1754],{"stretchy":1753},[2545,40767,40768,40770,40772],{},[1047,40769,18],{},[1047,40771,1288],{},[1047,40773,1733],{},[1047,40775,13675],{"mathvariant":1729},[2545,40777,40778,40780,40782],{},[1047,40779,13535],{},[1047,40781,1288],{},[1047,40783,1733],{},[1414,40785,1768],{"stretchy":1753},[1054,40787,40788],{"encoding":1056},"Loss = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_\\theta(a_n^t | s_n^t)}{P_{\\theta'}(a_n^t | s_n^t)}",[57,40790,40792,40813],{"className":40791,"ariaHidden":1062},[1061],[57,40793,40795,40798,40801,40804,40807,40810],{"className":40794},[1066],[57,40796],{"className":40797,"style":19250},[1070],[57,40799,8490],{"className":40800},[1075,1148],[57,40802,19257],{"className":40803},[1075,1148],[57,40805],{"className":40806,"style":1453},[1452],[57,40808,281],{"className":40809},[1457],[57,40811],{"className":40812,"style":1453},[1452],[57,40814,40816,40819,40822,40884,40887,40954,40957,41061,41064,41156,41159,41210,41213,41216,41267,41270],{"className":40815},[1066],[57,40817],{"className":40818,"style":32380},[1070],[57,40820,19155],{"className":40821},[1075],[57,40823,40825,40828,40881],{"className":40824},[1075],[57,40826],{"className":40827},[1079,1080],[57,40829,40831],{"className":40830},[1035],[57,40832,40834,40873],{"className":40833},[1087,1088],[57,40835,40837,40870],{"className":40836},[1092],[57,40838,40840,40851,40859],{"className":40839,"style":17068},[1096],[57,40841,40842,40845],{"style":16608},[57,40843],{"className":40844,"style":1105},[1104],[57,40846,40848],{"className":40847},[1075],[57,40849,7990],{"className":40850,"style":17080},[1075,1148],[57,40852,40853,40856],{"style":1230},[57,40854],{"className":40855,"style":1105},[1104],[57,40857],{"className":40858,"style":1238},[1237],[57,40860,40861,40864],{"style":16674},[57,40862],{"className":40863,"style":1105},[1104],[57,40865,40867],{"className":40866},[1075],[57,40868,574],{"className":40869},[1075],[57,40871,1183],{"className":40872},[1182],[57,40874,40876],{"className":40875},[1092],[57,40877,40879],{"className":40878,"style":17109},[1096],[57,40880],{},[57,40882],{"className":40883},[1269,1080],[57,40885],{"className":40886,"style":1853},[1452],[57,40888,40890],{"className":40889},[1888,15027],[57,40891,40893,40946],{"className":40892},[1087,1088],[57,40894,40896,40943],{"className":40895},[1092],[57,40897,40899,40919,40929],{"className":40898,"style":17130},[1096],[57,40900,40901,40904],{"style":17133},[57,40902],{"className":40903,"style":16221},[1104],[57,40905,40907],{"className":40906},[1109,1110,1111,1112],[57,40908,40910,40913,40916],{"className":40909},[1075,1112],[57,40911,1288],{"className":40912},[1075,1148,1112],[57,40914,281],{"className":40915},[1457,1112],[57,40917,574],{"className":40918},[1075,1112],[57,40920,40921,40924],{"style":16230},[57,40922],{"className":40923,"style":16221},[1104],[57,40925,40926],{},[57,40927,2549],{"className":40928},[1888,2633,16239],[57,40930,40931,40934],{"style":17164},[57,40932],{"className":40933,"style":16221},[1104],[57,40935,40937],{"className":40936},[1109,1110,1111,1112],[57,40938,40940],{"className":40939},[1075,1112],[57,40941,7990],{"className":40942,"style":17080},[1075,1148,1112],[57,40944,1183],{"className":40945},[1182],[57,40947,40949],{"className":40948},[1092],[57,40950,40952],{"className":40951,"style":17183},[1096],[57,40953],{},[57,40955],{"className":40956,"style":1853},[1452],[57,40958,40960],{"className":40959},[1888,15027],[57,40961,40963,41053],{"className":40962},[1087,1088],[57,40964,40966,41050],{"className":40965},[1092],[57,40967,40969,40989,40999],{"className":40968,"style":17939},[1096],[57,40970,40971,40974],{"style":17133},[57,40972],{"className":40973,"style":16221},[1104],[57,40975,40977],{"className":40976},[1109,1110,1111,1112],[57,40978,40980,40983,40986],{"className":40979},[1075,1112],[57,40981,1733],{"className":40982},[1075,1148,1112],[57,40984,281],{"className":40985},[1457,1112],[57,40987,574],{"className":40988},[1075,1112],[57,40990,40991,40994],{"style":16230},[57,40992],{"className":40993,"style":16221},[1104],[57,40995,40996],{},[57,40997,2549],{"className":40998},[1888,2633,16239],[57,41000,41001,41004],{"style":17972},[57,41002],{"className":41003,"style":16221},[1104],[57,41005,41007],{"className":41006},[1109,1110,1111,1112],[57,41008,41010],{"className":41009},[1075,1112],[57,41011,41013,41016],{"className":41012},[1075,1112],[57,41014,738],{"className":41015,"style":3907},[1075,1148,1112],[57,41017,41019],{"className":41018},[1152],[57,41020,41022,41042],{"className":41021},[1087,1088],[57,41023,41025,41039],{"className":41024},[1092],[57,41026,41028],{"className":41027,"style":5022},[1096],[57,41029,41030,41033],{"style":18002},[57,41031],{"className":41032,"style":1169},[1104],[57,41034,41036],{"className":41035},[1109,1173,1174,1112],[57,41037,1288],{"className":41038},[1075,1148,1112],[57,41040,1183],{"className":41041},[1182],[57,41043,41045],{"className":41044},[1092],[57,41046,41048],{"className":41047,"style":5044},[1096],[57,41049],{},[57,41051,1183],{"className":41052},[1182],[57,41054,41056],{"className":41055},[1092],[57,41057,41059],{"className":41058,"style":17183},[1096],[57,41060],{},[57,41062],{"className":41063,"style":1853},[1452],[57,41065,41067,41070],{"className":41066},[1075],[57,41068,1730],{"className":41069},[1075,1148],[57,41071,41073],{"className":41072},[1152],[57,41074,41076,41148],{"className":41075},[1087,1088],[57,41077,41079,41145],{"className":41078},[1092],[57,41080,41082,41125],{"className":41081,"style":25824},[1096],[57,41083,41084,41087],{"style":18101},[57,41085],{"className":41086,"style":1493},[1104],[57,41088,41090],{"className":41089},[1109,1110,1111,1112],[57,41091,41093],{"className":41092},[1075,1112],[57,41094,41096,41099],{"className":41095},[1075,1112],[57,41097,14745],{"className":41098,"style":3932},[1075,1148,1112],[57,41100,41102],{"className":41101},[1152],[57,41103,41105],{"className":41104},[1087],[57,41106,41108],{"className":41107},[1092],[57,41109,41111],{"className":41110,"style":19935},[1096],[57,41112,41113,41116],{"style":19938},[57,41114],{"className":41115,"style":1169},[1104],[57,41117,41119],{"className":41118},[1109,1173,1174,1112],[57,41120,41122],{"className":41121},[1075,1112],[57,41123,19775],{"className":41124},[1075,1112],[57,41126,41127,41130],{"style":17216},[57,41128],{"className":41129,"style":1493},[1104],[57,41131,41133],{"className":41132},[1109,1110,1111,1112],[57,41134,41136,41139,41142],{"className":41135},[1075,1112],[57,41137,28613],{"className":41138},[1075,1148,1112],[57,41140,1730],{"className":41141},[1075,1148,1112],[57,41143,6336],{"className":41144,"style":6749},[1075,1148,1112],[57,41146,1183],{"className":41147},[1182],[57,41149,41151],{"className":41150},[1092],[57,41152,41154],{"className":41153,"style":18131},[1096],[57,41155],{},[57,41157,1754],{"className":41158},[1079],[57,41160,41162,41165],{"className":41161},[1075],[57,41163,13535],{"className":41164},[1075,1148],[57,41166,41168],{"className":41167},[1152],[57,41169,41171,41202],{"className":41170},[1087,1088],[57,41172,41174,41199],{"className":41173},[1092],[57,41175,41177,41188],{"className":41176,"style":18098},[1096],[57,41178,41179,41182],{"style":18101},[57,41180],{"className":41181,"style":1493},[1104],[57,41183,41185],{"className":41184},[1109,1110,1111,1112],[57,41186,1288],{"className":41187},[1075,1148,1112],[57,41189,41190,41193],{"style":17216},[57,41191],{"className":41192,"style":1493},[1104],[57,41194,41196],{"className":41195},[1109,1110,1111,1112],[57,41197,1733],{"className":41198},[1075,1148,1112],[57,41200,1183],{"className":41201},[1182],[57,41203,41205],{"className":41204},[1092],[57,41206,41208],{"className":41207,"style":18131},[1096],[57,41209],{},[57,41211,1759],{"className":41212},[1849],[57,41214],{"className":41215,"style":1853},[1452],[57,41217,41219,41222],{"className":41218},[1075],[57,41220,18],{"className":41221},[1075,1148],[57,41223,41225],{"className":41224},[1152],[57,41226,41228,41259],{"className":41227},[1087,1088],[57,41229,41231,41256],{"className":41230},[1092],[57,41232,41234,41245],{"className":41233,"style":18098},[1096],[57,41235,41236,41239],{"style":18101},[57,41237],{"className":41238,"style":1493},[1104],[57,41240,41242],{"className":41241},[1109,1110,1111,1112],[57,41243,1288],{"className":41244},[1075,1148,1112],[57,41246,41247,41250],{"style":17216},[57,41248],{"className":41249,"style":1493},[1104],[57,41251,41253],{"className":41252},[1109,1110,1111,1112],[57,41254,1733],{"className":41255},[1075,1148,1112],[57,41257,1183],{"className":41258},[1182],[57,41260,41262],{"className":41261},[1092],[57,41263,41265],{"className":41264,"style":18131},[1096],[57,41266],{},[57,41268,1768],{"className":41269},[1269],[57,41271,41273,41276,41657],{"className":41272},[1075],[57,41274],{"className":41275},[1079,1080],[57,41277,41279],{"className":41278},[1035],[57,41280,41282,41649],{"className":41281},[1087,1088],[57,41283,41285,41646],{"className":41284},[1092],[57,41286,41288,41479,41487],{"className":41287,"style":37982},[1096],[57,41289,41290,41293],{"style":16608},[57,41291],{"className":41292,"style":1105},[1104],[57,41294,41296,41368,41371,41422,41425,41476],{"className":41295},[1075],[57,41297,41299,41302],{"className":41298},[1075],[57,41300,6333],{"className":41301,"style":3907},[1075,1148],[57,41303,41305],{"className":41304},[1152],[57,41306,41308,41360],{"className":41307},[1087,1088],[57,41309,41311,41357],{"className":41310},[1092],[57,41312,41314],{"className":41313,"style":2208},[1096],[57,41315,41316,41319],{"style":8131},[57,41317],{"className":41318,"style":1493},[1104],[57,41320,41322],{"className":41321},[1109,1110,1111,1112],[57,41323,41325],{"className":41324},[1075,1112],[57,41326,41328,41331],{"className":41327},[1075,1112],[57,41329,14745],{"className":41330,"style":3932},[1075,1148,1112],[57,41332,41334],{"className":41333},[1152],[57,41335,41337],{"className":41336},[1087],[57,41338,41340],{"className":41339},[1092],[57,41341,41343],{"className":41342,"style":19935},[1096],[57,41344,41345,41348],{"style":19938},[57,41346],{"className":41347,"style":1169},[1104],[57,41349,41351],{"className":41350},[1109,1173,1174,1112],[57,41352,41354],{"className":41353},[1075,1112],[57,41355,19775],{"className":41356},[1075,1112],[57,41358,1183],{"className":41359},[1182],[57,41361,41363],{"className":41362},[1092],[57,41364,41366],{"className":41365,"style":2230},[1096],[57,41367],{},[57,41369,1754],{"className":41370},[1079],[57,41372,41374,41377],{"className":41373},[1075],[57,41375,18],{"className":41376},[1075,1148],[57,41378,41380],{"className":41379},[1152],[57,41381,41383,41414],{"className":41382},[1087,1088],[57,41384,41386,41411],{"className":41385},[1092],[57,41387,41389,41400],{"className":41388,"style":38084},[1096],[57,41390,41391,41394],{"style":18101},[57,41392],{"className":41393,"style":1493},[1104],[57,41395,41397],{"className":41396},[1109,1110,1111,1112],[57,41398,1288],{"className":41399},[1075,1148,1112],[57,41401,41402,41405],{"style":17319},[57,41403],{"className":41404,"style":1493},[1104],[57,41406,41408],{"className":41407},[1109,1110,1111,1112],[57,41409,1733],{"className":41410},[1075,1148,1112],[57,41412,1183],{"className":41413},[1182],[57,41415,41417],{"className":41416},[1092],[57,41418,41420],{"className":41419,"style":18131},[1096],[57,41421],{},[57,41423,13675],{"className":41424},[1075],[57,41426,41428,41431],{"className":41427},[1075],[57,41429,13535],{"className":41430},[1075,1148],[57,41432,41434],{"className":41433},[1152],[57,41435,41437,41468],{"className":41436},[1087,1088],[57,41438,41440,41465],{"className":41439},[1092],[57,41441,41443,41454],{"className":41442,"style":38084},[1096],[57,41444,41445,41448],{"style":18101},[57,41446],{"className":41447,"style":1493},[1104],[57,41449,41451],{"className":41450},[1109,1110,1111,1112],[57,41452,1288],{"className":41453},[1075,1148,1112],[57,41455,41456,41459],{"style":17319},[57,41457],{"className":41458,"style":1493},[1104],[57,41460,41462],{"className":41461},[1109,1110,1111,1112],[57,41463,1733],{"className":41464},[1075,1148,1112],[57,41466,1183],{"className":41467},[1182],[57,41469,41471],{"className":41470},[1092],[57,41472,41474],{"className":41473,"style":18131},[1096],[57,41475],{},[57,41477,1768],{"className":41478},[1269],[57,41480,41481,41484],{"style":1230},[57,41482],{"className":41483,"style":1105},[1104],[57,41485],{"className":41486,"style":1238},[1237],[57,41488,41489,41492],{"style":16674},[57,41490],{"className":41491,"style":1105},[1104],[57,41493,41495,41535,41538,41589,41592,41643],{"className":41494},[1075],[57,41496,41498,41501],{"className":41497},[1075],[57,41499,6333],{"className":41500,"style":3907},[1075,1148],[57,41502,41504],{"className":41503},[1152],[57,41505,41507,41527],{"className":41506},[1087,1088],[57,41508,41510,41524],{"className":41509},[1092],[57,41511,41513],{"className":41512,"style":2208},[1096],[57,41514,41515,41518],{"style":8131},[57,41516],{"className":41517,"style":1493},[1104],[57,41519,41521],{"className":41520},[1109,1110,1111,1112],[57,41522,14745],{"className":41523,"style":3932},[1075,1148,1112],[57,41525,1183],{"className":41526},[1182],[57,41528,41530],{"className":41529},[1092],[57,41531,41533],{"className":41532,"style":2230},[1096],[57,41534],{},[57,41536,1754],{"className":41537},[1079],[57,41539,41541,41544],{"className":41540},[1075],[57,41542,18],{"className":41543},[1075,1148],[57,41545,41547],{"className":41546},[1152],[57,41548,41550,41581],{"className":41549},[1087,1088],[57,41551,41553,41578],{"className":41552},[1092],[57,41554,41556,41567],{"className":41555,"style":19640},[1096],[57,41557,41558,41561],{"style":18101},[57,41559],{"className":41560,"style":1493},[1104],[57,41562,41564],{"className":41563},[1109,1110,1111,1112],[57,41565,1288],{"className":41566},[1075,1148,1112],[57,41568,41569,41572],{"style":1489},[57,41570],{"className":41571,"style":1493},[1104],[57,41573,41575],{"className":41574},[1109,1110,1111,1112],[57,41576,1733],{"className":41577},[1075,1148,1112],[57,41579,1183],{"className":41580},[1182],[57,41582,41584],{"className":41583},[1092],[57,41585,41587],{"className":41586,"style":18131},[1096],[57,41588],{},[57,41590,13675],{"className":41591},[1075],[57,41593,41595,41598],{"className":41594},[1075],[57,41596,13535],{"className":41597},[1075,1148],[57,41599,41601],{"className":41600},[1152],[57,41602,41604,41635],{"className":41603},[1087,1088],[57,41605,41607,41632],{"className":41606},[1092],[57,41608,41610,41621],{"className":41609,"style":19640},[1096],[57,41611,41612,41615],{"style":18101},[57,41613],{"className":41614,"style":1493},[1104],[57,41616,41618],{"className":41617},[1109,1110,1111,1112],[57,41619,1288],{"className":41620},[1075,1148,1112],[57,41622,41623,41626],{"style":1489},[57,41624],{"className":41625,"style":1493},[1104],[57,41627,41629],{"className":41628},[1109,1110,1111,1112],[57,41630,1733],{"className":41631},[1075,1148,1112],[57,41633,1183],{"className":41634},[1182],[57,41636,41638],{"className":41637},[1092],[57,41639,41641],{"className":41640,"style":18131},[1096],[57,41642],{},[57,41644,1768],{"className":41645},[1269],[57,41647,1183],{"className":41648},[1182],[57,41650,41652],{"className":41651},[1092],[57,41653,41655],{"className":41654,"style":16739},[1096],[57,41656],{},[57,41658],{"className":41659},[1269,1080],[14,41661,41662,41663,41666,41667,743,41695,41756,41757,41980],{},"这里有一个 ",[11865,41664,41665],{},"隐含的前提","：重要性采样要求 ",[57,41668,41670,41683],{"className":41669},[1019],[57,41671,41673],{"className":41672},[1023],[1025,41674,41675],{"xmlns":1027},[1029,41676,41677,41681],{},[1032,41678,41679],{},[1047,41680,14745],{},[1054,41682,35330],{"encoding":1056},[57,41684,41686],{"className":41685,"ariaHidden":1062},[1061],[57,41687,41689,41692],{"className":41688},[1066],[57,41690],{"className":41691,"style":1360},[1070],[57,41693,14745],{"className":41694,"style":3932},[1075,1148],[57,41696,41698,41715],{"className":41697},[1019],[57,41699,41701],{"className":41700},[1023],[1025,41702,41703],{"xmlns":1027},[1029,41704,41705,41713],{},[1032,41706,41707],{},[1418,41708,41709,41711],{},[1047,41710,14745],{},[1414,41712,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1054,41714,35267],{"encoding":1056},[57,41716,41718],{"className":41717,"ariaHidden":1062},[1061],[57,41719,41721,41724],{"className":41720},[1066],[57,41722],{"className":41723,"style":35277},[1070],[57,41725,41727,41730],{"className":41726},[1075],[57,41728,14745],{"className":41729,"style":3932},[1075,1148],[57,41731,41733],{"className":41732},[1152],[57,41734,41736],{"className":41735},[1087],[57,41737,41739],{"className":41738},[1092],[57,41740,41742],{"className":41741,"style":35277},[1096],[57,41743,41744,41747],{"style":1489},[57,41745],{"className":41746,"style":1493},[1104],[57,41748,41750],{"className":41749},[1109,1110,1111,1112],[57,41751,41753],{"className":41752},[1075,1112],[57,41754,19775],{"className":41755},[1075,1112]," 不能差太多，否则比值 ",[57,41758,41760,41790],{"className":41759},[1019],[57,41761,41763],{"className":41762},[1023],[1025,41764,41765],{"xmlns":1027},[1029,41766,41767,41787],{},[1032,41768,41769],{},[1035,41770,41771,41777],{},[1044,41772,41773,41775],{},[1047,41774,6333],{},[1047,41776,14745],{},[1044,41778,41779,41781],{},[1047,41780,6333],{},[1418,41782,41783,41785],{},[1047,41784,14745],{},[1414,41786,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1054,41788,41789],{"encoding":1056},"\\frac{P_\\theta}{P_{\\theta'}}",[57,41791,41793],{"className":41792,"ariaHidden":1062},[1061],[57,41794,41796,41800],{"className":41795},[1066],[57,41797],{"className":41798,"style":41799},[1070],"height:1.4226em;vertical-align:-0.5284em;",[57,41801,41803,41806,41977],{"className":41802},[1075],[57,41804],{"className":41805},[1079,1080],[57,41807,41809],{"className":41808},[1035],[57,41810,41812,41968],{"className":41811},[1087,1088],[57,41813,41815,41965],{"className":41814},[1092],[57,41816,41819,41905,41913],{"className":41817,"style":41818},[1096],"height:0.8942em;",[57,41820,41821,41824],{"style":19297},[57,41822],{"className":41823,"style":1105},[1104],[57,41825,41827],{"className":41826},[1109,1110,1111,1112],[57,41828,41830],{"className":41829},[1075,1112],[57,41831,41833,41836],{"className":41832},[1075,1112],[57,41834,6333],{"className":41835,"style":3907},[1075,1148,1112],[57,41837,41839],{"className":41838},[1152],[57,41840,41842,41896],{"className":41841},[1087,1088],[57,41843,41845,41893],{"className":41844},[1092],[57,41846,41848],{"className":41847,"style":1162},[1096],[57,41849,41851,41855],{"style":41850},"top:-2.3448em;margin-left:-0.1389em;margin-right:0.0714em;",[57,41852],{"className":41853,"style":41854},[1104],"height:2.6068em;",[57,41856,41858],{"className":41857},[1109,1173,1174,1112],[57,41859,41861],{"className":41860},[1075,1112],[57,41862,41864,41867],{"className":41863},[1075,1112],[57,41865,14745],{"className":41866,"style":3932},[1075,1148,1112],[57,41868,41870],{"className":41869},[1152],[57,41871,41873],{"className":41872},[1087],[57,41874,41876],{"className":41875},[1092],[57,41877,41880],{"className":41878,"style":41879},[1096],"height:0.8496em;",[57,41881,41883,41887],{"style":41882},"top:-2.8496em;margin-right:0.1em;",[57,41884],{"className":41885,"style":41886},[1104],"height:2.5556em;",[57,41888,41890],{"className":41889},[1075,1112],[57,41891,19775],{"className":41892},[1075,1112],[57,41894,1183],{"className":41895},[1182],[57,41897,41899],{"className":41898},[1092],[57,41900,41903],{"className":41901,"style":41902},[1096],"height:0.262em;",[57,41904],{},[57,41906,41907,41910],{"style":1230},[57,41908],{"className":41909,"style":1105},[1104],[57,41911],{"className":41912,"style":1238},[1237],[57,41914,41916,41919],{"style":41915},"top:-3.4159em;",[57,41917],{"className":41918,"style":1105},[1104],[57,41920,41922],{"className":41921},[1109,1110,1111,1112],[57,41923,41925],{"className":41924},[1075,1112],[57,41926,41928,41931],{"className":41927},[1075,1112],[57,41929,6333],{"className":41930,"style":3907},[1075,1148,1112],[57,41932,41934],{"className":41933},[1152],[57,41935,41937,41957],{"className":41936},[1087,1088],[57,41938,41940,41954],{"className":41939},[1092],[57,41941,41943],{"className":41942,"style":1162},[1096],[57,41944,41945,41948],{"style":14964},[57,41946],{"className":41947,"style":1169},[1104],[57,41949,41951],{"className":41950},[1109,1173,1174,1112],[57,41952,14745],{"className":41953,"style":3932},[1075,1148,1112],[57,41955,1183],{"className":41956},[1182],[57,41958,41960],{"className":41959},[1092],[57,41961,41963],{"className":41962,"style":1190},[1096],[57,41964],{},[57,41966,1183],{"className":41967},[1182],[57,41969,41971],{"className":41970},[1092],[57,41972,41975],{"className":41973,"style":41974},[1096],"height:0.5284em;",[57,41976],{},[57,41978],{"className":41979},[1269,1080]," 的方差会爆炸，估计就会失真。换句话说，我们需要给\"新旧策略的差距\"施加一个约束。PPO 给出了两种约束方式：",[41982,41983,41985],"h4",{"id":41984},"ppo-penalty在-surrogate-上加-kl-惩罚","PPO-Penalty：在 surrogate 上加 KL 惩罚",[14,41987,41988,41989,42021,42022,42050],{},"如何让训练策略与参考策略不至于偏离太远？最直观的做法是给目标函数加一个 KL 散度惩罚项。KL 散度衡量两个分布的差异：差异越小，KL 越接近 0；差异越大，KL 越大。我们用一个权重 ",[57,41990,41992,42007],{"className":41991},[1019],[57,41993,41995],{"className":41994},[1023],[1025,41996,41997],{"xmlns":1027},[1029,41998,41999,42004],{},[1032,42000,42001],{},[1047,42002,42003],{},"β",[1054,42005,42006],{"encoding":1056},"\\beta",[57,42008,42010],{"className":42009,"ariaHidden":1062},[1061],[57,42011,42013,42017],{"className":42012},[1066],[57,42014],{"className":42015,"style":42016},[1070],"height:0.8889em;vertical-align:-0.1944em;",[57,42018,42003],{"className":42019,"style":42020},[1075,1148],"margin-right:0.05278em;"," 来控制这一惩罚的强度（实际中 ",[57,42023,42025,42038],{"className":42024},[1019],[57,42026,42028],{"className":42027},[1023],[1025,42029,42030],{"xmlns":1027},[1029,42031,42032,42036],{},[1032,42033,42034],{},[1047,42035,42003],{},[1054,42037,42006],{"encoding":1056},[57,42039,42041],{"className":42040,"ariaHidden":1062},[1061],[57,42042,42044,42047],{"className":42043},[1066],[57,42045],{"className":42046,"style":42016},[1070],[57,42048,42003],{"className":42049,"style":42020},[1075,1148]," 还会做自适应调整）：",[57,42052,42054],{"className":42053},[15139],[57,42055,42057,42264],{"className":42056},[1019],[57,42058,42060],{"className":42059},[1023],[1025,42061,42062],{"xmlns":1027,"display":10210},[1029,42063,42064,42261],{},[1032,42065,42066,42068,42070,42072,42079,42081,42083,42089,42103,42121,42139,42141,42149,42151,42159,42161,42227,42229,42231,42233,42239,42241,42247,42249,42259],{},[1047,42067,8490],{},[1047,42069,1748],{},[1047,42071,13535],{},[1044,42073,42074,42076],{},[1047,42075,13535],{},[3999,42077,42078],{},"ppo",[1414,42080,281],{},[1414,42082,19155],{},[1035,42084,42085,42087],{},[1038,42086,574],{},[1047,42088,7990],{},[15472,42090,42091,42093,42101],{},[1414,42092,2549],{},[1032,42094,42095,42097,42099],{},[1047,42096,1288],{},[1414,42098,281],{},[1038,42100,574],{},[1047,42102,7990],{},[15472,42104,42105,42107,42115],{},[1414,42106,2549],{},[1032,42108,42109,42111,42113],{},[1047,42110,1733],{},[1414,42112,281],{},[1038,42114,574],{},[1044,42116,42117,42119],{},[1047,42118,738],{},[1047,42120,1288],{},[2545,42122,42123,42125,42131],{},[1047,42124,1730],{},[1418,42126,42127,42129],{},[1047,42128,14745],{},[1414,42130,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,42132,42133,42135,42137],{},[1047,42134,28613],{},[1047,42136,1730],{},[1047,42138,6336],{},[1414,42140,1754],{"stretchy":1753},[2545,42142,42143,42145,42147],{},[1047,42144,13535],{},[1047,42146,1288],{},[1047,42148,1733],{},[1414,42150,1759],{"separator":1062},[2545,42152,42153,42155,42157],{},[1047,42154,18],{},[1047,42156,1288],{},[1047,42158,1733],{},[1414,42160,1768],{"stretchy":1753},[1035,42162,42163,42193],{},[1032,42164,42165,42171,42173,42181,42183,42191],{},[1044,42166,42167,42169],{},[1047,42168,6333],{},[1047,42170,14745],{},[1414,42172,1754],{"stretchy":1753},[2545,42174,42175,42177,42179],{},[1047,42176,18],{},[1047,42178,1288],{},[1047,42180,1733],{},[1047,42182,13675],{"mathvariant":1729},[2545,42184,42185,42187,42189],{},[1047,42186,13535],{},[1047,42188,1288],{},[1047,42190,1733],{},[1414,42192,1768],{"stretchy":1753},[1032,42194,42195,42205,42207,42215,42217,42225],{},[1044,42196,42197,42199],{},[1047,42198,6333],{},[1418,42200,42201,42203],{},[1047,42202,14745],{},[1414,42204,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,42206,1754],{"stretchy":1753},[2545,42208,42209,42211,42213],{},[1047,42210,18],{},[1047,42212,1288],{},[1047,42214,1733],{},[1047,42216,13675],{"mathvariant":1729},[2545,42218,42219,42221,42223],{},[1047,42220,13535],{},[1047,42222,1288],{},[1047,42224,1733],{},[1414,42226,1768],{"stretchy":1753},[1414,42228,6487],{},[1047,42230,42003],{},[3999,42232,8013],{},[1032,42234,42235,42237],{},[1047,42236,1525],{"mathvariant":1729},[1047,42238,8490],{"mathvariant":1729},[1414,42240,1754],{"stretchy":1753},[1044,42242,42243,42245],{},[1047,42244,6333],{},[1047,42246,14745],{},[1414,42248,1759],{"separator":1062},[1044,42250,42251,42253],{},[1047,42252,6333],{},[1418,42254,42255,42257],{},[1047,42256,14745],{},[1414,42258,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,42260,1768],{"stretchy":1753},[1054,42262,42263],{"encoding":1056},"Loss_{\\text{ppo}} = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)} + \\beta\\,\\mathrm{KL}(P_{\\theta}, P_{\\theta'})",[57,42265,42267,42337,43196],{"className":42266,"ariaHidden":1062},[1061],[57,42268,42270,42274,42277,42281,42328,42331,42334],{"className":42269},[1066],[57,42271],{"className":42272,"style":42273},[1070],"height:0.9694em;vertical-align:-0.2861em;",[57,42275,8490],{"className":42276},[1075,1148],[57,42278,42280],{"className":42279},[1075,1148],"os",[57,42282,42284,42287],{"className":42283},[1075],[57,42285,13535],{"className":42286},[1075,1148],[57,42288,42290],{"className":42289},[1152],[57,42291,42293,42319],{"className":42292},[1087,1088],[57,42294,42296,42316],{"className":42295},[1092],[57,42297,42299],{"className":42298,"style":8990},[1096],[57,42300,42301,42304],{"style":2211},[57,42302],{"className":42303,"style":1493},[1104],[57,42305,42307],{"className":42306},[1109,1110,1111,1112],[57,42308,42310],{"className":42309},[1075,1112],[57,42311,42313],{"className":42312},[1075,6043,1112],[57,42314,42078],{"className":42315},[1075,1112],[57,42317,1183],{"className":42318},[1182],[57,42320,42322],{"className":42321},[1092],[57,42323,42326],{"className":42324,"style":42325},[1096],"height:0.2861em;",[57,42327],{},[57,42329],{"className":42330,"style":1453},[1452],[57,42332,281],{"className":42333},[1457],[57,42335],{"className":42336,"style":1453},[1452],[57,42338,42340,42343,42346,42408,42411,42478,42481,42585,42588,42680,42683,42734,42737,42740,42791,42794,43187,43190,43193],{"className":42339},[1066],[57,42341],{"className":42342,"style":32380},[1070],[57,42344,19155],{"className":42345},[1075],[57,42347,42349,42352,42405],{"className":42348},[1075],[57,42350],{"className":42351},[1079,1080],[57,42353,42355],{"className":42354},[1035],[57,42356,42358,42397],{"className":42357},[1087,1088],[57,42359,42361,42394],{"className":42360},[1092],[57,42362,42364,42375,42383],{"className":42363,"style":17068},[1096],[57,42365,42366,42369],{"style":16608},[57,42367],{"className":42368,"style":1105},[1104],[57,42370,42372],{"className":42371},[1075],[57,42373,7990],{"className":42374,"style":17080},[1075,1148],[57,42376,42377,42380],{"style":1230},[57,42378],{"className":42379,"style":1105},[1104],[57,42381],{"className":42382,"style":1238},[1237],[57,42384,42385,42388],{"style":16674},[57,42386],{"className":42387,"style":1105},[1104],[57,42389,42391],{"className":42390},[1075],[57,42392,574],{"className":42393},[1075],[57,42395,1183],{"className":42396},[1182],[57,42398,42400],{"className":42399},[1092],[57,42401,42403],{"className":42402,"style":17109},[1096],[57,42404],{},[57,42406],{"className":42407},[1269,1080],[57,42409],{"className":42410,"style":1853},[1452],[57,42412,42414],{"className":42413},[1888,15027],[57,42415,42417,42470],{"className":42416},[1087,1088],[57,42418,42420,42467],{"className":42419},[1092],[57,42421,42423,42443,42453],{"className":42422,"style":17130},[1096],[57,42424,42425,42428],{"style":17133},[57,42426],{"className":42427,"style":16221},[1104],[57,42429,42431],{"className":42430},[1109,1110,1111,1112],[57,42432,42434,42437,42440],{"className":42433},[1075,1112],[57,42435,1288],{"className":42436},[1075,1148,1112],[57,42438,281],{"className":42439},[1457,1112],[57,42441,574],{"className":42442},[1075,1112],[57,42444,42445,42448],{"style":16230},[57,42446],{"className":42447,"style":16221},[1104],[57,42449,42450],{},[57,42451,2549],{"className":42452},[1888,2633,16239],[57,42454,42455,42458],{"style":17164},[57,42456],{"className":42457,"style":16221},[1104],[57,42459,42461],{"className":42460},[1109,1110,1111,1112],[57,42462,42464],{"className":42463},[1075,1112],[57,42465,7990],{"className":42466,"style":17080},[1075,1148,1112],[57,42468,1183],{"className":42469},[1182],[57,42471,42473],{"className":42472},[1092],[57,42474,42476],{"className":42475,"style":17183},[1096],[57,42477],{},[57,42479],{"className":42480,"style":1853},[1452],[57,42482,42484],{"className":42483},[1888,15027],[57,42485,42487,42577],{"className":42486},[1087,1088],[57,42488,42490,42574],{"className":42489},[1092],[57,42491,42493,42513,42523],{"className":42492,"style":17939},[1096],[57,42494,42495,42498],{"style":17133},[57,42496],{"className":42497,"style":16221},[1104],[57,42499,42501],{"className":42500},[1109,1110,1111,1112],[57,42502,42504,42507,42510],{"className":42503},[1075,1112],[57,42505,1733],{"className":42506},[1075,1148,1112],[57,42508,281],{"className":42509},[1457,1112],[57,42511,574],{"className":42512},[1075,1112],[57,42514,42515,42518],{"style":16230},[57,42516],{"className":42517,"style":16221},[1104],[57,42519,42520],{},[57,42521,2549],{"className":42522},[1888,2633,16239],[57,42524,42525,42528],{"style":17972},[57,42526],{"className":42527,"style":16221},[1104],[57,42529,42531],{"className":42530},[1109,1110,1111,1112],[57,42532,42534],{"className":42533},[1075,1112],[57,42535,42537,42540],{"className":42536},[1075,1112],[57,42538,738],{"className":42539,"style":3907},[1075,1148,1112],[57,42541,42543],{"className":42542},[1152],[57,42544,42546,42566],{"className":42545},[1087,1088],[57,42547,42549,42563],{"className":42548},[1092],[57,42550,42552],{"className":42551,"style":5022},[1096],[57,42553,42554,42557],{"style":18002},[57,42555],{"className":42556,"style":1169},[1104],[57,42558,42560],{"className":42559},[1109,1173,1174,1112],[57,42561,1288],{"className":42562},[1075,1148,1112],[57,42564,1183],{"className":42565},[1182],[57,42567,42569],{"className":42568},[1092],[57,42570,42572],{"className":42571,"style":5044},[1096],[57,42573],{},[57,42575,1183],{"className":42576},[1182],[57,42578,42580],{"className":42579},[1092],[57,42581,42583],{"className":42582,"style":17183},[1096],[57,42584],{},[57,42586],{"className":42587,"style":1853},[1452],[57,42589,42591,42594],{"className":42590},[1075],[57,42592,1730],{"className":42593},[1075,1148],[57,42595,42597],{"className":42596},[1152],[57,42598,42600,42672],{"className":42599},[1087,1088],[57,42601,42603,42669],{"className":42602},[1092],[57,42604,42606,42649],{"className":42605,"style":25824},[1096],[57,42607,42608,42611],{"style":18101},[57,42609],{"className":42610,"style":1493},[1104],[57,42612,42614],{"className":42613},[1109,1110,1111,1112],[57,42615,42617],{"className":42616},[1075,1112],[57,42618,42620,42623],{"className":42619},[1075,1112],[57,42621,14745],{"className":42622,"style":3932},[1075,1148,1112],[57,42624,42626],{"className":42625},[1152],[57,42627,42629],{"className":42628},[1087],[57,42630,42632],{"className":42631},[1092],[57,42633,42635],{"className":42634,"style":19935},[1096],[57,42636,42637,42640],{"style":19938},[57,42638],{"className":42639,"style":1169},[1104],[57,42641,42643],{"className":42642},[1109,1173,1174,1112],[57,42644,42646],{"className":42645},[1075,1112],[57,42647,19775],{"className":42648},[1075,1112],[57,42650,42651,42654],{"style":17216},[57,42652],{"className":42653,"style":1493},[1104],[57,42655,42657],{"className":42656},[1109,1110,1111,1112],[57,42658,42660,42663,42666],{"className":42659},[1075,1112],[57,42661,28613],{"className":42662},[1075,1148,1112],[57,42664,1730],{"className":42665},[1075,1148,1112],[57,42667,6336],{"className":42668,"style":6749},[1075,1148,1112],[57,42670,1183],{"className":42671},[1182],[57,42673,42675],{"className":42674},[1092],[57,42676,42678],{"className":42677,"style":18131},[1096],[57,42679],{},[57,42681,1754],{"className":42682},[1079],[57,42684,42686,42689],{"className":42685},[1075],[57,42687,13535],{"className":42688},[1075,1148],[57,42690,42692],{"className":42691},[1152],[57,42693,42695,42726],{"className":42694},[1087,1088],[57,42696,42698,42723],{"className":42697},[1092],[57,42699,42701,42712],{"className":42700,"style":18098},[1096],[57,42702,42703,42706],{"style":18101},[57,42704],{"className":42705,"style":1493},[1104],[57,42707,42709],{"className":42708},[1109,1110,1111,1112],[57,42710,1288],{"className":42711},[1075,1148,1112],[57,42713,42714,42717],{"style":17216},[57,42715],{"className":42716,"style":1493},[1104],[57,42718,42720],{"className":42719},[1109,1110,1111,1112],[57,42721,1733],{"className":42722},[1075,1148,1112],[57,42724,1183],{"className":42725},[1182],[57,42727,42729],{"className":42728},[1092],[57,42730,42732],{"className":42731,"style":18131},[1096],[57,42733],{},[57,42735,1759],{"className":42736},[1849],[57,42738],{"className":42739,"style":1853},[1452],[57,42741,42743,42746],{"className":42742},[1075],[57,42744,18],{"className":42745},[1075,1148],[57,42747,42749],{"className":42748},[1152],[57,42750,42752,42783],{"className":42751},[1087,1088],[57,42753,42755,42780],{"className":42754},[1092],[57,42756,42758,42769],{"className":42757,"style":18098},[1096],[57,42759,42760,42763],{"style":18101},[57,42761],{"className":42762,"style":1493},[1104],[57,42764,42766],{"className":42765},[1109,1110,1111,1112],[57,42767,1288],{"className":42768},[1075,1148,1112],[57,42770,42771,42774],{"style":17216},[57,42772],{"className":42773,"style":1493},[1104],[57,42775,42777],{"className":42776},[1109,1110,1111,1112],[57,42778,1733],{"className":42779},[1075,1148,1112],[57,42781,1183],{"className":42782},[1182],[57,42784,42786],{"className":42785},[1092],[57,42787,42789],{"className":42788,"style":18131},[1096],[57,42790],{},[57,42792,1768],{"className":42793},[1269],[57,42795,42797,42800,43184],{"className":42796},[1075],[57,42798],{"className":42799},[1079,1080],[57,42801,42803],{"className":42802},[1035],[57,42804,42806,43176],{"className":42805},[1087,1088],[57,42807,42809,43173],{"className":42808},[1092],[57,42810,42812,43003,43011],{"className":42811,"style":37982},[1096],[57,42813,42814,42817],{"style":16608},[57,42815],{"className":42816,"style":1105},[1104],[57,42818,42820,42892,42895,42946,42949,43000],{"className":42819},[1075],[57,42821,42823,42826],{"className":42822},[1075],[57,42824,6333],{"className":42825,"style":3907},[1075,1148],[57,42827,42829],{"className":42828},[1152],[57,42830,42832,42884],{"className":42831},[1087,1088],[57,42833,42835,42881],{"className":42834},[1092],[57,42836,42838],{"className":42837,"style":2208},[1096],[57,42839,42840,42843],{"style":8131},[57,42841],{"className":42842,"style":1493},[1104],[57,42844,42846],{"className":42845},[1109,1110,1111,1112],[57,42847,42849],{"className":42848},[1075,1112],[57,42850,42852,42855],{"className":42851},[1075,1112],[57,42853,14745],{"className":42854,"style":3932},[1075,1148,1112],[57,42856,42858],{"className":42857},[1152],[57,42859,42861],{"className":42860},[1087],[57,42862,42864],{"className":42863},[1092],[57,42865,42867],{"className":42866,"style":19935},[1096],[57,42868,42869,42872],{"style":19938},[57,42870],{"className":42871,"style":1169},[1104],[57,42873,42875],{"className":42874},[1109,1173,1174,1112],[57,42876,42878],{"className":42877},[1075,1112],[57,42879,19775],{"className":42880},[1075,1112],[57,42882,1183],{"className":42883},[1182],[57,42885,42887],{"className":42886},[1092],[57,42888,42890],{"className":42889,"style":2230},[1096],[57,42891],{},[57,42893,1754],{"className":42894},[1079],[57,42896,42898,42901],{"className":42897},[1075],[57,42899,18],{"className":42900},[1075,1148],[57,42902,42904],{"className":42903},[1152],[57,42905,42907,42938],{"className":42906},[1087,1088],[57,42908,42910,42935],{"className":42909},[1092],[57,42911,42913,42924],{"className":42912,"style":38084},[1096],[57,42914,42915,42918],{"style":18101},[57,42916],{"className":42917,"style":1493},[1104],[57,42919,42921],{"className":42920},[1109,1110,1111,1112],[57,42922,1288],{"className":42923},[1075,1148,1112],[57,42925,42926,42929],{"style":17319},[57,42927],{"className":42928,"style":1493},[1104],[57,42930,42932],{"className":42931},[1109,1110,1111,1112],[57,42933,1733],{"className":42934},[1075,1148,1112],[57,42936,1183],{"className":42937},[1182],[57,42939,42941],{"className":42940},[1092],[57,42942,42944],{"className":42943,"style":18131},[1096],[57,42945],{},[57,42947,13675],{"className":42948},[1075],[57,42950,42952,42955],{"className":42951},[1075],[57,42953,13535],{"className":42954},[1075,1148],[57,42956,42958],{"className":42957},[1152],[57,42959,42961,42992],{"className":42960},[1087,1088],[57,42962,42964,42989],{"className":42963},[1092],[57,42965,42967,42978],{"className":42966,"style":38084},[1096],[57,42968,42969,42972],{"style":18101},[57,42970],{"className":42971,"style":1493},[1104],[57,42973,42975],{"className":42974},[1109,1110,1111,1112],[57,42976,1288],{"className":42977},[1075,1148,1112],[57,42979,42980,42983],{"style":17319},[57,42981],{"className":42982,"style":1493},[1104],[57,42984,42986],{"className":42985},[1109,1110,1111,1112],[57,42987,1733],{"className":42988},[1075,1148,1112],[57,42990,1183],{"className":42991},[1182],[57,42993,42995],{"className":42994},[1092],[57,42996,42998],{"className":42997,"style":18131},[1096],[57,42999],{},[57,43001,1768],{"className":43002},[1269],[57,43004,43005,43008],{"style":1230},[57,43006],{"className":43007,"style":1105},[1104],[57,43009],{"className":43010,"style":1238},[1237],[57,43012,43013,43016],{"style":16674},[57,43014],{"className":43015,"style":1105},[1104],[57,43017,43019,43062,43065,43116,43119,43170],{"className":43018},[1075],[57,43020,43022,43025],{"className":43021},[1075],[57,43023,6333],{"className":43024,"style":3907},[1075,1148],[57,43026,43028],{"className":43027},[1152],[57,43029,43031,43054],{"className":43030},[1087,1088],[57,43032,43034,43051],{"className":43033},[1092],[57,43035,43037],{"className":43036,"style":2208},[1096],[57,43038,43039,43042],{"style":8131},[57,43040],{"className":43041,"style":1493},[1104],[57,43043,43045],{"className":43044},[1109,1110,1111,1112],[57,43046,43048],{"className":43047},[1075,1112],[57,43049,14745],{"className":43050,"style":3932},[1075,1148,1112],[57,43052,1183],{"className":43053},[1182],[57,43055,43057],{"className":43056},[1092],[57,43058,43060],{"className":43059,"style":2230},[1096],[57,43061],{},[57,43063,1754],{"className":43064},[1079],[57,43066,43068,43071],{"className":43067},[1075],[57,43069,18],{"className":43070},[1075,1148],[57,43072,43074],{"className":43073},[1152],[57,43075,43077,43108],{"className":43076},[1087,1088],[57,43078,43080,43105],{"className":43079},[1092],[57,43081,43083,43094],{"className":43082,"style":19640},[1096],[57,43084,43085,43088],{"style":18101},[57,43086],{"className":43087,"style":1493},[1104],[57,43089,43091],{"className":43090},[1109,1110,1111,1112],[57,43092,1288],{"className":43093},[1075,1148,1112],[57,43095,43096,43099],{"style":1489},[57,43097],{"className":43098,"style":1493},[1104],[57,43100,43102],{"className":43101},[1109,1110,1111,1112],[57,43103,1733],{"className":43104},[1075,1148,1112],[57,43106,1183],{"className":43107},[1182],[57,43109,43111],{"className":43110},[1092],[57,43112,43114],{"className":43113,"style":18131},[1096],[57,43115],{},[57,43117,13675],{"className":43118},[1075],[57,43120,43122,43125],{"className":43121},[1075],[57,43123,13535],{"className":43124},[1075,1148],[57,43126,43128],{"className":43127},[1152],[57,43129,43131,43162],{"className":43130},[1087,1088],[57,43132,43134,43159],{"className":43133},[1092],[57,43135,43137,43148],{"className":43136,"style":19640},[1096],[57,43138,43139,43142],{"style":18101},[57,43140],{"className":43141,"style":1493},[1104],[57,43143,43145],{"className":43144},[1109,1110,1111,1112],[57,43146,1288],{"className":43147},[1075,1148,1112],[57,43149,43150,43153],{"style":1489},[57,43151],{"className":43152,"style":1493},[1104],[57,43154,43156],{"className":43155},[1109,1110,1111,1112],[57,43157,1733],{"className":43158},[1075,1148,1112],[57,43160,1183],{"className":43161},[1182],[57,43163,43165],{"className":43164},[1092],[57,43166,43168],{"className":43167,"style":18131},[1096],[57,43169],{},[57,43171,1768],{"className":43172},[1269],[57,43174,1183],{"className":43175},[1182],[57,43177,43179],{"className":43178},[1092],[57,43180,43182],{"className":43181,"style":16739},[1096],[57,43183],{},[57,43185],{"className":43186},[1269,1080],[57,43188],{"className":43189,"style":2595},[1452],[57,43191,6487],{"className":43192},[1506],[57,43194],{"className":43195,"style":2595},[1452],[57,43197,43199,43202,43205,43208,43215,43218,43261,43264,43267,43339],{"className":43198},[1066],[57,43200],{"className":43201,"style":1831},[1070],[57,43203,42003],{"className":43204,"style":42020},[1075,1148],[57,43206],{"className":43207,"style":1853},[1452],[57,43209,43211],{"className":43210},[1075],[57,43212,43214],{"className":43213},[1075,1838],"KL",[57,43216,1754],{"className":43217},[1079],[57,43219,43221,43224],{"className":43220},[1075],[57,43222,6333],{"className":43223,"style":3907},[1075,1148],[57,43225,43227],{"className":43226},[1152],[57,43228,43230,43253],{"className":43229},[1087,1088],[57,43231,43233,43250],{"className":43232},[1092],[57,43234,43236],{"className":43235,"style":2208},[1096],[57,43237,43238,43241],{"style":8131},[57,43239],{"className":43240,"style":1493},[1104],[57,43242,43244],{"className":43243},[1109,1110,1111,1112],[57,43245,43247],{"className":43246},[1075,1112],[57,43248,14745],{"className":43249,"style":3932},[1075,1148,1112],[57,43251,1183],{"className":43252},[1182],[57,43254,43256],{"className":43255},[1092],[57,43257,43259],{"className":43258,"style":2230},[1096],[57,43260],{},[57,43262,1759],{"className":43263},[1849],[57,43265],{"className":43266,"style":1853},[1452],[57,43268,43270,43273],{"className":43269},[1075],[57,43271,6333],{"className":43272,"style":3907},[1075,1148],[57,43274,43276],{"className":43275},[1152],[57,43277,43279,43331],{"className":43278},[1087,1088],[57,43280,43282,43328],{"className":43281},[1092],[57,43283,43285],{"className":43284,"style":2208},[1096],[57,43286,43287,43290],{"style":8131},[57,43288],{"className":43289,"style":1493},[1104],[57,43291,43293],{"className":43292},[1109,1110,1111,1112],[57,43294,43296],{"className":43295},[1075,1112],[57,43297,43299,43302],{"className":43298},[1075,1112],[57,43300,14745],{"className":43301,"style":3932},[1075,1148,1112],[57,43303,43305],{"className":43304},[1152],[57,43306,43308],{"className":43307},[1087],[57,43309,43311],{"className":43310},[1092],[57,43312,43314],{"className":43313,"style":19935},[1096],[57,43315,43316,43319],{"style":19938},[57,43317],{"className":43318,"style":1169},[1104],[57,43320,43322],{"className":43321},[1109,1173,1174,1112],[57,43323,43325],{"className":43324},[1075,1112],[57,43326,19775],{"className":43327},[1075,1112],[57,43329,1183],{"className":43330},[1182],[57,43332,43334],{"className":43333},[1092],[57,43335,43337],{"className":43336,"style":2230},[1096],[57,43338],{},[57,43340,1768],{"className":43341},[1269],[41982,43343,43345],{"id":43344},"ppo-clip截断重要性采样比","PPO-Clip：截断重要性采样比",[14,43347,43348,43349,43352,43353,727],{},"PPO 还有一种更常用的实现，用 ",[11865,43350,43351],{},"截断（clip）重要性采样比"," 来代替 KL 惩罚，同样起到限制新旧策略偏差的作用。它的目标函数由两部分组成（下方红色与蓝色项），最终取两者的 ",[11865,43354,43355],{},"较小值",[13410,43357,43358,43708],{},[13413,43359,43360,43363,43364,43416,43417,43707],{},[11865,43361,43362],{},"红色部分","：原始的 surrogate 目标 ",[57,43365,43367,43386],{"className":43366},[1019],[57,43368,43370],{"className":43369},[1023],[1025,43371,43372],{"xmlns":1027},[1029,43373,43374,43383],{},[1032,43375,43376,43379,43381],{},[1047,43377,43378],{},"ρ",[1414,43380,2539],{},[1047,43382,1730],{},[1054,43384,43385],{"encoding":1056},"\\rho \\cdot A",[57,43387,43389,43407],{"className":43388,"ariaHidden":1062},[1061],[57,43390,43392,43395,43398,43401,43404],{"className":43391},[1066],[57,43393],{"className":43394,"style":2588},[1070],[57,43396,43378],{"className":43397},[1075,1148],[57,43399],{"className":43400,"style":2595},[1452],[57,43402,2539],{"className":43403},[1506],[57,43405],{"className":43406,"style":2595},[1452],[57,43408,43410,43413],{"className":43409},[1066],[57,43411],{"className":43412,"style":19250},[1070],[57,43414,1730],{"className":43415},[1075,1148],"，其中 ",[57,43418,43420,43478],{"className":43419},[1019],[57,43421,43423],{"className":43422},[1023],[1025,43424,43425],{"xmlns":1027},[1029,43426,43427,43475],{},[1032,43428,43429,43431,43433],{},[1047,43430,43378],{},[1414,43432,281],{},[1035,43434,43435,43453],{},[1032,43436,43437,43443,43445,43447,43449,43451],{},[1044,43438,43439,43441],{},[1047,43440,6333],{},[1047,43442,14745],{},[1414,43444,1754],{"stretchy":1753},[1047,43446,18],{},[1414,43448,13675],{},[1047,43450,13535],{},[1414,43452,1768],{"stretchy":1753},[1032,43454,43455,43465,43467,43469,43471,43473],{},[1044,43456,43457,43459],{},[1047,43458,6333],{},[1418,43460,43461,43463],{},[1047,43462,14745],{},[1414,43464,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,43466,1754],{"stretchy":1753},[1047,43468,18],{},[1414,43470,13675],{},[1047,43472,13535],{},[1414,43474,1768],{"stretchy":1753},[1054,43476,43477],{"encoding":1056},"\\rho = \\frac{P_\\theta(a \\mid s)}{P_{\\theta'}(a \\mid s)}",[57,43479,43481,43499],{"className":43480,"ariaHidden":1062},[1061],[57,43482,43484,43487,43490,43493,43496],{"className":43483},[1066],[57,43485],{"className":43486,"style":2430},[1070],[57,43488,43378],{"className":43489},[1075,1148],[57,43491],{"className":43492,"style":1453},[1452],[57,43494,281],{"className":43495},[1457],[57,43497],{"className":43498,"style":1453},[1452],[57,43500,43502,43506],{"className":43501},[1066],[57,43503],{"className":43504,"style":43505},[1070],"height:1.5384em;vertical-align:-0.5284em;",[57,43507,43509,43512,43704],{"className":43508},[1075],[57,43510],{"className":43511},[1079,1080],[57,43513,43515],{"className":43514},[1035],[57,43516,43518,43696],{"className":43517},[1087,1088],[57,43519,43521,43693],{"className":43520},[1092],[57,43522,43524,43619,43627],{"className":43523,"style":33415},[1096],[57,43525,43526,43529],{"style":19297},[57,43527],{"className":43528,"style":1105},[1104],[57,43530,43532],{"className":43531},[1109,1110,1111,1112],[57,43533,43535,43604,43607,43610,43613,43616],{"className":43534},[1075,1112],[57,43536,43538,43541],{"className":43537},[1075,1112],[57,43539,6333],{"className":43540,"style":3907},[1075,1148,1112],[57,43542,43544],{"className":43543},[1152],[57,43545,43547,43596],{"className":43546},[1087,1088],[57,43548,43550,43593],{"className":43549},[1092],[57,43551,43553],{"className":43552,"style":1162},[1096],[57,43554,43555,43558],{"style":41850},[57,43556],{"className":43557,"style":41854},[1104],[57,43559,43561],{"className":43560},[1109,1173,1174,1112],[57,43562,43564],{"className":43563},[1075,1112],[57,43565,43567,43570],{"className":43566},[1075,1112],[57,43568,14745],{"className":43569,"style":3932},[1075,1148,1112],[57,43571,43573],{"className":43572},[1152],[57,43574,43576],{"className":43575},[1087],[57,43577,43579],{"className":43578},[1092],[57,43580,43582],{"className":43581,"style":41879},[1096],[57,43583,43584,43587],{"style":41882},[57,43585],{"className":43586,"style":41886},[1104],[57,43588,43590],{"className":43589},[1075,1112],[57,43591,19775],{"className":43592},[1075,1112],[57,43594,1183],{"className":43595},[1182],[57,43597,43599],{"className":43598},[1092],[57,43600,43602],{"className":43601,"style":41902},[1096],[57,43603],{},[57,43605,1754],{"className":43606},[1079,1112],[57,43608,18],{"className":43609},[1075,1148,1112],[57,43611,13675],{"className":43612},[1457,1112],[57,43614,13535],{"className":43615},[1075,1148,1112],[57,43617,1768],{"className":43618},[1269,1112],[57,43620,43621,43624],{"style":1230},[57,43622],{"className":43623,"style":1105},[1104],[57,43625],{"className":43626,"style":1238},[1237],[57,43628,43629,43632],{"style":33449},[57,43630],{"className":43631,"style":1105},[1104],[57,43633,43635],{"className":43634},[1109,1110,1111,1112],[57,43636,43638,43678,43681,43684,43687,43690],{"className":43637},[1075,1112],[57,43639,43641,43644],{"className":43640},[1075,1112],[57,43642,6333],{"className":43643,"style":3907},[1075,1148,1112],[57,43645,43647],{"className":43646},[1152],[57,43648,43650,43670],{"className":43649},[1087,1088],[57,43651,43653,43667],{"className":43652},[1092],[57,43654,43656],{"className":43655,"style":1162},[1096],[57,43657,43658,43661],{"style":14964},[57,43659],{"className":43660,"style":1169},[1104],[57,43662,43664],{"className":43663},[1109,1173,1174,1112],[57,43665,14745],{"className":43666,"style":3932},[1075,1148,1112],[57,43668,1183],{"className":43669},[1182],[57,43671,43673],{"className":43672},[1092],[57,43674,43676],{"className":43675,"style":1190},[1096],[57,43677],{},[57,43679,1754],{"className":43680},[1079,1112],[57,43682,18],{"className":43683},[1075,1148,1112],[57,43685,13675],{"className":43686},[1457,1112],[57,43688,13535],{"className":43689},[1075,1148,1112],[57,43691,1768],{"className":43692},[1269,1112],[57,43694,1183],{"className":43695},[1182],[57,43697,43699],{"className":43698},[1092],[57,43700,43702],{"className":43701,"style":41974},[1096],[57,43703],{},[57,43705],{"className":43706},[1269,1080]," 是重要性采样比；",[13413,43709,43710,43713,43714,43743,43744,43847,43848,43876,43877,43905],{},[11865,43711,43712],{},"蓝色部分","：把 ",[57,43715,43717,43731],{"className":43716},[1019],[57,43718,43720],{"className":43719},[1023],[1025,43721,43722],{"xmlns":1027},[1029,43723,43724,43728],{},[1032,43725,43726],{},[1047,43727,43378],{},[1054,43729,43730],{"encoding":1056},"\\rho",[57,43732,43734],{"className":43733,"ariaHidden":1062},[1061],[57,43735,43737,43740],{"className":43736},[1066],[57,43738],{"className":43739,"style":2430},[1070],[57,43741,43378],{"className":43742},[1075,1148]," 截断到 ",[57,43745,43747,43780],{"className":43746},[1019],[57,43748,43750],{"className":43749},[1023],[1025,43751,43752],{"xmlns":1027},[1029,43753,43754,43777],{},[1032,43755,43756,43758,43760,43762,43765,43767,43769,43771,43773,43775],{},[1414,43757,12096],{"stretchy":1753},[1038,43759,574],{},[1414,43761,19155],{},[1047,43763,43764],{},"ϵ",[1414,43766,1759],{"separator":1062},[3999,43768,8013],{},[1038,43770,574],{},[1414,43772,6487],{},[1047,43774,43764],{},[1414,43776,12411],{"stretchy":1753},[1054,43778,43779],{"encoding":1056},"[1-\\epsilon,\\, 1+\\epsilon]",[57,43781,43783,43804,43835],{"className":43782,"ariaHidden":1062},[1061],[57,43784,43786,43789,43792,43795,43798,43801],{"className":43785},[1066],[57,43787],{"className":43788,"style":1831},[1070],[57,43790,12096],{"className":43791},[1079],[57,43793,574],{"className":43794},[1075],[57,43796],{"className":43797,"style":2595},[1452],[57,43799,19155],{"className":43800},[1506],[57,43802],{"className":43803,"style":2595},[1452],[57,43805,43807,43811,43814,43817,43820,43823,43826,43829,43832],{"className":43806},[1066],[57,43808],{"className":43809,"style":43810},[1070],"height:0.8389em;vertical-align:-0.1944em;",[57,43812,43764],{"className":43813},[1075,1148],[57,43815,1759],{"className":43816},[1849],[57,43818],{"className":43819,"style":1853},[1452],[57,43821],{"className":43822,"style":1853},[1452],[57,43824,574],{"className":43825},[1075],[57,43827],{"className":43828,"style":2595},[1452],[57,43830,6487],{"className":43831},[1506],[57,43833],{"className":43834,"style":2595},[1452],[57,43836,43838,43841,43844],{"className":43837},[1066],[57,43839],{"className":43840,"style":1831},[1070],[57,43842,43764],{"className":43843},[1075,1148],[57,43845,12411],{"className":43846},[1269]," 区间内之后再乘以 ",[57,43849,43851,43864],{"className":43850},[1019],[57,43852,43854],{"className":43853},[1023],[1025,43855,43856],{"xmlns":1027},[1029,43857,43858,43862],{},[1032,43859,43860],{},[1047,43861,1730],{},[1054,43863,1730],{"encoding":1056},[57,43865,43867],{"className":43866,"ariaHidden":1062},[1061],[57,43868,43870,43873],{"className":43869},[1066],[57,43871],{"className":43872,"style":19250},[1070],[57,43874,1730],{"className":43875},[1075,1148],"；当 ",[57,43878,43880,43893],{"className":43879},[1019],[57,43881,43883],{"className":43882},[1023],[1025,43884,43885],{"xmlns":1027},[1029,43886,43887,43891],{},[1032,43888,43889],{},[1047,43890,43378],{},[1054,43892,43730],{"encoding":1056},[57,43894,43896],{"className":43895,"ariaHidden":1062},[1061],[57,43897,43899,43902],{"className":43898},[1066],[57,43900],{"className":43901,"style":2430},[1070],[57,43903,43378],{"className":43904},[1075,1148]," 落在区间内时返回原值，落在外面则返回最近的边界值。",[14,43907,43908,43909,43960,43961,43989,43990,43960,44041,44069],{},"对二者取 min 的目的是：当一个动作有正向优势 ",[57,43910,43912,43930],{"className":43911},[1019],[57,43913,43915],{"className":43914},[1023],[1025,43916,43917],{"xmlns":1027},[1029,43918,43919,43927],{},[1032,43920,43921,43923,43925],{},[1047,43922,1730],{},[1414,43924,11817],{},[1038,43926,560],{},[1054,43928,43929],{"encoding":1056},"A > 0",[57,43931,43933,43951],{"className":43932,"ariaHidden":1062},[1061],[57,43934,43936,43939,43942,43945,43948],{"className":43935},[1066],[57,43937],{"className":43938,"style":1552},[1070],[57,43940,1730],{"className":43941},[1075,1148],[57,43943],{"className":43944,"style":1453},[1452],[57,43946,11817],{"className":43947},[1457],[57,43949],{"className":43950,"style":1453},[1452],[57,43952,43954,43957],{"className":43953},[1066],[57,43955],{"className":43956,"style":2489},[1070],[57,43958,560],{"className":43959},[1075]," 时，限制 ",[57,43962,43964,43977],{"className":43963},[1019],[57,43965,43967],{"className":43966},[1023],[1025,43968,43969],{"xmlns":1027},[1029,43970,43971,43975],{},[1032,43972,43973],{},[1047,43974,43378],{},[1054,43976,43730],{"encoding":1056},[57,43978,43980],{"className":43979,"ariaHidden":1062},[1061],[57,43981,43983,43986],{"className":43982},[1066],[57,43984],{"className":43985,"style":2430},[1070],[57,43987,43378],{"className":43988},[1075,1148]," 不会被推得过高（避免步子迈太大）；当 ",[57,43991,43993,44011],{"className":43992},[1019],[57,43994,43996],{"className":43995},[1023],[1025,43997,43998],{"xmlns":1027},[1029,43999,44000,44008],{},[1032,44001,44002,44004,44006],{},[1047,44003,1730],{},[1414,44005,11488],{},[1038,44007,560],{},[1054,44009,44010],{"encoding":1056},"A \u003C 0",[57,44012,44014,44032],{"className":44013,"ariaHidden":1062},[1061],[57,44015,44017,44020,44023,44026,44029],{"className":44016},[1066],[57,44018],{"className":44019,"style":1552},[1070],[57,44021,1730],{"className":44022},[1075,1148],[57,44024],{"className":44025,"style":1453},[1452],[57,44027,11488],{"className":44028},[1457],[57,44030],{"className":44031,"style":1453},[1452],[57,44033,44035,44038],{"className":44034},[1066],[57,44036],{"className":44037,"style":2489},[1070],[57,44039,560],{"className":44040},[1075],[57,44042,44044,44057],{"className":44043},[1019],[57,44045,44047],{"className":44046},[1023],[1025,44048,44049],{"xmlns":1027},[1029,44050,44051,44055],{},[1032,44052,44053],{},[1047,44054,43378],{},[1054,44056,43730],{"encoding":1056},[57,44058,44060],{"className":44059,"ariaHidden":1062},[1061],[57,44061,44063,44066],{"className":44062},[1066],[57,44064],{"className":44065,"style":2430},[1070],[57,44067,43378],{"className":44068},[1075,1148]," 不会被压得过低。这样既保证了\"敢于改进策略\"，又避免了\"一次更新偏离太多\"。",[57,44071,44073],{"className":44072},[15139],[57,44074,44076,44412],{"className":44075},[1019],[57,44077,44079],{"className":44078},[1023],[1025,44080,44081],{"xmlns":1027,"display":10210},[1029,44082,44083,44409],{},[1032,44084,44085,44087,44089,44091,44098,44100,44102,44108,44122,44140,44143,44145],{},[1047,44086,8490],{},[1047,44088,1748],{},[1047,44090,13535],{},[1044,44092,44093,44095],{},[1047,44094,13535],{},[3999,44096,44097],{},"ppo2",[1414,44099,281],{},[1414,44101,19155],{},[1035,44103,44104,44106],{},[1038,44105,574],{},[1047,44107,7990],{},[15472,44109,44110,44112,44120],{},[1414,44111,2549],{},[1032,44113,44114,44116,44118],{},[1047,44115,1288],{},[1414,44117,281],{},[1038,44119,574],{},[1047,44121,7990],{},[15472,44123,44124,44126,44134],{},[1414,44125,2549],{},[1032,44127,44128,44130,44132],{},[1047,44129,1733],{},[1414,44131,281],{},[1038,44133,574],{},[1044,44135,44136,44138],{},[1047,44137,738],{},[1047,44139,1288],{},[1047,44141,44142],{},"min",[1414,44144,1776],{},[1032,44146,44147,44149,44258,44260,44262,44407],{},[1414,44148,1754],{"fence":1062},[15163,44150,44152,44170,44172,44180,44182,44190,44192],{"mathcolor":44151},"red",[2545,44153,44154,44156,44162],{},[1047,44155,1730],{},[1418,44157,44158,44160],{},[1047,44159,14745],{},[1414,44161,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,44163,44164,44166,44168],{},[1047,44165,28613],{},[1047,44167,1730],{},[1047,44169,6336],{},[1414,44171,1754],{"stretchy":1753},[2545,44173,44174,44176,44178],{},[1047,44175,13535],{},[1047,44177,1288],{},[1047,44179,1733],{},[1414,44181,1759],{"separator":1062},[2545,44183,44184,44186,44188],{},[1047,44185,18],{},[1047,44187,1288],{},[1047,44189,1733],{},[1414,44191,1768],{"stretchy":1753},[1035,44193,44194,44224],{},[1032,44195,44196,44202,44204,44212,44214,44222],{},[1044,44197,44198,44200],{},[1047,44199,6333],{},[1047,44201,14745],{},[1414,44203,1754],{"stretchy":1753},[2545,44205,44206,44208,44210],{},[1047,44207,18],{},[1047,44209,1288],{},[1047,44211,1733],{},[1047,44213,13675],{"mathvariant":1729},[2545,44215,44216,44218,44220],{},[1047,44217,13535],{},[1047,44219,1288],{},[1047,44221,1733],{},[1414,44223,1768],{"stretchy":1753},[1032,44225,44226,44236,44238,44246,44248,44256],{},[1044,44227,44228,44230],{},[1047,44229,6333],{},[1418,44231,44232,44234],{},[1047,44233,14745],{},[1414,44235,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,44237,1754],{"stretchy":1753},[2545,44239,44240,44242,44244],{},[1047,44241,18],{},[1047,44243,1288],{},[1047,44245,1733],{},[1047,44247,13675],{"mathvariant":1729},[2545,44249,44250,44252,44254],{},[1047,44251,13535],{},[1047,44253,1288],{},[1047,44255,1733],{},[1414,44257,1768],{"stretchy":1753},[1414,44259,1759],{"separator":1062},[3999,44261,8013],{},[15163,44263,44265,44275,44367,44385,44387,44395,44397,44405],{"mathcolor":44264},"skyblue",[1032,44266,44267,44269,44271,44273],{},[1047,44268,3667],{"mathvariant":1729},[1047,44270,3628],{"mathvariant":1729},[1047,44272,1745],{"mathvariant":1729},[1047,44274,14],{"mathvariant":1729},[1032,44276,44277,44279,44345,44347,44349,44351,44353,44355,44357,44359,44361,44363,44365],{},[1414,44278,1754],{"fence":1062},[1035,44280,44281,44311],{},[1032,44282,44283,44289,44291,44299,44301,44309],{},[1044,44284,44285,44287],{},[1047,44286,6333],{},[1047,44288,14745],{},[1414,44290,1754],{"stretchy":1753},[2545,44292,44293,44295,44297],{},[1047,44294,18],{},[1047,44296,1288],{},[1047,44298,1733],{},[1047,44300,13675],{"mathvariant":1729},[2545,44302,44303,44305,44307],{},[1047,44304,13535],{},[1047,44306,1288],{},[1047,44308,1733],{},[1414,44310,1768],{"stretchy":1753},[1032,44312,44313,44323,44325,44333,44335,44343],{},[1044,44314,44315,44317],{},[1047,44316,6333],{},[1418,44318,44319,44321],{},[1047,44320,14745],{},[1414,44322,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1414,44324,1754],{"stretchy":1753},[2545,44326,44327,44329,44331],{},[1047,44328,18],{},[1047,44330,1288],{},[1047,44332,1733],{},[1047,44334,13675],{"mathvariant":1729},[2545,44336,44337,44339,44341],{},[1047,44338,13535],{},[1047,44340,1288],{},[1047,44342,1733],{},[1414,44344,1768],{"stretchy":1753},[1414,44346,1759],{"separator":1062},[3999,44348,8013],{},[1038,44350,574],{},[1414,44352,19155],{},[1047,44354,43764],{},[1414,44356,1759],{"separator":1062},[3999,44358,8013],{},[1038,44360,574],{},[1414,44362,6487],{},[1047,44364,43764],{},[1414,44366,1768],{"fence":1062},[2545,44368,44369,44371,44377],{},[1047,44370,1730],{},[1418,44372,44373,44375],{},[1047,44374,14745],{},[1414,44376,19775],{"mathvariant":1729,"lspace":15155,"rspace":15155},[1032,44378,44379,44381,44383],{},[1047,44380,28613],{},[1047,44382,1730],{},[1047,44384,6336],{},[1414,44386,1754],{"stretchy":1753},[2545,44388,44389,44391,44393],{},[1047,44390,13535],{},[1047,44392,1288],{},[1047,44394,1733],{},[1414,44396,1759],{"separator":1062},[2545,44398,44399,44401,44403],{},[1047,44400,18],{},[1047,44402,1288],{},[1047,44404,1733],{},[1414,44406,1768],{"stretchy":1753},[1414,44408,1768],{"fence":1062},[1054,44410,44411],{"encoding":1056},"Loss_{\\text{ppo2}} = -\\frac{1}{N} \\sum_{n=1}^{N} \\sum_{t=1}^{T_n} \\min\\left(\\textcolor{red}{A_{\\theta'}^{GAE}(s_n^t, a_n^t) \\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)}},\\,\\textcolor{skyblue}{\\mathrm{clip}\\left(\\frac{P_{\\theta}(a_n^t|s_n^t)}{P_{\\theta'}(a_n^t|s_n^t)},\\, 1-\\epsilon,\\, 1+\\epsilon\\right) A_{\\theta'}^{GAE}(s_n^t, a_n^t)}\\right)",[57,44413,44415,44482],{"className":44414,"ariaHidden":1062},[1061],[57,44416,44418,44421,44424,44427,44473,44476,44479],{"className":44417},[1066],[57,44419],{"className":44420,"style":42273},[1070],[57,44422,8490],{"className":44423},[1075,1148],[57,44425,42280],{"className":44426},[1075,1148],[57,44428,44430,44433],{"className":44429},[1075],[57,44431,13535],{"className":44432},[1075,1148],[57,44434,44436],{"className":44435},[1152],[57,44437,44439,44465],{"className":44438},[1087,1088],[57,44440,44442,44462],{"className":44441},[1092],[57,44443,44445],{"className":44444,"style":3813},[1096],[57,44446,44447,44450],{"style":2211},[57,44448],{"className":44449,"style":1493},[1104],[57,44451,44453],{"className":44452},[1109,1110,1111,1112],[57,44454,44456],{"className":44455},[1075,1112],[57,44457,44459],{"className":44458},[1075,6043,1112],[57,44460,44097],{"className":44461},[1075,1112],[57,44463,1183],{"className":44464},[1182],[57,44466,44468],{"className":44467},[1092],[57,44469,44471],{"className":44470,"style":42325},[1096],[57,44472],{},[57,44474],{"className":44475,"style":1453},[1452],[57,44477,281],{"className":44478},[1457],[57,44480],{"className":44481,"style":1453},[1452],[57,44483,44485,44488,44491,44553,44556,44623,44626,44730,44733,44736,44739],{"className":44484},[1066],[57,44486],{"className":44487,"style":32380},[1070],[57,44489,19155],{"className":44490},[1075],[57,44492,44494,44497,44550],{"className":44493},[1075],[57,44495],{"className":44496},[1079,1080],[57,44498,44500],{"className":44499},[1035],[57,44501,44503,44542],{"className":44502},[1087,1088],[57,44504,44506,44539],{"className":44505},[1092],[57,44507,44509,44520,44528],{"className":44508,"style":17068},[1096],[57,44510,44511,44514],{"style":16608},[57,44512],{"className":44513,"style":1105},[1104],[57,44515,44517],{"className":44516},[1075],[57,44518,7990],{"className":44519,"style":17080},[1075,1148],[57,44521,44522,44525],{"style":1230},[57,44523],{"className":44524,"style":1105},[1104],[57,44526],{"className":44527,"style":1238},[1237],[57,44529,44530,44533],{"style":16674},[57,44531],{"className":44532,"style":1105},[1104],[57,44534,44536],{"className":44535},[1075],[57,44537,574],{"className":44538},[1075],[57,44540,1183],{"className":44541},[1182],[57,44543,44545],{"className":44544},[1092],[57,44546,44548],{"className":44547,"style":17109},[1096],[57,44549],{},[57,44551],{"className":44552},[1269,1080],[57,44554],{"className":44555,"style":1853},[1452],[57,44557,44559],{"className":44558},[1888,15027],[57,44560,44562,44615],{"className":44561},[1087,1088],[57,44563,44565,44612],{"className":44564},[1092],[57,44566,44568,44588,44598],{"className":44567,"style":17130},[1096],[57,44569,44570,44573],{"style":17133},[57,44571],{"className":44572,"style":16221},[1104],[57,44574,44576],{"className":44575},[1109,1110,1111,1112],[57,44577,44579,44582,44585],{"className":44578},[1075,1112],[57,44580,1288],{"className":44581},[1075,1148,1112],[57,44583,281],{"className":44584},[1457,1112],[57,44586,574],{"className":44587},[1075,1112],[57,44589,44590,44593],{"style":16230},[57,44591],{"className":44592,"style":16221},[1104],[57,44594,44595],{},[57,44596,2549],{"className":44597},[1888,2633,16239],[57,44599,44600,44603],{"style":17164},[57,44601],{"className":44602,"style":16221},[1104],[57,44604,44606],{"className":44605},[1109,1110,1111,1112],[57,44607,44609],{"className":44608},[1075,1112],[57,44610,7990],{"className":44611,"style":17080},[1075,1148,1112],[57,44613,1183],{"className":44614},[1182],[57,44616,44618],{"className":44617},[1092],[57,44619,44621],{"className":44620,"style":17183},[1096],[57,44622],{},[57,44624],{"className":44625,"style":1853},[1452],[57,44627,44629],{"className":44628},[1888,15027],[57,44630,44632,44722],{"className":44631},[1087,1088],[57,44633,44635,44719],{"className":44634},[1092],[57,44636,44638,44658,44668],{"className":44637,"style":17939},[1096],[57,44639,44640,44643],{"style":17133},[57,44641],{"className":44642,"style":16221},[1104],[57,44644,44646],{"className":44645},[1109,1110,1111,1112],[57,44647,44649,44652,44655],{"className":44648},[1075,1112],[57,44650,1733],{"className":44651},[1075,1148,1112],[57,44653,281],{"className":44654},[1457,1112],[57,44656,574],{"className":44657},[1075,1112],[57,44659,44660,44663],{"style":16230},[57,44661],{"className":44662,"style":16221},[1104],[57,44664,44665],{},[57,44666,2549],{"className":44667},[1888,2633,16239],[57,44669,44670,44673],{"style":17972},[57,44671],{"className":44672,"style":16221},[1104],[57,44674,44676],{"className":44675},[1109,1110,1111,1112],[57,44677,44679],{"className":44678},[1075,1112],[57,44680,44682,44685],{"className":44681},[1075,1112],[57,44683,738],{"className":44684,"style":3907},[1075,1148,1112],[57,44686,44688],{"className":44687},[1152],[57,44689,44691,44711],{"className":44690},[1087,1088],[57,44692,44694,44708],{"className":44693},[1092],[57,44695,44697],{"className":44696,"style":5022},[1096],[57,44698,44699,44702],{"style":18002},[57,44700],{"className":44701,"style":1169},[1104],[57,44703,44705],{"className":44704},[1109,1173,1174,1112],[57,44706,1288],{"className":44707},[1075,1148,1112],[57,44709,1183],{"className":44710},[1182],[57,44712,44714],{"className":44713},[1092],[57,44715,44717],{"className":44716,"style":5044},[1096],[57,44718],{},[57,44720,1183],{"className":44721},[1182],[57,44723,44725],{"className":44724},[1092],[57,44726,44728],{"className":44727,"style":17183},[1096],[57,44729],{},[57,44731],{"className":44732,"style":1853},[1452],[57,44734,44142],{"className":44735},[1888],[57,44737],{"className":44738,"style":1853},[1452],[57,44740,44742,44748,44843,44846,44897,44900,44903,44954,44957,45352,45355,45358,45361,45369,45372,45838,45841,45934,45937,45988,45991,45994,46045,46048],{"className":44741},[1898],[57,44743,44745],{"className":44744,"style":1903},[1079,1902],[57,44746,1754],{"className":44747},[1907,1111],[57,44749,44752,44755],{"className":44750,"style":44751},[1075],"color:red;",[57,44753,1730],{"className":44754,"style":44751},[1075,1148],[57,44756,44758],{"className":44757},[1152],[57,44759,44761,44835],{"className":44760},[1087,1088],[57,44762,44764,44832],{"className":44763},[1092],[57,44765,44767,44811],{"className":44766,"style":25824},[1096],[57,44768,44769,44772],{"style":18101},[57,44770],{"className":44771,"style":1493},[1104],[57,44773,44775],{"className":44774,"style":44751},[1109,1110,1111,1112],[57,44776,44778],{"className":44777,"style":44751},[1075,1112],[57,44779,44781,44785],{"className":44780,"style":44751},[1075,1112],[57,44782,14745],{"className":44783,"style":44784},[1075,1148,1112],"margin-right:0.02778em;color:red;",[57,44786,44788],{"className":44787},[1152],[57,44789,44791],{"className":44790},[1087],[57,44792,44794],{"className":44793},[1092],[57,44795,44797],{"className":44796,"style":19935},[1096],[57,44798,44799,44802],{"style":19938},[57,44800],{"className":44801,"style":1169},[1104],[57,44803,44805],{"className":44804,"style":44751},[1109,1173,1174,1112],[57,44806,44808],{"className":44807,"style":44751},[1075,1112],[57,44809,19775],{"className":44810,"style":44751},[1075,1112],[57,44812,44813,44816],{"style":17216},[57,44814],{"className":44815,"style":1493},[1104],[57,44817,44819],{"className":44818,"style":44751},[1109,1110,1111,1112],[57,44820,44822,44825,44828],{"className":44821,"style":44751},[1075,1112],[57,44823,28613],{"className":44824,"style":44751},[1075,1148,1112],[57,44826,1730],{"className":44827,"style":44751},[1075,1148,1112],[57,44829,6336],{"className":44830,"style":44831},[1075,1148,1112],"margin-right:0.05764em;color:red;",[57,44833,1183],{"className":44834},[1182],[57,44836,44838],{"className":44837},[1092],[57,44839,44841],{"className":44840,"style":18131},[1096],[57,44842],{},[57,44844,1754],{"className":44845,"style":44751},[1079],[57,44847,44849,44852],{"className":44848,"style":44751},[1075],[57,44850,13535],{"className":44851,"style":44751},[1075,1148],[57,44853,44855],{"className":44854},[1152],[57,44856,44858,44889],{"className":44857},[1087,1088],[57,44859,44861,44886],{"className":44860},[1092],[57,44862,44864,44875],{"className":44863,"style":18098},[1096],[57,44865,44866,44869],{"style":18101},[57,44867],{"className":44868,"style":1493},[1104],[57,44870,44872],{"className":44871,"style":44751},[1109,1110,1111,1112],[57,44873,1288],{"className":44874,"style":44751},[1075,1148,1112],[57,44876,44877,44880],{"style":17216},[57,44878],{"className":44879,"style":1493},[1104],[57,44881,44883],{"className":44882,"style":44751},[1109,1110,1111,1112],[57,44884,1733],{"className":44885,"style":44751},[1075,1148,1112],[57,44887,1183],{"className":44888},[1182],[57,44890,44892],{"className":44891},[1092],[57,44893,44895],{"className":44894,"style":18131},[1096],[57,44896],{},[57,44898,1759],{"className":44899,"style":44751},[1849],[57,44901],{"className":44902,"style":1853},[1452],[57,44904,44906,44909],{"className":44905,"style":44751},[1075],[57,44907,18],{"className":44908,"style":44751},[1075,1148],[57,44910,44912],{"className":44911},[1152],[57,44913,44915,44946],{"className":44914},[1087,1088],[57,44916,44918,44943],{"className":44917},[1092],[57,44919,44921,44932],{"className":44920,"style":18098},[1096],[57,44922,44923,44926],{"style":18101},[57,44924],{"className":44925,"style":1493},[1104],[57,44927,44929],{"className":44928,"style":44751},[1109,1110,1111,1112],[57,44930,1288],{"className":44931,"style":44751},[1075,1148,1112],[57,44933,44934,44937],{"style":17216},[57,44935],{"className":44936,"style":1493},[1104],[57,44938,44940],{"className":44939,"style":44751},[1109,1110,1111,1112],[57,44941,1733],{"className":44942,"style":44751},[1075,1148,1112],[57,44944,1183],{"className":44945},[1182],[57,44947,44949],{"className":44948},[1092],[57,44950,44952],{"className":44951,"style":18131},[1096],[57,44953],{},[57,44955,1768],{"className":44956,"style":44751},[1269],[57,44958,44960,44963,45349],{"className":44959,"style":44751},[1075],[57,44961],{"className":44962},[1079,1080],[57,44964,44966],{"className":44965},[1035],[57,44967,44969,45341],{"className":44968},[1087,1088],[57,44970,44972,45338],{"className":44971},[1092],[57,44973,44975,45167,45176],{"className":44974,"style":37982},[1096],[57,44976,44977,44980],{"style":16608},[57,44978],{"className":44979,"style":1105},[1104],[57,44981,44983,45056,45059,45110,45113,45164],{"className":44982,"style":44751},[1075],[57,44984,44986,44990],{"className":44985,"style":44751},[1075],[57,44987,6333],{"className":44988,"style":44989},[1075,1148],"margin-right:0.13889em;color:red;",[57,44991,44993],{"className":44992},[1152],[57,44994,44996,45048],{"className":44995},[1087,1088],[57,44997,44999,45045],{"className":44998},[1092],[57,45000,45002],{"className":45001,"style":2208},[1096],[57,45003,45004,45007],{"style":8131},[57,45005],{"className":45006,"style":1493},[1104],[57,45008,45010],{"className":45009,"style":44751},[1109,1110,1111,1112],[57,45011,45013],{"className":45012,"style":44751},[1075,1112],[57,45014,45016,45019],{"className":45015,"style":44751},[1075,1112],[57,45017,14745],{"className":45018,"style":44784},[1075,1148,1112],[57,45020,45022],{"className":45021},[1152],[57,45023,45025],{"className":45024},[1087],[57,45026,45028],{"className":45027},[1092],[57,45029,45031],{"className":45030,"style":19935},[1096],[57,45032,45033,45036],{"style":19938},[57,45034],{"className":45035,"style":1169},[1104],[57,45037,45039],{"className":45038,"style":44751},[1109,1173,1174,1112],[57,45040,45042],{"className":45041,"style":44751},[1075,1112],[57,45043,19775],{"className":45044,"style":44751},[1075,1112],[57,45046,1183],{"className":45047},[1182],[57,45049,45051],{"className":45050},[1092],[57,45052,45054],{"className":45053,"style":2230},[1096],[57,45055],{},[57,45057,1754],{"className":45058,"style":44751},[1079],[57,45060,45062,45065],{"className":45061,"style":44751},[1075],[57,45063,18],{"className":45064,"style":44751},[1075,1148],[57,45066,45068],{"className":45067},[1152],[57,45069,45071,45102],{"className":45070},[1087,1088],[57,45072,45074,45099],{"className":45073},[1092],[57,45075,45077,45088],{"className":45076,"style":38084},[1096],[57,45078,45079,45082],{"style":18101},[57,45080],{"className":45081,"style":1493},[1104],[57,45083,45085],{"className":45084,"style":44751},[1109,1110,1111,1112],[57,45086,1288],{"className":45087,"style":44751},[1075,1148,1112],[57,45089,45090,45093],{"style":17319},[57,45091],{"className":45092,"style":1493},[1104],[57,45094,45096],{"className":45095,"style":44751},[1109,1110,1111,1112],[57,45097,1733],{"className":45098,"style":44751},[1075,1148,1112],[57,45100,1183],{"className":45101},[1182],[57,45103,45105],{"className":45104},[1092],[57,45106,45108],{"className":45107,"style":18131},[1096],[57,45109],{},[57,45111,13675],{"className":45112,"style":44751},[1075],[57,45114,45116,45119],{"className":45115,"style":44751},[1075],[57,45117,13535],{"className":45118,"style":44751},[1075,1148],[57,45120,45122],{"className":45121},[1152],[57,45123,45125,45156],{"className":45124},[1087,1088],[57,45126,45128,45153],{"className":45127},[1092],[57,45129,45131,45142],{"className":45130,"style":38084},[1096],[57,45132,45133,45136],{"style":18101},[57,45134],{"className":45135,"style":1493},[1104],[57,45137,45139],{"className":45138,"style":44751},[1109,1110,1111,1112],[57,45140,1288],{"className":45141,"style":44751},[1075,1148,1112],[57,45143,45144,45147],{"style":17319},[57,45145],{"className":45146,"style":1493},[1104],[57,45148,45150],{"className":45149,"style":44751},[1109,1110,1111,1112],[57,45151,1733],{"className":45152,"style":44751},[1075,1148,1112],[57,45154,1183],{"className":45155},[1182],[57,45157,45159],{"className":45158},[1092],[57,45160,45162],{"className":45161,"style":18131},[1096],[57,45163],{},[57,45165,1768],{"className":45166,"style":44751},[1269],[57,45168,45169,45172],{"style":1230},[57,45170],{"className":45171,"style":1105},[1104],[57,45173],{"className":45174,"style":45175},[1237],"color:red;border-bottom-width:0.04em;",[57,45177,45178,45181],{"style":16674},[57,45179],{"className":45180,"style":1105},[1104],[57,45182,45184,45227,45230,45281,45284,45335],{"className":45183,"style":44751},[1075],[57,45185,45187,45190],{"className":45186,"style":44751},[1075],[57,45188,6333],{"className":45189,"style":44989},[1075,1148],[57,45191,45193],{"className":45192},[1152],[57,45194,45196,45219],{"className":45195},[1087,1088],[57,45197,45199,45216],{"className":45198},[1092],[57,45200,45202],{"className":45201,"style":2208},[1096],[57,45203,45204,45207],{"style":8131},[57,45205],{"className":45206,"style":1493},[1104],[57,45208,45210],{"className":45209,"style":44751},[1109,1110,1111,1112],[57,45211,45213],{"className":45212,"style":44751},[1075,1112],[57,45214,14745],{"className":45215,"style":44784},[1075,1148,1112],[57,45217,1183],{"className":45218},[1182],[57,45220,45222],{"className":45221},[1092],[57,45223,45225],{"className":45224,"style":2230},[1096],[57,45226],{},[57,45228,1754],{"className":45229,"style":44751},[1079],[57,45231,45233,45236],{"className":45232,"style":44751},[1075],[57,45234,18],{"className":45235,"style":44751},[1075,1148],[57,45237,45239],{"className":45238},[1152],[57,45240,45242,45273],{"className":45241},[1087,1088],[57,45243,45245,45270],{"className":45244},[1092],[57,45246,45248,45259],{"className":45247,"style":19640},[1096],[57,45249,45250,45253],{"style":18101},[57,45251],{"className":45252,"style":1493},[1104],[57,45254,45256],{"className":45255,"style":44751},[1109,1110,1111,1112],[57,45257,1288],{"className":45258,"style":44751},[1075,1148,1112],[57,45260,45261,45264],{"style":1489},[57,45262],{"className":45263,"style":1493},[1104],[57,45265,45267],{"className":45266,"style":44751},[1109,1110,1111,1112],[57,45268,1733],{"className":45269,"style":44751},[1075,1148,1112],[57,45271,1183],{"className":45272},[1182],[57,45274,45276],{"className":45275},[1092],[57,45277,45279],{"className":45278,"style":18131},[1096],[57,45280],{},[57,45282,13675],{"className":45283,"style":44751},[1075],[57,45285,45287,45290],{"className":45286,"style":44751},[1075],[57,45288,13535],{"className":45289,"style":44751},[1075,1148],[57,45291,45293],{"className":45292},[1152],[57,45294,45296,45327],{"className":45295},[1087,1088],[57,45297,45299,45324],{"className":45298},[1092],[57,45300,45302,45313],{"className":45301,"style":19640},[1096],[57,45303,45304,45307],{"style":18101},[57,45305],{"className":45306,"style":1493},[1104],[57,45308,45310],{"className":45309,"style":44751},[1109,1110,1111,1112],[57,45311,1288],{"className":45312,"style":44751},[1075,1148,1112],[57,45314,45315,45318],{"style":1489},[57,45316],{"className":45317,"style":1493},[1104],[57,45319,45321],{"className":45320,"style":44751},[1109,1110,1111,1112],[57,45322,1733],{"className":45323,"style":44751},[1075,1148,1112],[57,45325,1183],{"className":45326},[1182],[57,45328,45330],{"className":45329},[1092],[57,45331,45333],{"className":45332,"style":18131},[1096],[57,45334],{},[57,45336,1768],{"className":45337,"style":44751},[1269],[57,45339,1183],{"className":45340},[1182],[57,45342,45344],{"className":45343},[1092],[57,45345,45347],{"className":45346,"style":16739},[1096],[57,45348],{},[57,45350],{"className":45351},[1269,1080],[57,45353,1759],{"className":45354},[1849],[57,45356],{"className":45357,"style":1853},[1452],[57,45359],{"className":45360,"style":1853},[1452],[57,45362,45365],{"className":45363,"style":45364},[1075],"color:skyblue;",[57,45366,45368],{"className":45367,"style":45364},[1075,1838],"clip",[57,45370],{"className":45371,"style":1853},[1452],[57,45373,45375,45384,45780,45783,45787,45790,45793,45797,45800,45803,45806,45809,45812,45815,45818,45821,45824,45827,45830],{"className":45374,"style":45364},[1898],[57,45376,45379],{"className":45377,"style":45378},[1079,1902],"color:skyblue;top:0em;",[57,45380,45382],{"className":45381,"style":45364},[1907,1111],[57,45383,1754],{"style":45364},[57,45385,45387,45390,45777],{"className":45386,"style":45364},[1075],[57,45388],{"className":45389},[1079,1080],[57,45391,45393],{"className":45392},[1035],[57,45394,45396,45769],{"className":45395},[1087,1088],[57,45397,45399,45766],{"className":45398},[1092],[57,45400,45402,45595,45604],{"className":45401,"style":37982},[1096],[57,45403,45404,45407],{"style":16608},[57,45405],{"className":45406,"style":1105},[1104],[57,45408,45410,45484,45487,45538,45541,45592],{"className":45409,"style":45364},[1075],[57,45411,45413,45417],{"className":45412,"style":45364},[1075],[57,45414,6333],{"className":45415,"style":45416},[1075,1148],"margin-right:0.13889em;color:skyblue;",[57,45418,45420],{"className":45419},[1152],[57,45421,45423,45476],{"className":45422},[1087,1088],[57,45424,45426,45473],{"className":45425},[1092],[57,45427,45429],{"className":45428,"style":2208},[1096],[57,45430,45431,45434],{"style":8131},[57,45432],{"className":45433,"style":1493},[1104],[57,45435,45437],{"className":45436,"style":45364},[1109,1110,1111,1112],[57,45438,45440],{"className":45439,"style":45364},[1075,1112],[57,45441,45443,45447],{"className":45442,"style":45364},[1075,1112],[57,45444,14745],{"className":45445,"style":45446},[1075,1148,1112],"margin-right:0.02778em;color:skyblue;",[57,45448,45450],{"className":45449},[1152],[57,45451,45453],{"className":45452},[1087],[57,45454,45456],{"className":45455},[1092],[57,45457,45459],{"className":45458,"style":19935},[1096],[57,45460,45461,45464],{"style":19938},[57,45462],{"className":45463,"style":1169},[1104],[57,45465,45467],{"className":45466,"style":45364},[1109,1173,1174,1112],[57,45468,45470],{"className":45469,"style":45364},[1075,1112],[57,45471,19775],{"className":45472,"style":45364},[1075,1112],[57,45474,1183],{"className":45475},[1182],[57,45477,45479],{"className":45478},[1092],[57,45480,45482],{"className":45481,"style":2230},[1096],[57,45483],{},[57,45485,1754],{"className":45486,"style":45364},[1079],[57,45488,45490,45493],{"className":45489,"style":45364},[1075],[57,45491,18],{"className":45492,"style":45364},[1075,1148],[57,45494,45496],{"className":45495},[1152],[57,45497,45499,45530],{"className":45498},[1087,1088],[57,45500,45502,45527],{"className":45501},[1092],[57,45503,45505,45516],{"className":45504,"style":38084},[1096],[57,45506,45507,45510],{"style":18101},[57,45508],{"className":45509,"style":1493},[1104],[57,45511,45513],{"className":45512,"style":45364},[1109,1110,1111,1112],[57,45514,1288],{"className":45515,"style":45364},[1075,1148,1112],[57,45517,45518,45521],{"style":17319},[57,45519],{"className":45520,"style":1493},[1104],[57,45522,45524],{"className":45523,"style":45364},[1109,1110,1111,1112],[57,45525,1733],{"className":45526,"style":45364},[1075,1148,1112],[57,45528,1183],{"className":45529},[1182],[57,45531,45533],{"className":45532},[1092],[57,45534,45536],{"className":45535,"style":18131},[1096],[57,45537],{},[57,45539,13675],{"className":45540,"style":45364},[1075],[57,45542,45544,45547],{"className":45543,"style":45364},[1075],[57,45545,13535],{"className":45546,"style":45364},[1075,1148],[57,45548,45550],{"className":45549},[1152],[57,45551,45553,45584],{"className":45552},[1087,1088],[57,45554,45556,45581],{"className":45555},[1092],[57,45557,45559,45570],{"className":45558,"style":38084},[1096],[57,45560,45561,45564],{"style":18101},[57,45562],{"className":45563,"style":1493},[1104],[57,45565,45567],{"className":45566,"style":45364},[1109,1110,1111,1112],[57,45568,1288],{"className":45569,"style":45364},[1075,1148,1112],[57,45571,45572,45575],{"style":17319},[57,45573],{"className":45574,"style":1493},[1104],[57,45576,45578],{"className":45577,"style":45364},[1109,1110,1111,1112],[57,45579,1733],{"className":45580,"style":45364},[1075,1148,1112],[57,45582,1183],{"className":45583},[1182],[57,45585,45587],{"className":45586},[1092],[57,45588,45590],{"className":45589,"style":18131},[1096],[57,45591],{},[57,45593,1768],{"className":45594,"style":45364},[1269],[57,45596,45597,45600],{"style":1230},[57,45598],{"className":45599,"style":1105},[1104],[57,45601],{"className":45602,"style":45603},[1237],"color:skyblue;border-bottom-width:0.04em;",[57,45605,45606,45609],{"style":16674},[57,45607],{"className":45608,"style":1105},[1104],[57,45610,45612,45655,45658,45709,45712,45763],{"className":45611,"style":45364},[1075],[57,45613,45615,45618],{"className":45614,"style":45364},[1075],[57,45616,6333],{"className":45617,"style":45416},[1075,1148],[57,45619,45621],{"className":45620},[1152],[57,45622,45624,45647],{"className":45623},[1087,1088],[57,45625,45627,45644],{"className":45626},[1092],[57,45628,45630],{"className":45629,"style":2208},[1096],[57,45631,45632,45635],{"style":8131},[57,45633],{"className":45634,"style":1493},[1104],[57,45636,45638],{"className":45637,"style":45364},[1109,1110,1111,1112],[57,45639,45641],{"className":45640,"style":45364},[1075,1112],[57,45642,14745],{"className":45643,"style":45446},[1075,1148,1112],[57,45645,1183],{"className":45646},[1182],[57,45648,45650],{"className":45649},[1092],[57,45651,45653],{"className":45652,"style":2230},[1096],[57,45654],{},[57,45656,1754],{"className":45657,"style":45364},[1079],[57,45659,45661,45664],{"className":45660,"style":45364},[1075],[57,45662,18],{"className":45663,"style":45364},[1075,1148],[57,45665,45667],{"className":45666},[1152],[57,45668,45670,45701],{"className":45669},[1087,1088],[57,45671,45673,45698],{"className":45672},[1092],[57,45674,45676,45687],{"className":45675,"style":19640},[1096],[57,45677,45678,45681],{"style":18101},[57,45679],{"className":45680,"style":1493},[1104],[57,45682,45684],{"className":45683,"style":45364},[1109,1110,1111,1112],[57,45685,1288],{"className":45686,"style":45364},[1075,1148,1112],[57,45688,45689,45692],{"style":1489},[57,45690],{"className":45691,"style":1493},[1104],[57,45693,45695],{"className":45694,"style":45364},[1109,1110,1111,1112],[57,45696,1733],{"className":45697,"style":45364},[1075,1148,1112],[57,45699,1183],{"className":45700},[1182],[57,45702,45704],{"className":45703},[1092],[57,45705,45707],{"className":45706,"style":18131},[1096],[57,45708],{},[57,45710,13675],{"className":45711,"style":45364},[1075],[57,45713,45715,45718],{"className":45714,"style":45364},[1075],[57,45716,13535],{"className":45717,"style":45364},[1075,1148],[57,45719,45721],{"className":45720},[1152],[57,45722,45724,45755],{"className":45723},[1087,1088],[57,45725,45727,45752],{"className":45726},[1092],[57,45728,45730,45741],{"className":45729,"style":19640},[1096],[57,45731,45732,45735],{"style":18101},[57,45733],{"className":45734,"style":1493},[1104],[57,45736,45738],{"className":45737,"style":45364},[1109,1110,1111,1112],[57,45739,1288],{"className":45740,"style":45364},[1075,1148,1112],[57,45742,45743,45746],{"style":1489},[57,45744],{"className":45745,"style":1493},[1104],[57,45747,45749],{"className":45748,"style":45364},[1109,1110,1111,1112],[57,45750,1733],{"className":45751,"style":45364},[1075,1148,1112],[57,45753,1183],{"className":45754},[1182],[57,45756,45758],{"className":45757},[1092],[57,45759,45761],{"className":45760,"style":18131},[1096],[57,45762],{},[57,45764,1768],{"className":45765,"style":45364},[1269],[57,45767,1183],{"className":45768},[1182],[57,45770,45772],{"className":45771},[1092],[57,45773,45775],{"className":45774,"style":16739},[1096],[57,45776],{},[57,45778],{"className":45779},[1269,1080],[57,45781,1759],{"className":45782,"style":45364},[1849],[57,45784],{"className":45785,"style":45786},[1452],"color:skyblue;margin-right:0.1667em;",[57,45788],{"className":45789,"style":45786},[1452],[57,45791,574],{"className":45792,"style":45364},[1075],[57,45794],{"className":45795,"style":45796},[1452],"color:skyblue;margin-right:0.2222em;",[57,45798,19155],{"className":45799,"style":45364},[1506],[57,45801],{"className":45802,"style":45796},[1452],[57,45804,43764],{"className":45805,"style":45364},[1075,1148],[57,45807,1759],{"className":45808,"style":45364},[1849],[57,45810],{"className":45811,"style":45786},[1452],[57,45813],{"className":45814,"style":45786},[1452],[57,45816,574],{"className":45817,"style":45364},[1075],[57,45819],{"className":45820,"style":45796},[1452],[57,45822,6487],{"className":45823,"style":45364},[1506],[57,45825],{"className":45826,"style":45796},[1452],[57,45828,43764],{"className":45829,"style":45364},[1075,1148],[57,45831,45833],{"className":45832,"style":45378},[1269,1902],[57,45834,45836],{"className":45835,"style":45364},[1907,1111],[57,45837,1768],{"style":45364},[57,45839],{"className":45840,"style":1853},[1452],[57,45842,45844,45847],{"className":45843,"style":45364},[1075],[57,45845,1730],{"className":45846,"style":45364},[1075,1148],[57,45848,45850],{"className":45849},[1152],[57,45851,45853,45926],{"className":45852},[1087,1088],[57,45854,45856,45923],{"className":45855},[1092],[57,45857,45859,45902],{"className":45858,"style":25824},[1096],[57,45860,45861,45864],{"style":18101},[57,45862],{"className":45863,"style":1493},[1104],[57,45865,45867],{"className":45866,"style":45364},[1109,1110,1111,1112],[57,45868,45870],{"className":45869,"style":45364},[1075,1112],[57,45871,45873,45876],{"className":45872,"style":45364},[1075,1112],[57,45874,14745],{"className":45875,"style":45446},[1075,1148,1112],[57,45877,45879],{"className":45878},[1152],[57,45880,45882],{"className":45881},[1087],[57,45883,45885],{"className":45884},[1092],[57,45886,45888],{"className":45887,"style":19935},[1096],[57,45889,45890,45893],{"style":19938},[57,45891],{"className":45892,"style":1169},[1104],[57,45894,45896],{"className":45895,"style":45364},[1109,1173,1174,1112],[57,45897,45899],{"className":45898,"style":45364},[1075,1112],[57,45900,19775],{"className":45901,"style":45364},[1075,1112],[57,45903,45904,45907],{"style":17216},[57,45905],{"className":45906,"style":1493},[1104],[57,45908,45910],{"className":45909,"style":45364},[1109,1110,1111,1112],[57,45911,45913,45916,45919],{"className":45912,"style":45364},[1075,1112],[57,45914,28613],{"className":45915,"style":45364},[1075,1148,1112],[57,45917,1730],{"className":45918,"style":45364},[1075,1148,1112],[57,45920,6336],{"className":45921,"style":45922},[1075,1148,1112],"margin-right:0.05764em;color:skyblue;",[57,45924,1183],{"className":45925},[1182],[57,45927,45929],{"className":45928},[1092],[57,45930,45932],{"className":45931,"style":18131},[1096],[57,45933],{},[57,45935,1754],{"className":45936,"style":45364},[1079],[57,45938,45940,45943],{"className":45939,"style":45364},[1075],[57,45941,13535],{"className":45942,"style":45364},[1075,1148],[57,45944,45946],{"className":45945},[1152],[57,45947,45949,45980],{"className":45948},[1087,1088],[57,45950,45952,45977],{"className":45951},[1092],[57,45953,45955,45966],{"className":45954,"style":18098},[1096],[57,45956,45957,45960],{"style":18101},[57,45958],{"className":45959,"style":1493},[1104],[57,45961,45963],{"className":45962,"style":45364},[1109,1110,1111,1112],[57,45964,1288],{"className":45965,"style":45364},[1075,1148,1112],[57,45967,45968,45971],{"style":17216},[57,45969],{"className":45970,"style":1493},[1104],[57,45972,45974],{"className":45973,"style":45364},[1109,1110,1111,1112],[57,45975,1733],{"className":45976,"style":45364},[1075,1148,1112],[57,45978,1183],{"className":45979},[1182],[57,45981,45983],{"className":45982},[1092],[57,45984,45986],{"className":45985,"style":18131},[1096],[57,45987],{},[57,45989,1759],{"className":45990,"style":45364},[1849],[57,45992],{"className":45993,"style":1853},[1452],[57,45995,45997,46000],{"className":45996,"style":45364},[1075],[57,45998,18],{"className":45999,"style":45364},[1075,1148],[57,46001,46003],{"className":46002},[1152],[57,46004,46006,46037],{"className":46005},[1087,1088],[57,46007,46009,46034],{"className":46008},[1092],[57,46010,46012,46023],{"className":46011,"style":18098},[1096],[57,46013,46014,46017],{"style":18101},[57,46015],{"className":46016,"style":1493},[1104],[57,46018,46020],{"className":46019,"style":45364},[1109,1110,1111,1112],[57,46021,1288],{"className":46022,"style":45364},[1075,1148,1112],[57,46024,46025,46028],{"style":17216},[57,46026],{"className":46027,"style":1493},[1104],[57,46029,46031],{"className":46030,"style":45364},[1109,1110,1111,1112],[57,46032,1733],{"className":46033,"style":45364},[1075,1148,1112],[57,46035,1183],{"className":46036},[1182],[57,46038,46040],{"className":46039},[1092],[57,46041,46043],{"className":46042,"style":18131},[1096],[57,46044],{},[57,46046,1768],{"className":46047,"style":45364},[1269],[57,46049,46051],{"className":46050,"style":1903},[1269,1902],[57,46052,1768],{"className":46053},[1907,1111],[14,46055,46056],{},"PPO-Clip 由于实现简单、效果稳定，是当前应用最广泛的版本，也是 RLHF/GRPO 等大模型对齐算法的基础。",[10,46058,46059],{"id":46059},"参考资料",[14,46061,46062],{},[18,46063,46066],{"href":46064,"rel":46065},"https://www.bilibili.com/video/BV1iz421h7gb/",[22],"零基础学习强化学习算法：ppo",{"title":52,"searchDepth":66,"depth":66,"links":46068},[46069,46070,46071,46072,46075,46079],{"id":13399,"depth":66,"text":13399},{"id":13405,"depth":66,"text":13405},{"id":14723,"depth":66,"text":14723},{"id":20252,"depth":66,"text":20253,"children":46073},[46074],{"id":28481,"depth":72,"text":28482},{"id":33037,"depth":66,"text":33038,"children":46076},[46077,46078],{"id":33235,"depth":72,"text":33235},{"id":33228,"depth":72,"text":35243},{"id":46059,"depth":66,"text":46059},"记一下学习 PPO中学习到的东西",{"date":46082,"image":46083,"alt":46084,"tags":46085,"published":963},"8 Mar 2026","/blogs-img/blog2.jpg","PPO学习笔记",[46086,961],"Reinforcement learning","/blogs/ppo",{"title":13394,"description":46080},"blogs/2. PPO学习笔记","pCK9BRxyBBqY_YKiJBmT8uZH6lLudARB31wOAuwxMuM",{"id":46092,"title":46093,"body":46094,"description":46093,"extension":955,"meta":47042,"navigation":963,"ogImage":47044,"path":47049,"seo":47050,"stem":47051,"__hash__":47052},"content/blogs/3. 记录一下我的 zsh 配置.md","记录一下我的 zsh 配置",{"type":7,"value":46095,"toc":47019},[46096,46098,46101,46105,46108,46111,46114,46117,46120,46163,46170,46174,46177,46180,46228,46242,46259,46269,46272,46275,46278,46281,46287,46290,46310,46313,46316,46318,46337,46354,46356,46362,46367,46385,46388,46401,46424,46428,46431,46434,46439,46442,46446,46533,46537,46738,46742,46910,46913,46916,46923,46931,46934,46937,46940,46943,46946,46995,47001,47004,47011,47016],[10,46097,976],{"id":976},[14,46099,46100],{},"最近想写点东西记录一下自己折腾这些工具的经历，之前一直没想到具体写什么。今年实验室服务器迁移，刚好需要重新配一遍 zsh，干脆顺手把整套方案写下来，省得下次换机器再翻一堆旧博客。",[10,46102,46104],{"id":46103},"什么是-zsh","什么是 Zsh",[14,46106,46107],{},"Zsh 也是一种 shell，可以看成是一个更好用的 Bash——完全兼容 Bash 的语法，但补全、插件、主题方面强了不止一个量级。目前 macOS 的默认 shell 已经是 Zsh 了。配上插件之后，用起来在 Linux 下能省一大堆敲键盘的功夫。",[14,46109,46110],{},"Zsh 通过大部分主流 Linux 发行版的包管理器都能直接装。要是在没 sudo 权限的服务器上，也可以用 conda 装一份用户级别的，绕开权限问题。",[10,46112,46113],{"id":46113},"oh-my-zsh",[14,46115,46116],{},"Zsh 固然好用，但初始时只有白板没有任何插件，离理想的使用体验还差点意思，这里就轮到 oh-my-zsh 登场了。",[14,46118,46119],{},"oh-my-zsh 是社区为 zsh 收集的一个第三方插件和主题合集，国内可以通过南京大学镜像站快速安装：",[47,46121,46125],{"className":46122,"code":46123,"language":46124,"meta":52,"style":52},"language-bash shiki shiki-themes github-light dracula","git clone https://mirror.nju.edu.cn/git/ohmyzsh.git\ncd ohmyzsh/tools\nREMOTE=https://mirror.nju.edu.cn/git/ohmyzsh.git sh install.sh\n","bash",[54,46126,46127,46138,46146],{"__ignoreMap":52},[57,46128,46129,46132,46135],{"class":59,"line":60},[57,46130,46131],{"class":3153},"git",[57,46133,46134],{"class":293}," clone",[57,46136,46137],{"class":293}," https://mirror.nju.edu.cn/git/ohmyzsh.git\n",[57,46139,46140,46143],{"class":59,"line":66},[57,46141,46142],{"class":10193},"cd",[57,46144,46145],{"class":293}," ohmyzsh/tools\n",[57,46147,46148,46152,46154,46157,46160],{"class":59,"line":72},[57,46149,46151],{"class":46150},"scEb-","REMOTE",[57,46153,281],{"class":280},[57,46155,46156],{"class":293},"https://mirror.nju.edu.cn/git/ohmyzsh.git",[57,46158,46159],{"class":3153}," sh",[57,46161,46162],{"class":293}," install.sh\n",[14,46164,46165,46166,46169],{},"通过这三条指令可以在境内快速装好 oh-my-zsh，安装过程中也会问你要不要把 zsh 设为默认 shell。要是当时手抖没设置，事后用 ",[54,46167,46168],{},"chsh -s $(which zsh)"," 也能补救。",[10,46171,46173],{"id":46172},"powerlevel10k","Powerlevel10k",[14,46175,46176],{},"oh-my-zsh 自带的主题我感觉还是不够美观、不够能折腾，于是这里再引入一个 Powerlevel10k 主题。在众多主题中，Powerlevel10k 因为美观和高度可定制脱颖而出，截至 2026 年 4 月 28 日，它在 GitHub 上已累计收获 54k 个 star。",[14,46178,46179],{},"国内安装可以用 gitee 的镜像：",[47,46181,46183],{"className":46122,"code":46182,"language":46124,"meta":52,"style":52},"git clone --depth=1 https://gitee.com/romkatv/powerlevel10k.git \"${ZSH_CUSTOM:-$HOME/.oh-my-zsh/custom}/themes/powerlevel10k\"\n",[54,46184,46185],{"__ignoreMap":52},[57,46186,46187,46189,46191,46194,46197,46199,46202,46205,46208,46211,46213,46215,46217,46219,46222,46225],{"class":59,"line":60},[57,46188,46131],{"class":3153},[57,46190,46134],{"class":293},[57,46192,46193],{"class":273}," --depth=1",[57,46195,46196],{"class":293}," https://gitee.com/romkatv/powerlevel10k.git",[57,46198,11986],{"class":289},[57,46200,46201],{"class":293},"${",[57,46203,46204],{"class":46150},"ZSH_CUSTOM",[57,46206,46207],{"class":280},":-",[57,46209,46210],{"class":46150},"$HOME",[57,46212,3549],{"class":280},[57,46214,1818],{"class":293},[57,46216,46113],{"class":46150},[57,46218,3549],{"class":280},[57,46220,46221],{"class":46150},"custom",[57,46223,46224],{"class":293},"}/themes/powerlevel10k",[57,46226,46227],{"class":289},"\"\n",[14,46229,46230,46231,46234,46235,46238,46239,727],{},"之后修改 ",[54,46232,46233],{},"~/.zshrc","，把 ",[54,46236,46237],{},"ZSH_THEME"," 改成 ",[54,46240,46241],{},"powerlevel10k/powerlevel10k",[47,46243,46245],{"className":46122,"code":46244,"language":46124,"meta":52,"style":52},"ZSH_THEME=\"powerlevel10k/powerlevel10k\"\n",[54,46246,46247],{"__ignoreMap":52},[57,46248,46249,46251,46253,46255,46257],{"class":59,"line":60},[57,46250,46237],{"class":46150},[57,46252,281],{"class":280},[57,46254,297],{"class":289},[57,46256,46241],{"class":293},[57,46258,46227],{"class":289},[14,46260,46261,46264,46265,46268],{},[54,46262,46263],{},"source ~/.zshrc"," 或者重启终端后，会自动进入 Powerlevel10k 的配置向导，跟着选自己喜欢的风格就行。后面想再调，随时 ",[54,46266,46267],{},"p10k configure"," 就能重来一遍。",[14,46270,46271],{},"为了能完整显示主题里的图标，建议装一款 Nerd Font 字体。我个人用的是 0xProto Nerd Font，大家也可以挑别人推荐的字体，比如 FiraCode、JetBrainsMono、MesloLGS 之类的都行。装完之后记得在终端模拟器（iTerm2 / Windows Terminal / Alacritty / Kitty 等）的设置里把字体切换过去。",[10,46273,46274],{"id":46274},"插件设置",[14,46276,46277],{},"主题搞定了，下一步就是装上几个真正能帮我们提高生产力的插件。",[248,46279,46280],{"id":46280},"zsh-autosuggestions",[14,46282,46283,46284,46286],{},"非常有用的一个插件，根据历史记录和补全信息在你输入时实时给出灰色提示，按 ",[54,46285,31383],{}," 就能接受补全，能省下不少重复敲键盘的功夫。",[14,46288,46289],{},"安装指令：",[47,46291,46293],{"className":46122,"code":46292,"language":46124,"meta":52,"style":52},"git clone https://github.com/zsh-users/zsh-autosuggestions $ZSH_CUSTOM/plugins/zsh-autosuggestions\n",[54,46294,46295],{"__ignoreMap":52},[57,46296,46297,46299,46301,46304,46307],{"class":59,"line":60},[57,46298,46131],{"class":3153},[57,46300,46134],{"class":293},[57,46302,46303],{"class":293}," https://github.com/zsh-users/zsh-autosuggestions",[57,46305,46306],{"class":46150}," $ZSH_CUSTOM",[57,46308,46309],{"class":293},"/plugins/zsh-autosuggestions\n",[248,46311,46312],{"id":46312},"zsh-syntax-highlighting",[14,46314,46315],{},"提供指令和文件路径的高亮显示，方便你在按下回车之前就检查指令和路径有没有写错，相当于给终端加了层\"实时校对\"。",[14,46317,46289],{},[47,46319,46321],{"className":46122,"code":46320,"language":46124,"meta":52,"style":52},"git clone https://github.com/zsh-users/zsh-syntax-highlighting $ZSH_CUSTOM/plugins/zsh-syntax-highlighting\n",[54,46322,46323],{"__ignoreMap":52},[57,46324,46325,46327,46329,46332,46334],{"class":59,"line":60},[57,46326,46131],{"class":3153},[57,46328,46134],{"class":293},[57,46330,46331],{"class":293}," https://github.com/zsh-users/zsh-syntax-highlighting",[57,46333,46306],{"class":46150},[57,46335,46336],{"class":293},"/plugins/zsh-syntax-highlighting\n",[46338,46339,46340],"blockquote",{},[14,46341,46342,46343,46345,46346,46349,46350,46353],{},"小坑提醒：",[54,46344,46312],{}," 必须放在 ",[54,46347,46348],{},"plugins=(...)"," 列表的 ",[11865,46351,46352],{},"最后一项","，否则可能会和其他插件打架。",[248,46355,46131],{"id":46131},[14,46357,46358,46359,246],{},"oh-my-zsh 自带的插件，把大量常用 git 指令简化成了别名，具体怎么用可以参考 ",[54,46360,46361],{},"$ZSH/plugins/git/git.plugin.zsh",[14,46363,46364],{},[1000,46365],{"alt":46131,"src":46366},"zsh/git.png",[14,46368,46369,46370,46373,46374,46377,46378,46381,46382,222],{},"如图所示，alias 表示别名，等号左边是缩写指令，右边是原指令，比如用 ",[54,46371,46372],{},"gst"," 就能代替 ",[54,46375,46376],{},"git status","，用 ",[54,46379,46380],{},"gco"," 代替 ",[54,46383,46384],{},"git checkout",[248,46386,46387],{"id":46387},"启用插件",[14,46389,46390,46391,46393,46394,46397,46398,46400],{},"最后别忘了在 ",[54,46392,46233],{}," 里把上面三个插件加到 ",[54,46395,46396],{},"plugins"," 列表里，然后 ",[54,46399,46263],{}," 让配置生效：",[47,46402,46404],{"className":46122,"code":46403,"language":46124,"meta":52,"style":52},"plugins=(git zsh-autosuggestions zsh-syntax-highlighting)\n",[54,46405,46406],{"__ignoreMap":52},[57,46407,46408,46410,46412,46414,46416,46419,46422],{"class":59,"line":60},[57,46409,46396],{"class":46150},[57,46411,281],{"class":280},[57,46413,1754],{"class":269},[57,46415,46131],{"class":293},[57,46417,46418],{"class":293}," zsh-autosuggestions",[57,46420,46421],{"class":293}," zsh-syntax-highlighting",[57,46423,3217],{"class":269},[10,46425,46427],{"id":46426},"kitty","Kitty",[14,46429,46430],{},"kitty 是一个基于 OpenGL 的可编程终端模拟器，适用于 Linux 与 macOS，具有真彩色，连字支持，键盘输入和图像渲染的协议扩展。它还提供了类似于 GNU Screen 或 tmux 的平铺功能。",[14,46432,46433],{},"kitty 在 ~/.config/kitty/kitty.conf 中存储配置，可以调整字体，颜色，光标，透明度和回滚行为。",[14,46435,46436],{},[1000,46437],{"alt":46426,"src":46438},"zsh/kitty.png",[14,46440,46441],{},"我的项目结构如图所示，接下来会放出我的配置文件：",[248,46443,46445],{"id":46444},"kittyconf","kitty.conf",[47,46447,46451],{"className":46448,"code":46449,"language":46450,"meta":52,"style":52},"language-conf shiki shiki-themes github-light dracula","include themes/matugen.conf\nwindow_padding_width 5\nhide_window_decorations no\nbackground_opacity 0.8\nfont_family JetBrains Maple Mono \nfont_size 13.5\nremember_window_size no\nconfirm_os_window_close 0\nshell /bin/zsh --login\ncursor_trail 1 \ncursor_shape block\nshell_integration no-cursor\n# BEGIN_KITTY_THEME\n# Matugen\ninclude current-theme.conf\n# END_KITTY_THEME\n","conf",[54,46452,46453,46458,46463,46468,46473,46478,46483,46488,46493,46498,46503,46508,46513,46518,46523,46528],{"__ignoreMap":52},[57,46454,46455],{"class":59,"line":60},[57,46456,46457],{},"include themes/matugen.conf\n",[57,46459,46460],{"class":59,"line":66},[57,46461,46462],{},"window_padding_width 5\n",[57,46464,46465],{"class":59,"line":72},[57,46466,46467],{},"hide_window_decorations no\n",[57,46469,46470],{"class":59,"line":78},[57,46471,46472],{},"background_opacity 0.8\n",[57,46474,46475],{"class":59,"line":84},[57,46476,46477],{},"font_family JetBrains Maple Mono \n",[57,46479,46480],{"class":59,"line":90},[57,46481,46482],{},"font_size 13.5\n",[57,46484,46485],{"class":59,"line":96},[57,46486,46487],{},"remember_window_size no\n",[57,46489,46490],{"class":59,"line":102},[57,46491,46492],{},"confirm_os_window_close 0\n",[57,46494,46495],{"class":59,"line":108},[57,46496,46497],{},"shell /bin/zsh --login\n",[57,46499,46500],{"class":59,"line":114},[57,46501,46502],{},"cursor_trail 1 \n",[57,46504,46505],{"class":59,"line":120},[57,46506,46507],{},"cursor_shape block\n",[57,46509,46510],{"class":59,"line":126},[57,46511,46512],{},"shell_integration no-cursor\n",[57,46514,46515],{"class":59,"line":132},[57,46516,46517],{},"# BEGIN_KITTY_THEME\n",[57,46519,46520],{"class":59,"line":138},[57,46521,46522],{},"# Matugen\n",[57,46524,46525],{"class":59,"line":144},[57,46526,46527],{},"include current-theme.conf\n",[57,46529,46530],{"class":59,"line":150},[57,46531,46532],{},"# END_KITTY_THEME\n",[248,46534,46536],{"id":46535},"current-themeconf","current-theme.conf",[47,46538,46540],{"className":46448,"code":46539,"language":46450,"meta":52,"style":52},"cursor #e0e2e8\ncursor_text_color #c2c7ce\n\nforeground            #e0e2e8\nbackground            #101418\nselection_foreground  #23323f\nselection_background  #b8c8da\nurl_color             #99ccfa\n\n# black\ncolor8   #8c9198\ncolor0   #4c4c4c\n\n# red\ncolor1   #ac8a8c\ncolor9   #c49ea0\n\n# green\ncolor2   #8aac8b\ncolor10  #9ec49f\n\n# yellow\ncolor3   #aca98a\ncolor11  #c4c19e\n\n# blue\n/* color4  #8f8aac */\ncolor4  #99ccfa\ncolor12 #a39ec4\n\n# magenta\ncolor5   #ac8aac\ncolor13  #c49ec4\n\n# cyan\ncolor6   #8aacab\ncolor14  #9ec3c4\n\n# white\ncolor15   #e7e7e7\ncolor7  #f0f0f0\n",[54,46541,46542,46547,46552,46556,46561,46566,46571,46576,46581,46585,46590,46595,46600,46604,46609,46614,46619,46623,46628,46633,46638,46642,46647,46652,46657,46661,46666,46671,46676,46681,46685,46690,46695,46700,46704,46709,46714,46719,46723,46728,46733],{"__ignoreMap":52},[57,46543,46544],{"class":59,"line":60},[57,46545,46546],{},"cursor #e0e2e8\n",[57,46548,46549],{"class":59,"line":66},[57,46550,46551],{},"cursor_text_color #c2c7ce\n",[57,46553,46554],{"class":59,"line":72},[57,46555,3448],{"emptyLinePlaceholder":963},[57,46557,46558],{"class":59,"line":78},[57,46559,46560],{},"foreground            #e0e2e8\n",[57,46562,46563],{"class":59,"line":84},[57,46564,46565],{},"background            #101418\n",[57,46567,46568],{"class":59,"line":90},[57,46569,46570],{},"selection_foreground  #23323f\n",[57,46572,46573],{"class":59,"line":96},[57,46574,46575],{},"selection_background  #b8c8da\n",[57,46577,46578],{"class":59,"line":102},[57,46579,46580],{},"url_color             #99ccfa\n",[57,46582,46583],{"class":59,"line":108},[57,46584,3448],{"emptyLinePlaceholder":963},[57,46586,46587],{"class":59,"line":114},[57,46588,46589],{},"# black\n",[57,46591,46592],{"class":59,"line":120},[57,46593,46594],{},"color8   #8c9198\n",[57,46596,46597],{"class":59,"line":126},[57,46598,46599],{},"color0   #4c4c4c\n",[57,46601,46602],{"class":59,"line":132},[57,46603,3448],{"emptyLinePlaceholder":963},[57,46605,46606],{"class":59,"line":138},[57,46607,46608],{},"# red\n",[57,46610,46611],{"class":59,"line":144},[57,46612,46613],{},"color1   #ac8a8c\n",[57,46615,46616],{"class":59,"line":150},[57,46617,46618],{},"color9   #c49ea0\n",[57,46620,46621],{"class":59,"line":156},[57,46622,3448],{"emptyLinePlaceholder":963},[57,46624,46625],{"class":59,"line":162},[57,46626,46627],{},"# green\n",[57,46629,46630],{"class":59,"line":168},[57,46631,46632],{},"color2   #8aac8b\n",[57,46634,46635],{"class":59,"line":174},[57,46636,46637],{},"color10  #9ec49f\n",[57,46639,46640],{"class":59,"line":5790},[57,46641,3448],{"emptyLinePlaceholder":963},[57,46643,46644],{"class":59,"line":5811},[57,46645,46646],{},"# yellow\n",[57,46648,46649],{"class":59,"line":5830},[57,46650,46651],{},"color3   #aca98a\n",[57,46653,46654],{"class":59,"line":5849},[57,46655,46656],{},"color11  #c4c19e\n",[57,46658,46659],{"class":59,"line":5854},[57,46660,3448],{"emptyLinePlaceholder":963},[57,46662,46663],{"class":59,"line":5870},[57,46664,46665],{},"# blue\n",[57,46667,46668],{"class":59,"line":5876},[57,46669,46670],{},"/* color4  #8f8aac */\n",[57,46672,46673],{"class":59,"line":5882},[57,46674,46675],{},"color4  #99ccfa\n",[57,46677,46678],{"class":59,"line":5892},[57,46679,46680],{},"color12 #a39ec4\n",[57,46682,46683],{"class":59,"line":5916},[57,46684,3448],{"emptyLinePlaceholder":963},[57,46686,46687],{"class":59,"line":5921},[57,46688,46689],{},"# magenta\n",[57,46691,46692],{"class":59,"line":5927},[57,46693,46694],{},"color5   #ac8aac\n",[57,46696,46697],{"class":59,"line":5941},[57,46698,46699],{},"color13  #c49ec4\n",[57,46701,46702],{"class":59,"line":5946},[57,46703,3448],{"emptyLinePlaceholder":963},[57,46705,46706],{"class":59,"line":5952},[57,46707,46708],{},"# cyan\n",[57,46710,46711],{"class":59,"line":5967},[57,46712,46713],{},"color6   #8aacab\n",[57,46715,46716],{"class":59,"line":10273},[57,46717,46718],{},"color14  #9ec3c4\n",[57,46720,46721],{"class":59,"line":10279},[57,46722,3448],{"emptyLinePlaceholder":963},[57,46724,46725],{"class":59,"line":10285},[57,46726,46727],{},"# white\n",[57,46729,46730],{"class":59,"line":10312},[57,46731,46732],{},"color15   #e7e7e7\n",[57,46734,46735],{"class":59,"line":10340},[57,46736,46737],{},"color7  #f0f0f0\n",[248,46739,46741],{"id":46740},"themesmatugenconf","themes/matugen.conf",[47,46743,46744],{"className":46448,"code":46539,"language":46450,"meta":52,"style":52},[54,46745,46746,46750,46754,46758,46762,46766,46770,46774,46778,46782,46786,46790,46794,46798,46802,46806,46810,46814,46818,46822,46826,46830,46834,46838,46842,46846,46850,46854,46858,46862,46866,46870,46874,46878,46882,46886,46890,46894,46898,46902,46906],{"__ignoreMap":52},[57,46747,46748],{"class":59,"line":60},[57,46749,46546],{},[57,46751,46752],{"class":59,"line":66},[57,46753,46551],{},[57,46755,46756],{"class":59,"line":72},[57,46757,3448],{"emptyLinePlaceholder":963},[57,46759,46760],{"class":59,"line":78},[57,46761,46560],{},[57,46763,46764],{"class":59,"line":84},[57,46765,46565],{},[57,46767,46768],{"class":59,"line":90},[57,46769,46570],{},[57,46771,46772],{"class":59,"line":96},[57,46773,46575],{},[57,46775,46776],{"class":59,"line":102},[57,46777,46580],{},[57,46779,46780],{"class":59,"line":108},[57,46781,3448],{"emptyLinePlaceholder":963},[57,46783,46784],{"class":59,"line":114},[57,46785,46589],{},[57,46787,46788],{"class":59,"line":120},[57,46789,46594],{},[57,46791,46792],{"class":59,"line":126},[57,46793,46599],{},[57,46795,46796],{"class":59,"line":132},[57,46797,3448],{"emptyLinePlaceholder":963},[57,46799,46800],{"class":59,"line":138},[57,46801,46608],{},[57,46803,46804],{"class":59,"line":144},[57,46805,46613],{},[57,46807,46808],{"class":59,"line":150},[57,46809,46618],{},[57,46811,46812],{"class":59,"line":156},[57,46813,3448],{"emptyLinePlaceholder":963},[57,46815,46816],{"class":59,"line":162},[57,46817,46627],{},[57,46819,46820],{"class":59,"line":168},[57,46821,46632],{},[57,46823,46824],{"class":59,"line":174},[57,46825,46637],{},[57,46827,46828],{"class":59,"line":5790},[57,46829,3448],{"emptyLinePlaceholder":963},[57,46831,46832],{"class":59,"line":5811},[57,46833,46646],{},[57,46835,46836],{"class":59,"line":5830},[57,46837,46651],{},[57,46839,46840],{"class":59,"line":5849},[57,46841,46656],{},[57,46843,46844],{"class":59,"line":5854},[57,46845,3448],{"emptyLinePlaceholder":963},[57,46847,46848],{"class":59,"line":5870},[57,46849,46665],{},[57,46851,46852],{"class":59,"line":5876},[57,46853,46670],{},[57,46855,46856],{"class":59,"line":5882},[57,46857,46675],{},[57,46859,46860],{"class":59,"line":5892},[57,46861,46680],{},[57,46863,46864],{"class":59,"line":5916},[57,46865,3448],{"emptyLinePlaceholder":963},[57,46867,46868],{"class":59,"line":5921},[57,46869,46689],{},[57,46871,46872],{"class":59,"line":5927},[57,46873,46694],{},[57,46875,46876],{"class":59,"line":5941},[57,46877,46699],{},[57,46879,46880],{"class":59,"line":5946},[57,46881,3448],{"emptyLinePlaceholder":963},[57,46883,46884],{"class":59,"line":5952},[57,46885,46708],{},[57,46887,46888],{"class":59,"line":5967},[57,46889,46713],{},[57,46891,46892],{"class":59,"line":10273},[57,46893,46718],{},[57,46895,46896],{"class":59,"line":10279},[57,46897,3448],{"emptyLinePlaceholder":963},[57,46899,46900],{"class":59,"line":10285},[57,46901,46727],{},[57,46903,46904],{"class":59,"line":10312},[57,46905,46732],{},[57,46907,46908],{"class":59,"line":10340},[57,46909,46737],{},[10,46911,46912],{"id":46912},"一些软件",[248,46914,46915],{"id":46915},"nvm",[14,46917,46918,46919,46922],{},"非常好用的 node.js 版本管理器，无需 sudo 权限，能解决系统级 node 下 ",[54,46920,46921],{},"npm install -g"," 安装时各种权限问题。",[14,46924,46925,46926,46930],{},"参考 ",[18,46927,46928],{"href":46928,"rel":46929},"https://github.com/nvm-sh/nvm",[22]," 安装。",[248,46932,46933],{"id":46933},"conda",[14,46935,46936],{},"无需多言。",[248,46938,46939],{"id":46939},"fastfetch",[14,46941,46942],{},"获取本机发行版和硬件信息，比较酷炫，装逼使用。",[14,46944,46945],{},"可以进行美化，参考指令：",[47,46947,46949],{"className":46122,"code":46948,"language":46124,"meta":52,"style":52},"cd $HOME/.config\n\n# 下载并解压然后删除\nwget https://github.com/sqlsec/fastfetch/releases/download/v0.1/fastfetch-for-Linux.zip\nunzip fastfetch-for-Linux.zip && rm fastfetch-for-Linux.zip\n",[54,46950,46951,46961,46965,46970,46978],{"__ignoreMap":52},[57,46952,46953,46955,46958],{"class":59,"line":60},[57,46954,46142],{"class":10193},[57,46956,46957],{"class":46150}," $HOME",[57,46959,46960],{"class":293},"/.config\n",[57,46962,46963],{"class":59,"line":66},[57,46964,3448],{"emptyLinePlaceholder":963},[57,46966,46967],{"class":59,"line":72},[57,46968,46969],{"class":303},"# 下载并解压然后删除\n",[57,46971,46972,46975],{"class":59,"line":78},[57,46973,46974],{"class":3153},"wget",[57,46976,46977],{"class":293}," https://github.com/sqlsec/fastfetch/releases/download/v0.1/fastfetch-for-Linux.zip\n",[57,46979,46980,46983,46986,46989,46992],{"class":59,"line":84},[57,46981,46982],{"class":3153},"unzip",[57,46984,46985],{"class":293}," fastfetch-for-Linux.zip",[57,46987,46988],{"class":269}," && ",[57,46990,46991],{"class":3153},"rm",[57,46993,46994],{"class":293}," fastfetch-for-Linux.zip\n",[14,46996,46997,46998],{},"效果如图：\n",[1000,46999],{"alt":46939,"src":47000},"zsh/fastfetch.png",[248,47002,47003],{"id":47003},"onefetch",[14,47005,47006,47007,47010],{},"获取当前 git 项目信息，使用时需保证当前目录已经 git 初始化（即包含 ",[54,47008,47009],{},".git"," 目录）。",[14,47012,46997,47013],{},[1000,47014],{"alt":47003,"src":47015},"zsh/onefetch.png",[936,47017,47018],{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sCLZk, html code.shiki .sCLZk{--shiki-default:#6F42C1;--shiki-dark:#50FA7B}html pre.shiki code .sEzAm, html code.shiki .sEzAm{--shiki-default:#032F62;--shiki-dark:#F1FA8C}html pre.shiki code .sDgm9, html code.shiki .sDgm9{--shiki-default:#005CC5;--shiki-dark:#8BE9FD}html pre.shiki code .scEb-, html code.shiki .scEb-{--shiki-default:#24292E;--shiki-dark:#BD93F9}html pre.shiki code .szJfE, html code.shiki .szJfE{--shiki-default:#D73A49;--shiki-dark:#FF79C6}html pre.shiki code .soDru, html code.shiki .soDru{--shiki-default:#005CC5;--shiki-dark:#BD93F9}html pre.shiki code .sMWOi, html code.shiki .sMWOi{--shiki-default:#032F62;--shiki-dark:#E9F284}html pre.shiki code .scbbO, html code.shiki .scbbO{--shiki-default:#24292E;--shiki-dark:#F8F8F2}html pre.shiki code .sfgPZ, html code.shiki .sfgPZ{--shiki-default:#6A737D;--shiki-dark:#6272A4}",{"title":52,"searchDepth":66,"depth":66,"links":47020},[47021,47022,47023,47024,47025,47031,47036],{"id":976,"depth":66,"text":976},{"id":46103,"depth":66,"text":46104},{"id":46113,"depth":66,"text":46113},{"id":46172,"depth":66,"text":46173},{"id":46274,"depth":66,"text":46274,"children":47026},[47027,47028,47029,47030],{"id":46280,"depth":72,"text":46280},{"id":46312,"depth":72,"text":46312},{"id":46131,"depth":72,"text":46131},{"id":46387,"depth":72,"text":46387},{"id":46426,"depth":66,"text":46427,"children":47032},[47033,47034,47035],{"id":46444,"depth":72,"text":46445},{"id":46535,"depth":72,"text":46536},{"id":46740,"depth":72,"text":46741},{"id":46912,"depth":66,"text":46912,"children":47037},[47038,47039,47040,47041],{"id":46915,"depth":72,"text":46915},{"id":46933,"depth":72,"text":46933},{"id":46939,"depth":72,"text":46939},{"id":47003,"depth":72,"text":47003},{"date":47043,"image":47044,"alt":47045,"tags":47046,"published":963},"28 April 2026","/blogs-img/blog3.jpg","Zsh 配置",[47047,47048],"Linux","折腾笔记","/blogs/zsh",{"title":46093,"description":46093},"blogs/3. 记录一下我的 zsh 配置","HJxt7N1Hp2UuZpzLJWJCQuLpJNFDWThzx-XI5ReXWIw",{"id":47054,"title":47055,"body":47056,"description":47231,"extension":955,"meta":47232,"navigation":963,"ogImage":47234,"path":47239,"seo":47240,"stem":47241,"__hash__":47242},"content/blogs/4. Agent 相关的折腾记录（持续更新）.md","Agent 相关的折腾记录（持续更新）",{"type":7,"value":47057,"toc":47218},[47058,47060,47063,47066,47069,47073,47076,47080,47083,47086,47090,47093,47097,47100,47103,47106,47110,47113,47116,47123,47126,47137,47141,47144,47147,47150,47159,47168,47172,47183,47205],[10,47059,976],{"id":976},[14,47061,47062],{},"最近终于静下心来认真研究 Agent 生态。在此之前一直没下场深入折腾，主要是觉得这个领域生态变化太快、技术栈也不够成熟，迭代节奏完全跟不上。直到最近，Agent 相关的概念（Tools、MCP、Skills、Subagents、Hooks 等）逐渐收敛到一个相对稳定的形态，于是开始全面尝试。",[10,47064,47065],{"id":47065},"介绍一下我有哪些工具",[14,47067,47068],{},"截至目前，我体验过 Claude Code、Codex、Cursor，最近还试用了 SOLO Trae（支持国产）。下面分别说一下感受。",[248,47070,47072],{"id":47071},"claude-code","Claude Code",[14,47074,47075],{},"目前我用过编程体验最好的一个 Agent。聪明程度在一众 Coding Agent 里算是天花板级别，插件生态成熟，可以与 Cursor / VSCode 等编辑器协同使用：在终端里启动 Claude Code 后，能直接选中编辑器里的某段代码作为上下文同步过去；它给出的修改建议也能在编辑器里直观地 review 与 apply，整体闭环非常顺滑。",[248,47077,47079],{"id":47078},"codex","Codex",[14,47081,47082],{},"量大管饱，模型水平也相当能打，开 high reasoning 写代码完全顶得住。相比之下生态略弱一些，更偏\"原生 IDE 助手\"路线。",[14,47084,47085],{},"桌面端不错，浏览器自动化操作效果还挺好。",[248,47087,47089],{"id":47088},"cursor","Cursor",[14,47091,47092],{},"中规中矩。用 GPT codex 系列经常出现\"降智\"，Claude Sonnet 与 Opus 又容易过度思考；简单任务用 auto 模式既快又便宜。Tab 补全确实惊艳，对于\"手写古人\"来说值得一试。",[248,47094,47096],{"id":47095},"solo-trae","SOLO Trae",[14,47098,47099],{},"目前模型免费，可以用来跑一些轻量任务。问题在于自定义模型能力较弱、调用 Skill 还需要显式指定，加之经常需要排队，整体体验仍有较大提升空间。",[14,47101,47102],{},"最近 SOLO 更新了，可以使用制定模型了，热门国产模型基本都部署了，现在还是免费用，感觉平时也不会触发排队，比之前好多了，支持国产。",[14,47104,47105],{},"5 月底尝试了一下操作浏览器功能，非常拉，而且只有 auto 能操作浏览器，目前来看拉完了，远不如 codex。",[10,47107,47109],{"id":47108},"skills","Skills",[14,47111,47112],{},"最近发现 Skills 能显著扩展模型的能力边界——它本质上是一份\"在特定场景下注入到上下文的提示词包\"，让 Agent 在面对垂直任务时能够直接调用既定的方法论与工具链。后续打算系统挖掘一下 Skills 的设计模式与最佳实践。",[47114,47115],"hr",{},[14,47117,47118,47119,47122],{},"现在的 Skills 基本都能在 ",[18,47120,47121],{"href":47121},"skills.sh"," 这个网站上下载并获取，下面提到的 skills 也都是在这上面获取的。",[14,47124,47125],{},"发现一些比较好用的 Skills，下面简单介绍一下：",[13410,47127,47128,47131,47134],{},[13413,47129,47130],{},"Superpowers：耳熟能详了，性能非常强，brainstroming 感觉效果比绝大多数进行创新设计的 skill 效果都好，工作流大概就是设计，问你很多问题，然后 spec 模式，设计方案人工确定，之后 subagent 开始实施方案。",[13413,47132,47133],{},"find-skills：自动查找相关的 Skills。",[13413,47135,47136],{},"prompt-engineering-patterns：提示词设计范式。",[10,47138,47140],{"id":47139},"mcp","MCP",[14,47142,47143],{},"MCP（Model Context Protocol）方面体验还不算多。目前高频使用的是 Notion 的 MCP，可以让 AI 直接把整理好的内容写入 Notion 工作区，大大减少了\"复制 → 粘贴 → 排版\"的人肉环节。",[14,47145,47146],{},"最近使用了 CodeGraph，也是以 MCP 的形式提供服务，网上很流行，刚开始体验，使用一段时间再来谈谈自己的感受。",[10,47148,47149],{"id":47149},"杂谈",[14,47151,47152,47153,47158],{},"我还使用 CC-Switch 来管理多个 API 渠道。Claude Code 的主力渠道是公益站 ",[18,47154,47157],{"href":47155,"rel":47156},"https://anyrouter.top/register?aff=wwiE",[22],"AnyRouter","，唯一的痛点是稳定性偏弱；不过最近恢复正常了，准备再战一波。",[14,47160,47161,47162,47167],{},"还使用了 ",[18,47163,47166],{"href":47164,"rel":47165},"https://unity2.ai/register?ref=jOAqkDQC",[22],"Unity2"," 这个中转站，能开发票，价格很便宜，用起来也挺稳定。",[10,47169,47171],{"id":47170},"zcf","ZCF",[14,47173,47174,47175,47178,47179,47182],{},"ZCF，全称 ",[11865,47176,47177],{},"Zero-Config Code Flow","，是一个面向专业开发者的 CLI 工具，目标是在几分钟内完成 Claude Code 与 Codex 的端到端环境初始化。通过 ",[54,47180,47181],{},"npx zcf"," 即可一站式完成：配置目录创建、API/代理接入、MCP 服务接入、工作流导入、输出风格与记忆配置，以及常用工具安装。",[14,47184,47185,47186,47189,47190,47193,47194,47197,47198,47201,47202,47204],{},"最近体验了一下 ",[54,47187,47188],{},"zcf:init-project","，发现确实比 Claude Code 与 Codex 自带的 ",[54,47191,47192],{},"init"," 强不少。它会生成总体级与目录级的 ",[54,47195,47196],{},"CLAUDE.md"," 作为指导文件，还会生成 ",[54,47199,47200],{},".claude/index.json"," 作为文件索引并记录修改时间；之后每次用 Claude Code 修改代码，都会顺带同步更新 ",[54,47203,47196],{},"，把\"何时改了什么\"完整记录下来。这种做法极大提升了 AI 代码的可维护性，避免出现\"AI 拉了一大坨，谁也不敢动\"的局面。代价是会增加 token 消耗，但在我看来收益远大于成本。",[14,47206,47207,47208,194,47211,743,47214,47217],{},"zcf 在安装时还可以可选地安装一些 MCP 服务。我个人常用的组合是 ",[11865,47209,47210],{},"spec-workflow",[11865,47212,47213],{},"context7",[11865,47215,47216],{},"open-websearch"," 这三个，不过最近 spec-workflow 经常连不上，后面要找时间排查一下。",{"title":52,"searchDepth":66,"depth":66,"links":47219},[47220,47221,47227,47228,47229,47230],{"id":976,"depth":66,"text":976},{"id":47065,"depth":66,"text":47065,"children":47222},[47223,47224,47225,47226],{"id":47071,"depth":72,"text":47072},{"id":47078,"depth":72,"text":47079},{"id":47088,"depth":72,"text":47089},{"id":47095,"depth":72,"text":47096},{"id":47108,"depth":66,"text":47109},{"id":47139,"depth":66,"text":47140},{"id":47149,"depth":66,"text":47149},{"id":47170,"depth":66,"text":47171},"记录自己折腾一些智能体相关技术的经历",{"date":47233,"image":47234,"alt":47235,"tags":47236,"published":963},"30 April 2026","/blogs-img/blog4.jpg","Agent",[47235,47048,47237,47140,47109,47238],"workflow","harness","/blogs/agent",{"title":47055,"description":47231},"blogs/4. Agent 相关的折腾记录（持续更新）","FBIISUPIL5-ScneukQvaoUq-1e-iXhbiWYzjjZ0b9I4",{"id":47244,"title":47245,"body":47246,"description":47299,"extension":955,"meta":47300,"navigation":963,"ogImage":47302,"path":47306,"seo":47307,"stem":47308,"__hash__":47309},"content/blogs/5. OpenMP 学习笔记.md","OpenMP 学习笔记",{"type":7,"value":47247,"toc":47294},[47248,47250,47253,47257,47262,47271,47273],[10,47249,976],{"id":976},[14,47251,47252],{},"这学期选修了并行计算这门课程，但学期过半一点还没开始学（），之前的实验啥的全仰仗 GPT 大人，最近开始打算好好学一学，先从课程的第一部分：CPU 并行学习。",[10,47254,47256],{"id":47255},"openmp","OpenMP",[46338,47258,47259],{},[14,47260,47261],{},"OpenMP（Open Multi-Processing）是一套支持跨平台共享内存方式的多线程并发的编程API，使用C,C++和Fortran语言，可以在大多数的处理器体系和操作系统中运行，包括Solaris, AIX, HP-UX, GNU/Linux, Mac OS X, 和Microsoft Windows。包括一套编译器指令、库和一些能够影响运行行为的环境变量。",[14,47263,47264,47265,47270],{},"这个介绍摘自 ",[18,47266,47269],{"href":47267,"rel":47268},"https://zh.wikipedia.org/wiki/OpenMP",[22],"维基百科 OpenMP","，在目前的并行计算框架中，OpenMP 是最容易上手的一个框架，通过许多高层抽象隐藏了实现的具体细节，只需要串行程序中加入OpenMP预处理指令, 就可以实现串行程序的并行化。",[10,47272,46059],{"id":46059},[13410,47274,47275,47281,47287],{},[13413,47276,47277],{},[18,47278,47245],{"href":47279,"rel":47280},"https://parallel.zhangjikai.com/openmp.html",[22],[13413,47282,47283],{},[18,47284,47286],{"href":47267,"rel":47285},[22],"维基百科",[13413,47288,47289],{},[18,47290,47293],{"href":47291,"rel":47292},"https://www.bilibili.com/video/BV1SW411s7ST",[22],"Introduction to OpenMP - Tim Mattson (Intel)",{"title":52,"searchDepth":66,"depth":66,"links":47295},[47296,47297,47298],{"id":976,"depth":66,"text":976},{"id":47255,"depth":66,"text":47256},{"id":46059,"depth":66,"text":46059},"2026年春季学期并行计算课程的OpenMP学习笔记。",{"date":47301,"image":47302,"alt":47303,"tags":47304,"published":963},"1 May 2026","/blogs-img/blog5.jpg","并行计算学习笔记",[47047,47305],"并行计算","/blogs/openmp",{"title":47245,"description":47299},"blogs/5. OpenMP 学习笔记","V_NEV-_XjkKYLEccNpXv1GNiYYWcobHoNR-CB-AARoE",{"id":47311,"title":47312,"body":47313,"description":47550,"extension":955,"meta":47551,"navigation":963,"ogImage":47552,"path":47556,"seo":47557,"stem":47558,"__hash__":47559},"content/blogs/6. 从零开始在 TUI 服务器中启动 clash.md","从零开始在 TUI 服务器中启动 clash",{"type":7,"value":47314,"toc":47540},[47315,47317,47320,47323,47326,47329,47332,47336,47345,47352,47403,47406,47409,47416,47420,47439,47442,47445,47460,47463,47466,47503,47508,47518,47521,47537],[10,47316,976],{"id":976},[14,47318,47319],{},"相信国内的朋友在用自己实验室服务器的时候总会遇到各种网络问题，比如从 GitHub 上 clone 不下来代码，push 不上去代码，下载模型，数据集非常缓慢等。网络问题已经成为大家科研路上的一个重要绊脚石！！",[14,47321,47322],{},"为了帮助更多的人解决这个问题，我想分享一下我的经验。",[10,47324,47325],{"id":47325},"准备",[14,47327,47328],{},"这篇文章假设你已经学会了在自己电脑上使用任意一款有 GUI 页面的 Clash 进行翻墙，你需要有一个能正常工作的 Clash，正常运行的机场，并在自己的 Clash 里成功使用这个机场翻墙。",[14,47330,47331],{},"同时假设你没有服务器的 sudo 权限，服务器使用 Linux 系统，CPU 为 x86_64（amd64）版本。",[248,47333,47335],{"id":47334},"下载-mihomo-内核","下载 mihomo 内核",[14,47337,47338,47339,47344],{},"从这里 ",[18,47340,47343],{"href":47341,"rel":47342},"https://github.com/MetaCubeX/mihomo/releases",[22],"MetaCubeX/mihomo"," 下载最新的 mihomo 内核。",[14,47346,47347,47348,47351],{},"从中找到 ",[54,47349,47350],{},"mihomo-linux-amd64-compatible-v*.gz","（* 表示任意版本），然后下载，复制到你的服务器上，解压并重命名为 clash，指令如下：",[47,47353,47355],{"className":46122,"code":47354,"language":46124,"meta":52,"style":52},"gzip -d mihomo-linux-amd64-compatible-v*.gz\n\nmv mihomo-linux-amd64-compatible-v* clash\n\nchmod +x clash\n",[54,47356,47357,47373,47377,47389,47393],{"__ignoreMap":52},[57,47358,47359,47362,47365,47368,47370],{"class":59,"line":60},[57,47360,47361],{"class":3153},"gzip",[57,47363,47364],{"class":273}," -d",[57,47366,47367],{"class":293}," mihomo-linux-amd64-compatible-v",[57,47369,9461],{"class":265},[57,47371,47372],{"class":293},".gz\n",[57,47374,47375],{"class":59,"line":66},[57,47376,3448],{"emptyLinePlaceholder":963},[57,47378,47379,47382,47384,47386],{"class":59,"line":72},[57,47380,47381],{"class":3153},"mv",[57,47383,47367],{"class":293},[57,47385,9461],{"class":265},[57,47387,47388],{"class":293}," clash\n",[57,47390,47391],{"class":59,"line":78},[57,47392,3448],{"emptyLinePlaceholder":963},[57,47394,47395,47398,47401],{"class":59,"line":84},[57,47396,47397],{"class":3153},"chmod",[57,47399,47400],{"class":293}," +x",[57,47402,47388],{"class":293},[14,47404,47405],{},"我个人建议吧解压出的 clash 单独放置一个文件夹。",[248,47407,47408],{"id":47408},"准备机场配置",[14,47410,47411,47412,47415],{},"启动你的电脑中的 clash，点击订阅，右键你当前激活的订阅，点击编辑文件，然后将出现的 yaml 文件内容全部复制下来，在 clash 同目录下创建一个 ",[54,47413,47414],{},"config.yaml"," 文件，并将刚才复制的内容全部粘贴进去。",[248,47417,47419],{"id":47418},"geox-url-准备","geox-url 准备",[14,47421,47422,47423,7517,47428,198,47433,47438],{},"分别下载 ",[18,47424,47427],{"href":47425,"rel":47426},"https://fastly.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.dat",[22],"geoip",[18,47429,47432],{"href":47430,"rel":47431},"https://fastly.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.dat",[22],"geosite",[18,47434,47437],{"href":47435,"rel":47436},"https://fastly.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.metadb",[22],"mmdb"," 到同级文件夹。",[10,47440,47441],{"id":47441},"启动",[14,47443,47444],{},"执行下述指令，你的 clash 就启动成功了，从启动日志上可以看到你的监听端口，外部控制端口等信息。",[47,47446,47448],{"className":46122,"code":47447,"language":46124,"meta":52,"style":52},"./clash -d ./\n",[54,47449,47450],{"__ignoreMap":52},[57,47451,47452,47455,47457],{"class":59,"line":60},[57,47453,47454],{"class":3153},"./clash",[57,47456,47364],{"class":273},[57,47458,47459],{"class":293}," ./\n",[10,47461,47462],{"id":47462},"设置系统代理",[14,47464,47465],{},"在一个终端中启动 clash 后，在需要访问境外服务的终端上设置系统代理，指令如下：",[47,47467,47469],{"className":46122,"code":47468,"language":46124,"meta":52,"style":52},"export https_proxy=\"http://127.0.0.1:7890\"\nexport http_proxy=\"http://127.0.0.1:7890\"\n",[54,47470,47471,47488],{"__ignoreMap":52},[57,47472,47473,47476,47479,47481,47483,47486],{"class":59,"line":60},[57,47474,47475],{"class":280},"export",[57,47477,47478],{"class":46150}," https_proxy",[57,47480,281],{"class":280},[57,47482,297],{"class":289},[57,47484,47485],{"class":293},"http://127.0.0.1:7890",[57,47487,46227],{"class":289},[57,47489,47490,47492,47495,47497,47499,47501],{"class":59,"line":66},[57,47491,47475],{"class":280},[57,47493,47494],{"class":46150}," http_proxy",[57,47496,281],{"class":280},[57,47498,297],{"class":289},[57,47500,47485],{"class":293},[57,47502,46227],{"class":289},[14,47504,47505,47506,222],{},"注意这里没有输错，https_proxy 对应的就是 ",[54,47507,47485],{},[14,47509,47510,47511,47513,47514,47517],{},"这里的端口可能会根据 ",[54,47512,47414],{}," 文件的不同而有所变化，注意日志中的 ",[54,47515,47516],{},"mixed_port","，实际以这个为准。",[14,47519,47520],{},"到这里配置就结束了，可以链接一下谷歌自行测试联通性。",[47,47522,47524],{"className":46122,"code":47523,"language":46124,"meta":52,"style":52},"curl -v www.google.com\n",[54,47525,47526],{"__ignoreMap":52},[57,47527,47528,47531,47534],{"class":59,"line":60},[57,47529,47530],{"class":3153},"curl",[57,47532,47533],{"class":273}," -v",[57,47535,47536],{"class":293}," www.google.com\n",[936,47538,47539],{},"html pre.shiki code .sCLZk, html code.shiki .sCLZk{--shiki-default:#6F42C1;--shiki-dark:#50FA7B}html pre.shiki code .soDru, html code.shiki .soDru{--shiki-default:#005CC5;--shiki-dark:#BD93F9}html pre.shiki code .sEzAm, html code.shiki .sEzAm{--shiki-default:#032F62;--shiki-dark:#F1FA8C}html pre.shiki code .sJti5, html code.shiki .sJti5{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .szJfE, html code.shiki .szJfE{--shiki-default:#D73A49;--shiki-dark:#FF79C6}html pre.shiki code .scEb-, html code.shiki .scEb-{--shiki-default:#24292E;--shiki-dark:#BD93F9}html pre.shiki code .sMWOi, html code.shiki .sMWOi{--shiki-default:#032F62;--shiki-dark:#E9F284}",{"title":52,"searchDepth":66,"depth":66,"links":47541},[47542,47543,47548,47549],{"id":976,"depth":66,"text":976},{"id":47325,"depth":66,"text":47325,"children":47544},[47545,47546,47547],{"id":47334,"depth":72,"text":47335},{"id":47408,"depth":72,"text":47408},{"id":47418,"depth":72,"text":47419},{"id":47441,"depth":66,"text":47441},{"id":47462,"depth":66,"text":47462},"探究如何在纯 TUI 服务器上启动 clash，解决中国境内的网络问题",{"date":47301,"image":47552,"alt":47312,"tags":47553,"published":963},"/blogs-img/blog7.jpg",[47047,47554,47555],"折腾日记","实用小技能","/blogs/tui-clash",{"title":47312,"description":47550},"blogs/6. 从零开始在 TUI 服务器中启动 clash","AvoRP3GN22KzFCVh-Ow4yoThlOcNJmVa9ZY64wtyd4c",{"id":4,"title":5,"body":47561,"description":5,"extension":955,"meta":48218,"navigation":963,"ogImage":958,"path":964,"seo":48220,"stem":966,"__hash__":967},{"type":7,"value":47562,"toc":48203},[47563,47565,47570,47575,47577,47579,47581,47583,47667,47671,47681,47685,47693,47701,47703,47707,47709,47711,47843,47847,47849,47853,47859,47907,47911,47913,47915,48037,48039,48041,48043,48045,48047,48053,48067,48073,48081,48155,48157,48161,48165,48167,48169,48173,48175,48177,48183,48185,48187,48189,48191,48193,48195,48197,48199,48201],[10,47564,12],{"id":12},[14,47566,16,47567,24],{},[18,47568,23],{"href":20,"rel":47569},[22],[14,47571,27,47572,33],{},[18,47573,32],{"href":30,"rel":47574},[22],[10,47576,36],{"id":36},[14,47578,39],{},[14,47580,42],{},[10,47582,45],{"id":45},[47,47584,47585],{"className":49,"code":50,"language":51,"meta":52,"style":52},[54,47586,47587,47591,47595,47599,47603,47607,47611,47615,47619,47623,47627,47631,47635,47639,47643,47647,47651,47655,47659,47663],{"__ignoreMap":52},[57,47588,47589],{"class":59,"line":60},[57,47590,63],{},[57,47592,47593],{"class":59,"line":66},[57,47594,69],{},[57,47596,47597],{"class":59,"line":72},[57,47598,75],{},[57,47600,47601],{"class":59,"line":78},[57,47602,81],{},[57,47604,47605],{"class":59,"line":84},[57,47606,87],{},[57,47608,47609],{"class":59,"line":90},[57,47610,93],{},[57,47612,47613],{"class":59,"line":96},[57,47614,99],{},[57,47616,47617],{"class":59,"line":102},[57,47618,105],{},[57,47620,47621],{"class":59,"line":108},[57,47622,111],{},[57,47624,47625],{"class":59,"line":114},[57,47626,117],{},[57,47628,47629],{"class":59,"line":120},[57,47630,123],{},[57,47632,47633],{"class":59,"line":126},[57,47634,129],{},[57,47636,47637],{"class":59,"line":132},[57,47638,135],{},[57,47640,47641],{"class":59,"line":138},[57,47642,141],{},[57,47644,47645],{"class":59,"line":144},[57,47646,147],{},[57,47648,47649],{"class":59,"line":150},[57,47650,153],{},[57,47652,47653],{"class":59,"line":156},[57,47654,159],{},[57,47656,47657],{"class":59,"line":162},[57,47658,165],{},[57,47660,47661],{"class":59,"line":168},[57,47662,171],{},[57,47664,47665],{"class":59,"line":174},[57,47666,177],{},[14,47668,180,47669,184],{},[54,47670,183],{},[14,47672,47673,190,47675,194,47677,198,47679,202],{},[54,47674,189],{},[54,47676,193],{},[54,47678,197],{},[54,47680,201],{},[14,47682,47683,208],{},[54,47684,207],{},[14,47686,47687,214,47689,218,47691,222],{},[54,47688,213],{},[54,47690,217],{},[54,47692,221],{},[14,47694,47695,228,47697,232,47699,236],{},[54,47696,227],{},[54,47698,231],{},[54,47700,235],{},[10,47702,239],{"id":239},[14,47704,242,47705,246],{},[54,47706,245],{},[248,47708,250],{"id":250},[14,47710,253],{},[47,47712,47713],{"className":256,"code":257,"language":258,"meta":52,"style":52},[54,47714,47715,47729,47741,47753,47765,47777,47795,47809,47827,47839],{"__ignoreMap":52},[57,47716,47717,47719,47721,47723,47725,47727],{"class":59,"line":60},[57,47718,266],{"class":265},[57,47720,270],{"class":269},[57,47722,274],{"class":273},[57,47724,277],{"class":269},[57,47726,281],{"class":280},[57,47728,284],{"class":269},[57,47730,47731,47733,47735,47737,47739],{"class":59,"line":66},[57,47732,290],{"class":289},[57,47734,294],{"class":293},[57,47736,297],{"class":289},[57,47738,300],{"class":269},[57,47740,304],{"class":303},[57,47742,47743,47745,47747,47749,47751],{"class":59,"line":72},[57,47744,290],{"class":289},[57,47746,311],{"class":293},[57,47748,297],{"class":289},[57,47750,316],{"class":269},[57,47752,304],{"class":303},[57,47754,47755,47757,47759,47761,47763],{"class":59,"line":78},[57,47756,290],{"class":289},[57,47758,325],{"class":293},[57,47760,297],{"class":289},[57,47762,330],{"class":269},[57,47764,333],{"class":303},[57,47766,47767,47769,47771,47773,47775],{"class":59,"line":84},[57,47768,290],{"class":289},[57,47770,340],{"class":293},[57,47772,297],{"class":289},[57,47774,330],{"class":269},[57,47776,347],{"class":303},[57,47778,47779,47781,47783,47785,47787,47789,47791,47793],{"class":59,"line":90},[57,47780,290],{"class":289},[57,47782,354],{"class":293},[57,47784,297],{"class":289},[57,47786,359],{"class":269},[57,47788,297],{"class":289},[57,47790,364],{"class":293},[57,47792,297],{"class":289},[57,47794,369],{"class":269},[57,47796,47797,47799,47801,47803,47805,47807],{"class":59,"line":96},[57,47798,290],{"class":289},[57,47800,376],{"class":293},[57,47802,297],{"class":289},[57,47804,381],{"class":269},[57,47806,384],{"class":273},[57,47808,369],{"class":269},[57,47810,47811,47813,47815,47817,47819,47821,47823,47825],{"class":59,"line":102},[57,47812,290],{"class":289},[57,47814,393],{"class":293},[57,47816,297],{"class":289},[57,47818,398],{"class":269},[57,47820,297],{"class":289},[57,47822,403],{"class":293},[57,47824,297],{"class":289},[57,47826,369],{"class":269},[57,47828,47829,47831,47833,47835,47837],{"class":59,"line":108},[57,47830,290],{"class":289},[57,47832,414],{"class":293},[57,47834,297],{"class":289},[57,47836,330],{"class":269},[57,47838,421],{"class":303},[57,47840,47841],{"class":59,"line":114},[57,47842,426],{"class":269},[14,47844,429,47845,433],{},[57,47846,432],{},[14,47848,436],{},[248,47850,47851,443],{"id":439},[54,47852,442],{},[14,47854,47855,448,47857,452],{},[54,47856,442],{},[54,47858,451],{},[454,47860,47861,47871],{},[457,47862,47863],{},[460,47864,47865,47867,47869],{},[463,47866,465],{},[463,47868,468],{},[463,47870,471],{},[473,47872,47873,47887,47897],{},[460,47874,47875,47877,47883],{},[478,47876,480],{},[478,47878,47879,486,47881],{},[54,47880,485],{},[54,47882,489],{},[478,47884,47885,495],{},[54,47886,494],{},[460,47888,47889,47891,47895],{},[478,47890,500],{},[478,47892,47893,506],{},[54,47894,505],{},[478,47896,509],{},[460,47898,47899,47901,47905],{},[478,47900,514],{},[478,47902,47903,520],{},[54,47904,519],{},[478,47906,509],{},[14,47908,525,47909,529],{},[54,47910,528],{},[14,47912,532],{},[14,47914,535],{},[454,47916,47917,47929],{},[457,47918,47919],{},[460,47920,47921,47923,47925,47927],{},[463,47922,544],{},[463,47924,547],{},[463,47926,550],{},[463,47928,553],{},[473,47930,47931,47941,47953,47965,47977,47989,48001,48013,48025],{},[460,47932,47933,47935,47937,47939],{},[478,47934,560],{},[478,47936,563],{},[478,47938,566],{},[478,47940,569],{},[460,47942,47943,47945,47947,47949],{},[478,47944,574],{},[478,47946,577],{},[478,47948,580],{},[478,47950,47951],{},[54,47952,585],{},[460,47954,47955,47957,47959,47961],{},[478,47956,590],{},[478,47958,593],{},[478,47960,596],{},[478,47962,47963],{},[54,47964,601],{},[460,47966,47967,47969,47971,47973],{},[478,47968,606],{},[478,47970,609],{},[478,47972,612],{},[478,47974,47975],{},[54,47976,617],{},[460,47978,47979,47981,47983,47985],{},[478,47980,622],{},[478,47982,625],{},[478,47984,580],{},[478,47986,47987],{},[54,47988,632],{},[460,47990,47991,47993,47995,47997],{},[478,47992,637],{},[478,47994,640],{},[478,47996,580],{},[478,47998,47999],{},[54,48000,647],{},[460,48002,48003,48005,48007,48009],{},[478,48004,652],{},[478,48006,655],{},[478,48008,658],{},[478,48010,48011],{},[54,48012,663],{},[460,48014,48015,48017,48019,48021],{},[478,48016,668],{},[478,48018,671],{},[478,48020,674],{},[478,48022,48023],{},[54,48024,679],{},[460,48026,48027,48029,48031,48033],{},[478,48028,684],{},[478,48030,687],{},[478,48032,580],{},[478,48034,48035],{},[54,48036,694],{},[14,48038,697],{},[14,48040,700],{},[14,48042,703],{},[14,48044,706],{},[10,48046,709],{"id":709},[14,48048,712,48049,716,48051,720],{},[54,48050,715],{},[54,48052,719],{},[14,48054,723,48055,727,48057,731,48059,735,48061,739,48063,743,48065,747],{},[54,48056,726],{},[54,48058,730],{},[54,48060,734],{},[54,48062,738],{},[54,48064,742],{},[54,48066,746],{},[14,48068,750,48069,198,48071,757],{},[54,48070,753],{},[54,48072,756],{},[14,48074,760,48075,764,48077,768,48079,771],{},[54,48076,763],{},[54,48078,767],{},[54,48080,217],{},[454,48082,48083,48093],{},[457,48084,48085],{},[460,48086,48087,48089,48091],{},[463,48088,780],{},[463,48090,783],{},[463,48092,786],{},[473,48094,48095,48107,48119,48131,48143],{},[460,48096,48097,48101,48105],{},[478,48098,48099],{},[54,48100,795],{},[478,48102,48103],{},[54,48104,800],{},[478,48106,803],{},[460,48108,48109,48113,48117],{},[478,48110,48111],{},[54,48112,810],{},[478,48114,48115],{},[54,48116,800],{},[478,48118,817],{},[460,48120,48121,48125,48129],{},[478,48122,48123],{},[54,48124,824],{},[478,48126,48127],{},[54,48128,800],{},[478,48130,831],{},[460,48132,48133,48137,48141],{},[478,48134,48135],{},[54,48136,838],{},[478,48138,48139],{},[54,48140,800],{},[478,48142,845],{},[460,48144,48145,48149,48153],{},[478,48146,48147],{},[54,48148,852],{},[478,48150,48151],{},[54,48152,574],{},[478,48154,859],{},[248,48156,863],{"id":862},[14,48158,866,48159,870],{},[54,48160,869],{},[14,48162,873,48163,877],{},[54,48164,876],{},[14,48166,880],{},[14,48168,883],{},[14,48170,886,48171,890],{},[54,48172,889],{},[14,48174,893],{},[14,48176,896],{},[14,48178,48179,902,48181,906],{},[54,48180,901],{},[54,48182,905],{},[248,48184,910],{"id":909},[14,48186,913],{},[14,48188,916],{},[14,48190,919],{},[14,48192,922],{},[14,48194,925],{},[14,48196,928],{},[10,48198,931],{"id":931},[10,48200,934],{"id":934},[936,48202,938],{},{"title":52,"searchDepth":66,"depth":66,"links":48204},[48205,48206,48207,48208,48212,48216,48217],{"id":12,"depth":66,"text":12},{"id":36,"depth":66,"text":36},{"id":45,"depth":66,"text":45},{"id":239,"depth":66,"text":239,"children":48209},[48210,48211],{"id":250,"depth":72,"text":250},{"id":439,"depth":72,"text":948},{"id":709,"depth":66,"text":709,"children":48213},[48214,48215],{"id":862,"depth":72,"text":863},{"id":909,"depth":72,"text":910},{"id":931,"depth":66,"text":931},{"id":934,"depth":66,"text":934},{"date":957,"image":958,"alt":959,"tags":48219,"published":963},[961,962],{"title":5,"description":5},1781092795334]