index.html

<!DOCTYPE html>


<html class="theme-next gemini use-motion" lang="zh-Hans">
<head>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">


<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />


  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />


<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222">


  <meta name="keywords" content="Hola Bellman" />


<meta property="og:type" content="website">
<meta property="og:title" content="About Learning">
<meta property="og:url" content="http://yoursite.com/index.html">
<meta property="og:site_name" content="About Learning">
<meta property="og:locale" content="zh-Hans">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="About Learning">


<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    version: '5.1.4',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>


  <link rel="canonical" href="http://yoursite.com/"/>


  <title>About Learning</title><!-- hexo-inject:begin --><!-- hexo-inject:end -->
  

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="container sidebar-position-left 
  page-home">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">About Learning</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle"></p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      

    </ul>
  

</nav>


 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
  <section id="posts" class="posts-expand">
    
      
  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/11/18/强化学习漫谈-14-Multi-Agent-2/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/11/18/强化学习漫谈-14-Multi-Agent-2/" itemprop="url">强化学习漫谈 14：Multi Agent RL 之二</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-11-18T21:01:23+08:00">
                2018-11-18
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/11/18/强化学习漫谈-14-Multi-Agent-2/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/11/18/强化学习漫谈-14-Multi-Agent-2/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/11/18/强化学习漫谈-14-Multi-Agent-2/" class="leancloud_visitors" data-flag-title="强化学习漫谈 14：Multi Agent RL 之二">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>本节介绍<strong>去中心化策略的集中式学习</strong>(Centralized Learning of Decentralized Policies)这一类算法，具体会介绍三种算法：RLAR，COMA，QMIX。</p>
<p>这类算法的主要想法是：训练过程是离线的，所以可以有环境状态与joint action的全局观测(fully observation)；而执行过程是在线的，各个agent独自按照学习的策略进行每一步动作，策略不是状态$s$的函数，而是<strong>action-observation history </strong> $\mathcal{H}_{i,t}$的函数。<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/11/18/强化学习漫谈-14-Multi-Agent-2/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/11/18/强化学习漫谈-13-Multi-Agent-1/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/11/18/强化学习漫谈-13-Multi-Agent-1/" itemprop="url">强化学习漫谈 13：Multi Agent RL 之一</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-11-18T00:22:34+08:00">
                2018-11-18
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/11/18/强化学习漫谈-13-Multi-Agent-1/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/11/18/强化学习漫谈-13-Multi-Agent-1/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/11/18/强化学习漫谈-13-Multi-Agent-1/" class="leancloud_visitors" data-flag-title="强化学习漫谈 13：Multi Agent RL 之一">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>前面介绍了强化学习大致的理论框架和实际使用的一些方法，这个体系都是围绕着agent对environment进行观察、动作、得到反馈这样的交互方式来进行的。我们把这个问题的适用范围进一步扩大，如果有多个agent同时对environment进行观察、动作、得到反馈；如果这多个agent或者互相协助，或者互相竞争，或者互相博弈；如果这些agent对环境本身没有全貌的观察；如果每个agent的Q value受到其他agent的影响；… …；我们应该有什么样的理论框架和实际方法？我们面临什么样的机会和挑战？下面就从理论框架和实际方法两个方面介绍Multi-Agent（多智体）的强化学习。</p>
<h3 id="理论框架：Stochastic-Game-SG-与-Dec-POMDP"><a href="#理论框架：Stochastic-Game-SG-与-Dec-POMDP" class="headerlink" title="理论框架：Stochastic Game (SG) 与 Dec-POMDP"></a>理论框架：Stochastic Game (SG) 与 Dec-POMDP</h3>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/11/18/强化学习漫谈-13-Multi-Agent-1/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/" itemprop="url">强化学习漫谈12：DQN、DDPG、A3C</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-10-26T23:06:31+08:00">
                2018-10-26
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/" class="leancloud_visitors" data-flag-title="强化学习漫谈12：DQN、DDPG、A3C">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>Going Deep。DQN<a href="https://arxiv.org/abs/1312.5602" target="_blank" rel="noopener">论文1</a>、<a href="https://www.nature.com/articles/nature14236" target="_blank" rel="noopener">论文2</a>; DDPG<a href="https://arxiv.org/abs/1509.02971" target="_blank" rel="noopener">论文</a>；A3C<a href="https://arxiv.org/abs/1602.01783" target="_blank" rel="noopener">论文</a>。</p>
<h3 id="DQN：Deep-Q-Learning"><a href="#DQN：Deep-Q-Learning" class="headerlink" title="DQN：Deep Q Learning"></a>DQN：Deep Q Learning</h3><p>在Function Approximation中，我们希望得到的是模型的表达能力和泛化能力；另一方面，当状态数据来自于图像、视频、声音等原始数据时，我们也希望增强模型的特征提取能力，减少人工特征工程。因此运用深度学习对强化学习中的value function、policy function进行端到端的approximation，成为一个有趣的选项。</p>
<p>然而不做任何trick地直接应用深度学习，存在如下问题：</p>
<ol>
<li>强化学习中的return计算通常相对当前的time step而言有延迟，即使是Bootstrapping，也会有n-step的延迟。</li>
<li>深度学习的训练样本需要独立同分布i.i.d，而强化学习的状态、回报等都有很强的前后相关性。</li>
<li>深度学习适用于数据的概率分布是平稳(stationary)的情况，而强化学习中，策略提升(policy improvement)会带来状态分布的变化，有时候在参数BP的过程中，策略参数$\theta$的变化会带来状态分布的很大的变化，从而使训练过程震荡或者diverge。</li>
</ol>
<p>因此上面提到的两篇论文针对上面这三个问题，提出了实际工程实现时的一些办法：<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/10/26/强化学习漫谈-12-DQN-DDPG-A3C/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/10/24/强化学习漫谈-11-DPG/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/10/24/强化学习漫谈-11-DPG/" itemprop="url">强化学习漫谈 11：Deterministic Policy Gradient (DPG)</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-10-24T18:03:52+08:00">
                2018-10-24
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/10/24/强化学习漫谈-11-DPG/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/10/24/强化学习漫谈-11-DPG/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/10/24/强化学习漫谈-11-DPG/" class="leancloud_visitors" data-flag-title="强化学习漫谈 11：Deterministic Policy Gradient (DPG)">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>从随机策略到确定性策略，迈向了高维以致连续action空间的方法。<a href="http://www0.cs.ucl.ac.uk/staff/d.silver/web/Publications_files/deterministic-policy-gradients.pdf" target="_blank" rel="noopener">论文</a></p>
<h3 id="关于确定性策略梯度的疑问"><a href="#关于确定性策略梯度的疑问" class="headerlink" title="关于确定性策略梯度的疑问"></a>关于确定性策略梯度的疑问</h3><p>一个非常直观的猜想是，承接上一节的Policy Gredient方法，策略分布函数$\pi(a|s)$有一个初始分布，例如说正态分布$\mathcal{N}(a,\sigma)$，然后我们通过Actor-Critic或者别的方法对这个策略进行逐渐优化，最终收敛到最优策略$\pi^\ast$，这个过程中策略分布函数应该逐步向最优策略演化，并且最优策略直观上应该是方差很小，聚焦于某个或某段确定的值，极端情况下收敛到确定的点成为确定策略。这种情况下最优策略分布集中的区域其方差通常很小（或者说概率函数很<em>陡峭</em>），因此无论$\nabla_\theta\pi(a|s,\theta)$还是$\nabla_\theta\ln\pi(a|s,\theta)$都很大，造成训练过程不稳定（参考一下狄拉克函数求导？）。因此上一节的Policy Gradient Theorem是不能推广到确定性策略梯度方法的。幸运的是，论文中的Deterministic Policy Gradient Theorem给出了确定策略梯度在理论上的支撑。<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/10/24/强化学习漫谈-11-DPG/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/28/强化学习漫谈-10-Policy-Gradient/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/28/强化学习漫谈-10-Policy-Gradient/" itemprop="url">强化学习漫谈 10：Policy Gradient</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-28T13:47:42+08:00">
                2018-09-28
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/28/强化学习漫谈-10-Policy-Gradient/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/28/强化学习漫谈-10-Policy-Gradient/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/28/强化学习漫谈-10-Policy-Gradient/" class="leancloud_visitors" data-flag-title="强化学习漫谈 10：Policy Gradient">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>Value Function可以用参数化来回归，Policy $\pi(a|s)$ 能否也用参数化回归/拟合？如何通过梯度方法寻找最优策略？相比$\epsilon$-greedy的优点在哪里？</p>
<h3 id="参数化拟合Policy-pi-a-s"><a href="#参数化拟合Policy-pi-a-s" class="headerlink" title="参数化拟合Policy $\pi(a|s)$"></a>参数化拟合Policy $\pi(a|s)$</h3><p>从前两章得到启发，既然值函数$v(s)$或者action value $q(s,a)$可以用函数来拟合，通过调整参数$w$来学习，我们是否也可以将策略$\pi(a|s)$用函数来拟合，通过调整参数$\theta$来寻找最优策略？策略拟合函数表达为$\pi(a|s,\theta)$，例如，对于离散化action，该函数可以用exponential softmax表达为：<br>\begin{equation}<br>\pi(a|s,\theta)=\frac{\exp(h(s,a,\theta))}{\sum_b\exp(h(s,b,\theta))}<br>\end{equation}<br>其中$h(s,a,\theta)$可以通过神经网络学习得到（这样上式其实就是一个多分类器的最后一个softmax FC层），也可以是特征的线性回归$h(s,a,\theta)=\theta^{\mathsf{T}}x(s,a)$。<br>对于连续的action空间，我们可以用一个连续的概率密度函数来表达$\pi(a|s,\theta)$，比如正态分布。<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/28/强化学习漫谈-10-Policy-Gradient/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/" itemprop="url">强化学习漫谈 9：Off Policy Approximation</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-27T19:58:39+08:00">
                2018-09-27
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/" class="leancloud_visitors" data-flag-title="强化学习漫谈 9：Off Policy Approximation">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>从On Policy到Off Policy，不是简单加一个Importance Sampling Ratio这么简单。下面分两个部分：target of the update，以及distribution of the update。</p>
<h3 id="Target-of-Update"><a href="#Target-of-Update" class="headerlink" title="Target of Update"></a>Target of Update</h3><p>延续上一章On-Policy的Semi-Gradient方法，加入importance sampling ratio定义如下：<br>\begin{equation}<br>\rho_{t:n}=\prod_{i=s}^n\frac{\pi(A_i|S_i)}{b(A_i|S_i)}<br>\end{equation}<br>Per-step importance sampling ratio为：<br>\begin{equation}<br>\rho_t=\rho_{t:t}=\frac{\pi(A_t|S_t)}{b(A_t|S_t)}<br>\end{equation}<br>那么Off-Policy的Semi-Gradient自然推论如下：<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/27/强化学习漫谈-9-Off-Policy-Approximation/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/" itemprop="url">强化学习漫谈 8：On-Policy with Approximation</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-24T23:45:32+08:00">
                2018-09-24
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/" class="leancloud_visitors" data-flag-title="强化学习漫谈 8：On-Policy with Approximation">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>离开Tabular方法，进入Approximation的世界。这一章先讲On-Policy部分。</p>
<h3 id="On-Policy-Prediction-With-Approximation"><a href="#On-Policy-Prediction-With-Approximation" class="headerlink" title="On-Policy Prediction With Approximation"></a>On-Policy Prediction With Approximation</h3><p>用回归Regression的方法来近似值函数，这是强化学习中除了Tabular方法之外常见的方法。从状态空间抽象出特征构成特征空间，然后用特征作为输入，运用machine learning的方法进行回归。从machine learning的角度，我们自然会有如下的思路：</p>
<ol>
<li>特征空间比状态空间维度低很多，因此回归过程中参数的改变会改变很多或者所以状态的值函数。如何针对这一情况定义出loss function以确保状态空间中所有状态的值函数收敛到我们期望的目标？</li>
<li>定义出loss function后，针对训练样本，每一步的目标target是什么？</li>
<li>训练方法是什么？
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/24/强化学习漫谈-8-On-Policy-Approximation/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/19/强化学习漫谈-7-剪枝与Planning/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/19/强化学习漫谈-7-剪枝与Planning/" itemprop="url">强化学习漫谈 7：剪枝与Planning</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-19T10:05:29+08:00">
                2018-09-19
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/19/强化学习漫谈-7-剪枝与Planning/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/19/强化学习漫谈-7-剪枝与Planning/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/19/强化学习漫谈-7-剪枝与Planning/" class="leancloud_visitors" data-flag-title="强化学习漫谈 7：剪枝与Planning">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<p>在我们学习了Model后，我们就有了整棵树，那么可以如何通过恰当的剪枝简化我们的训练过程？</p>
<h3 id="Prioritized-Sweeping-and-Model-Learning"><a href="#Prioritized-Sweeping-and-Model-Learning" class="headerlink" title="Prioritized Sweeping and Model Learning"></a>Prioritized Sweeping and Model Learning</h3><p>在Model-free的假设下，无论Monte-Carlo法还是TD都是通过实验持续的估计并逼近value function，这样会有一些缺乏目标性。直观的说，我们应该更关注value变化大的$(s,a)$ pair，以及它的前序pair，因为任何pair的值函数的改变，都会back up到前序pair，这个值变化越大，对前序的状态下值的改变也越大，因此也越值得我们关注。具体的做法是：对于所有的$(s,a)$ pair，我们定义它的状态更新优先级；如果是deterministic policy，那么value的改变就是这个优先级，值改变越大的pair，其前馈back up的优先级越高；如果是stochastic policy，也就是存在$\pi(a|s)$概率，那么要综合考虑pair的值改变大小与这个概率大小；一旦确定了这个前馈back up的优先级，那么就建立一个优先级队列，每次更新值函数的时候，如果这个对应的优先级超过一定阈值，就把它插入到优先级队列的相应位置，然后按照优先级队列进行依次back up。<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/19/强化学习漫谈-7-剪枝与Planning/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/18/强化学习漫谈-6-Bootstrapping/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/18/强化学习漫谈-6-Bootstrapping/" itemprop="url">强化学习漫谈 6：Multi-Step Bootstrapping</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-18T20:13:21+08:00">
                2018-09-18
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/18/强化学习漫谈-6-Bootstrapping/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/18/强化学习漫谈-6-Bootstrapping/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/18/强化学习漫谈-6-Bootstrapping/" class="leancloud_visitors" data-flag-title="强化学习漫谈 6：Multi-Step Bootstrapping">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>


<p>TD与MC为两端，中间的就是Multi-Step Bootstrapping。</p>
<h3 id="基本概念"><a href="#基本概念" class="headerlink" title="基本概念"></a>基本概念</h3><p>上一章的TD是前进一步来做Bootstrapping，而之前的Monte_Carlo是每次走完一个episode后计算一个$G$值。这是在光谱中的两个极端，有没有一个折衷方案？</p>
<p>从另一个角度出发，我们希望决策是灵敏的，也就是每个Action基于的State步长都尽量的短；同时我们希望value的学习是更准确有效的，因此每次更新State Value (或者State Action Pair Value)的Bootstrapping步长都尽量的长。不幸的是，前面提到的方法中，这两个步长是一样的，（Sutton书中将其称为步长的专制 Tyranny of the time step）。因此我们希望有一种方法，区分对待这两个步长。<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/18/强化学习漫谈-6-Bootstrapping/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/09/17/强化学习漫谈-5-TD/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="刘志欣">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="About Learning">
    </span>

    
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">
                
                <a class="post-title-link" href="/2018/09/17/强化学习漫谈-5-TD/" itemprop="url">强化学习漫谈 5：One-Step, Tabular, Model-Free TD</a></h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-09-17T14:32:00+08:00">
                2018-09-17
              </time>
            

          </span>

          
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2018/09/17/强化学习漫谈-5-TD/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/2018/09/17/强化学习漫谈-5-TD/" itemprop="commentCount"></span>
                </a>
              </span>
            
          
             <span id="/2018/09/17/强化学习漫谈-5-TD/" class="leancloud_visitors" data-flag-title="强化学习漫谈 5：One-Step, Tabular, Model-Free TD">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
          <script type="text/x-mathjax-config">
MathJax.Hub.Config({
  TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>

<blockquote class="blockquote-center"><p>They are general methods for learning to make long-term predictions about dynamical systems</p>
</blockquote>
<p>简化，再简化。</p>
<h3 id="Policy-Prediction-Using-TD"><a href="#Policy-Prediction-Using-TD" class="headerlink" title="Policy Prediction Using TD"></a>Policy Prediction Using TD</h3><p>对于在线更新的本质，如果我们希望$x$最终达到$\mathcal{X}$, 那么每一轮的在线更新是：$x_{k+1}=x_k+\alpha(\mathcal{X}-x_k)$，其中$\alpha$为learning rate。对应到上一章的Monte-Carlo法，我们有在线更新的公式（上一章公式5）：<br>\begin{equation}<br>V_{n+1}=V_n+\frac{W_n}{C_n}[G_n-V_n]\notag<br>\end{equation}<br>或者写成update的方式：<br>\begin{equation}<br>V(S_t)\leftarrow V(S_t)+\alpha[G_t-V(S_t)]\notag<br>\end{equation}<br>说明我们希望$V(S_t)$向$G_t$的目标逐步靠近。为什么要以$G_t$为目标呢？<br>
          <!--noindex-->
          <div class="post-button text-center">
            <a class="btn" href="/2018/09/17/强化学习漫谈-5-TD/#more" rel="contents">
              阅读全文 &raquo;
            </a>
          </div>
          <!--/noindex-->
        
      
    </div>
    
    
    <footer class="post-footer">
      

        <div class="post-eof"></div>
      
    </footer>
  </div>
  
  
  </article>


  </section>

  
  <nav class="pagination">
    <span class="page-number current">1</span><a class="page-number" href="/page/2/">2</a><a class="extend next" rel="next" href="/page/2/"><i class="fa fa-angle-right"></i></a>
  </nav>


          </div>
          

        </div>
        
          
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      
      <section class="site-overview-wrap sidebar-panel sidebar-panel-active">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <p class="site-author-name" itemprop="name">刘志欣</p>
              <p class="site-description motion-element" itemprop="description"></p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/archives/">
              
                  <span class="site-state-item-count">16</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

              <div class="site-state-item site-state-tags">
                <a href="/tags/index.html">
                  <span class="site-state-item-count">1</span>
                  <span class="site-state-item-name">标签</span>
                </a>
              </div>
            

          </nav>

          
        </div>
      </section>

      
    </div>
  </aside>


      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">刘志欣</span>

  
</div>


  <div class="powered-by">由 <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a> 强力驱动</div>


  <span class="post-meta-divider">|</span>


  <div class="theme-info">主题 &mdash; <a class="theme-link" target="_blank" href="https://github.com/iissnan/hexo-theme-next">NexT.Gemini</a> v5.1.4</div>


      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  
<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>


    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

    <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

    <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

    <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  

  <script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script>


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.4"></script>


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script>


  <script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
  <script src="//unpkg.com/valine/dist/Valine.min.js"></script>
  
  <script type="text/javascript">
    var GUEST = ['nick','mail','link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item=>{
      return GUEST.indexOf(item)>-1;
    });
    new Valine({
        el: '#comments' ,
        verify: false,
        notify: false,
        appId: 'iC82lD4bRxRuaSG490Jjc8yr-gzGzoHsz',
        appKey: 'MfUVQHRcRFwpnRUTSXCAMjiu',
        placeholder: 'Say Something',
        avatar:'mm',
        guest_info:guest,
        pageSize:'10' || 10,
    });
  </script>


  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.4.js"></script>
  <script>AV.initialize("iC82lD4bRxRuaSG490Jjc8yr-gzGzoHsz", "MfUVQHRcRFwpnRUTSXCAMjiu");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>


    <script type="text/x-mathjax-config">
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
          processEscapes: true,
          skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
        }
      });
    </script>

    <script type="text/x-mathjax-config">
      MathJax.Hub.Queue(function() {
        var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
      });
    </script>
    <script type="text/javascript" src="//cdn.bootcss.com/mathjax/2.7.1/latest.js?config=TeX-AMS-MML_HTMLorMML"></script><!-- hexo-inject:begin --><!-- Begin: Injected MathJax -->
<script type="text/x-mathjax-config">
  MathJax.Hub.Config("");
</script>

<script type="text/x-mathjax-config">
  MathJax.Hub.Queue(function() {
    var all = MathJax.Hub.getAllJax(), i;
    for(i=0; i < all.length; i += 1) {
      all[i].SourceElement().parentNode.className += ' has-jax';
    }
  });
</script>

<script type="text/javascript" src="custom_mathjax_source">
</script>
<!-- End: Injected MathJax -->
<!-- hexo-inject:end -->
  

</body>
</html>