pytorch中的多种scheduler_pytorch 多个scheduler

pytorch中的多种scheduler

1. LambdaLR

1. 公式
   $$l{r}_{epoch}=l{r}_{initial}\ast Lambda(epoch)$$

2. 代码

   import torch
   import matplotlib.pyplot as plt
   
   model = torch.nn.Linear(2, 1)
   optimizer = torch.optim.SGD(model.parameters(), lr=100)
   lambda1 = lambda epoch: 0.65 ** epoch
   scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1)
   
   
   lrs = []
   
   for i in range(10):
       optimizer.step()
       lrs.append(optimizer.param_groups[0]["lr"])
   #     print("Factor = ", round(0.65 ** i,3)," , Learning Rate = ",round(optimizer.param_groups[0]["lr"],3))
       scheduler.step()
   
   plt.plot(range(10),lrs)
   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12
   13
   14
   15
   16
   17
   18

3. 图片

   

2. MultiplicativeLR

1. 公式
   $$l{r}_{epoch}=l{r}_{epoch-1}\ast Lambda(epoch)$$

2. 图片

   

3. StepLR

1. 公式

   

2. 图片

   

4. MultiStepLR

1. 公式

   

2. 代码

   model = torch.nn.Linear(2, 1)
   optimizer = torch.optim.SGD(model.parameters(), lr=100)
   scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[6,8,9], gamma=0.1)
   lrs = []
   
   for i in range(10):
       optimizer.step()
       lrs.append(optimizer.param_groups[0]["lr"])
   #     print("Factor = ",0.1 if i in [6,8,9] else 1," , Learning Rate = ",optimizer.param_groups[0]["lr"])
       scheduler.step()
   
   plt.plot(range(10),lrs)
   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12

3. 图片

   

5. CosineAnnealingLR

1. 公式
   $${\eta }_t={\eta }_{min}+\frac{1}{2}({\eta }_{max}-{\eta }_{min})(1+cos(\frac{T_{cur}}{T_{max}}\pi ))$$

2. 代码

   model = torch.nn.Linear(2, 1)
   optimizer = torch.optim.SGD(model.parameters(), lr=100)
   scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)#T_max表示半个周期的大小
   lrs = []
   
   
   for i in range(100):
       optimizer.step()
       lrs.append(optimizer.param_groups[0]["lr"])
   #     print("Factor = ",i," , Learning Rate = ",optimizer.param_groups[0]["lr"])
       scheduler.step()
   
   plt.plot(lrs)
   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12
   13

3. 图片

   

6. CosineAnnealingWarmRestarts

1. 公式
   $${\eta }_t={\eta }_{min}+\frac{1}{2}({\eta }_{max}-{\eta }_{min})(1+cos(\frac{T_{cur}}{T_i}\pi ))$$

2. 代码

      import torch
      import matplotlib.pyplot as plt
      
      model = torch.nn.Linear(2, 1)
      optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
      lr_sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=0.001, last_epoch=-1)
      #每经过T_0轮后，开始重新启动scheduler
      
      lrs = []
      
      for i in range(100):
          lr_sched.step()
          lrs.append(
              optimizer.param_groups[0]["lr"]
          )
      
      plt.plot(lrs)
   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12
   13
   14
   15
   16
   17