transformer_多头注意力机制代码笔记

transformer_多头注意力机制代码笔记

以GPT-2中多头注意力机制代码为例

class CausalSelfAttention(nn.Module):
    """
    因果掩码+多头自注意力机制
    
    A vanilla multi-head masked self-attention layer with a projection at the end.
    It is possible to use torch.nn.MultiheadAttention here but I am including an
    explicit implementation here to show that there is nothing too scary here.
    """

    def __init__(self, config):
        super().__init__()
        assert config.n_embd % config.n_head == 0
        # key, query, value projections for all heads, but in a batch
        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)  # Q/K/V的线性映射
        # output projection
        self.c_proj = nn.Linear(config.n_embd, config.n_embd)  # 输出线性映射
        # regularization
        self.attn_dropout = nn.Dropout(config.attn_pdrop)  # dropout正则化
        self.resid_dropout = nn.Dropout(config.resid_pdrop)
        # causal mask to ensure that attention is only applied to the left in the input sequence
        self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
                                     .view(1, 1, config.block_size, config.block_size))  # 构建casual mask矩阵
        self.n_head = config.n_head
        self.n_embd = config.n_embd

    def forward(self, x):
        B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)

        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
        q, k ,v  = self.c_attn(x).split(self.n_embd, dim=2)  # 同时计算QKV
        
        # 多头切分, nh*hs=C
        # 一定要先view (B, T, nh, hs), 再transpose，不能直接view到(B, nh, T, hs)  
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)  
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)

        # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)
        # 除以根号下dk保持方差稳定
        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
        # 因果掩码
        att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))
        att = F.softmax(att, dim=-1)
        att = self.attn_dropout(att)
        y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
        # contiguous:张量在底层存储时必须连续才可以使用view
        y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side

        # output projection
        y = self.resid_dropout(self.c_proj(y))
        return y
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

以下为对多头注意力机制代码做分步笔记
上述代码中初始化中定义了构建多头注意力机制代码的组件（结构），在forward的方法中将使用初始化中的组件构建多头注意力机制。
从forward方法开始阅读，当使用到初始化方法中的代码时再进行阅读

# x是输入的数据
# B是批大小
# T是序列长度
# C是embedding的维度
B, T, C = x.size()
1
2
3
4
5

# 通过self.c_atten初始化q，k，v
q, k ,v  = self.c_attn(x).split(self.n_embd, dim=2)




# 初始化q，k，v
# 验证数据维度与多头的数量是否一致
assert config.n_embd % config.n_head == 0
# 因为q，k，v是相同的大小的不同矩阵，通过线性映射获得初始化的q，k，v
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
1
2
3
4
5
6
7
8
9
10
11

# 多头切分, nh*hs=C，其中nh为头的个数，hs为头的维度
# 一定要先view (B, T, nh, hs), 再transpose，不能直接view到(B, nh, T, hs)  
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)  
q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)



# 其中self.n_head是配置文件中预先设置好的
self.n_head = config.n_head
# transpose是为了转换不同维度上的数据
1
2
3
4
5
6
7
8
9
10
11

# 此处是attention的公式
# causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)
# 除以根号下dk保持方差稳定
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
1
2
3
4

# 因果掩码
att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))
att = F.softmax(att, dim=-1)
att = self.attn_dropout(att)
y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
1
2
3
4
5

# contiguous:张量在底层存储时必须连续才可以使用view
y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side

# 前面讲C拆解为两个矩阵相乘，现在讲得到的结果返回最初的数据形状
# contiguous()是深拷贝
1
2
3
4
5

# output projection
y = self.resid_dropout(self.c_proj(y))



# output projection
self.c_proj = nn.Linear(config.n_embd, config.n_embd)  # 输出线性映射
# regularization
self.attn_dropout = nn.Dropout(config.attn_pdrop)  # dropout正则化
self.resid_dropout = nn.Dropout(config.resid_pdrop)
1
2
3
4
5
6
7
8
9
10