I was studying the transformer decoder code below in Keras/Tensorflow. It was not clear how they made making decisions.
In the first attention block below (self.attention_1), why did they use causal_mask instead of padding_mask in the call method? This first attention block processes the target sequence which is likely to contain padding data. Does it not worth preventing processing of irrelevant information(i.e the padding data) in the target sequence? i.e why did they use this:
attention_output_1 = self.attention_1(
query=inputs,
value=inputs,
key=inputs,
attention_mask=causal_mask)
instead of:
attention_output_1 = self.attention_1(
query=inputs,
value=inputs,
key=inputs,
attention_mask=padding_mask)
class TransformerDecoder(layers.Layer):
def __init__(self,embed_dim,dense_dim,num_heads,**kwargs):
super().__init__(**kwargs);
self.embed_dim=embed_dim;
self.dense_dim=dense_dim;
self.num_heads=num_heads;
self.attention_1=layers.MultiHeadAttention(
num_heads=num_heads,key_dim=embed_dim
);
self.attention_2=layers.MultiHeadAttention(
num_heads=num_heads,key_dim=embed_dim
);
self.dense_proj=keras.Sequential([
layers.Dense(dense_dim,activation="relu"),
layers.Dense(embed_dim)]);
self.layernorm_1=layers.LayerNormalization();
self.layernorm_2=layers.LayerNormalization();
self.layernorm_3=layers.LayerNormalization();
self.supports_masking=True;
def get_config(self):
config=super().get_config();
config.update({
"embed_dim":self.embed_dim,
"dense_dim":self.dense_dim,
"num_heads":self.num_heads
});
return config;
# Listing 11.34 Transformeer decoder method that generates causal mask
def get_causal_attention_mask(self,inputs):
input_shape=tf.shape(inputs);
batch_size,sequence_length=input_shape[0],input_shape[1];
i=tf.range(sequence_length)[:,tf.newaxis];
j=tf.range(sequence_length);
mask=tf.cast(i>=j,dtype="int32");
mask=tf.reshape(mask,(1,input_shape[1],input_shape[1]));
mult=tf.concat(
[tf.expand_dims(batch_size, axis=-1),
tf.constant([1,1],dtype=tf.int32)],axis=0);
return tf.tile(mask,mult);
#Listing 11.35 The foraward pass of the TransformerDecoder
def call(self,inputs,encoder_outputs,mask=None):
causal_mask=self.get_causal_attention_mask(inputs);
if mask is not None:
padding_mask=tf.cast(
mask[:,tf.newaxis,:],dtype="int32");
padding_mask=tf.minimum(padding_mask,causal_mask);
attention_output_1=self.attention_1(
query=inputs,
value=inputs,
key=inputs,
attention_mask=causal_mask);#TODO: why pass causal_mask instead of padding_mask here?
attention_output_1=self.layernorm_1(inputs+attention_output_1)
attention_output_2=self.attention_2(
query=attention_output_1,
value=encoder_outputs,
key=encoder_outputs,
attention_mask=padding_mask);
attention_output_2=self.layernorm_2(
attention_output_1+attention_output_2);
proj_output=self.dense_proj(attention_output_2);
return self.layernorm_3(attention_output_2 + proj_output);
```
