def _get_lstm_feature(self, sent, mask):
sent_embed = self.word_lookup(sent)
sent_embed = fun.dropout(sent_embed, self.dropout)
lstm_ip, id_unsort = self.get_pack_inputs(sent_embed, mask.sum(1))
lstm_out, _ = self.lstm(lstm_ip)
lstm_out = self.get_pad_outputs(lstm_out, sent_embed.size(1), id_unsort)
lstm_out = fun.relu(lstm_out)
hidden = self.hidden_linear(lstm_out)
hidden = fun.relu(fun.dropout(hidden, self.dropout))
feats = self.tag_linear(hidden)
return feats
@staticmethod
def log_sum_exp(x, axis):
x_max, _ = torch.max(x, axis, keepdim=True)
x_max_expand = x_max.expand(x.size())
return x_max + torch.log(torch.sum(torch.exp(x - x_max_expand), axis, keepdim=True))
def forward(self, emits, masks):
batch_size, step_num, tag_size = emits.size()
lengths = masks.sum(1).tolist()
_mask = torch.zeros(masks.size()).to(self.device)
for b in range(batch_size):
_mask[b][lengths[b]-1] = 1
_mask = _mask.byte()
_mask = _mask.view(batch_size, step_num, 1).expand_as(emits)
batch_trans = torch.cat([self.transitions for _ in range(batch_size)], 0).contiguous()
batch_trans = batch_trans.view(batch_size, tag_size, tag_size)
forward_var = torch.cat([self.alpha_0 for _ in range(batch_size)], 0).contiguous()
forward_var = forward_var.view(batch_size, tag_size, 1)
forward_var = forward_var + emits[:, 0, :].view(batch_size, tag_size, 1)
alpha = [forward_var]
max_scores = [torch.squeeze(forward_var)]
max_scores_pre = []
for t in range(1, step_num):
forward_var = forward_var.view(batch_size, tag_size, 1).expand(batch_size, tag_size, tag_size)
current = emits[:, t, :].view(batch_size, 1, tag_size).contiguous()
# alpha_t[i, j] = pre_score[j](col) + score(j->i) + score[i](row)
alpha_t = forward_var + current.expand(batch_size, tag_size, tag_size) + batch_trans
# cur_max_score score[i, :], cur_max_idx is j
cur_max_score, cur_max_idx = torch.max(alpha_t, 1, keepdim=True)
max_scores.append(torch.squeeze(cur_max_score, 1))
max_scores_pre.append(torch.squeeze(cur_max_idx, 1))
log_alpha_t = self.log_sum_exp(alpha_t, 1).view(batch_size, tag_size, 1)
forward_var = log_alpha_t
alpha.append(log_alpha_t)
alphas = torch.cat(alpha, 0).view(batch_size, step_num, tag_size)
last_alphas = torch.masked_select(alphas, _mask).view(batch_size, tag_size, 1)
# forward var max value is add.....
alpha_z = torch.sum(self.log_sum_exp(last_alphas, 1))
return alpha_z, max_scores, max_scores_pre
def score_path(self, emits, tags, mask):
sent_len = mask.sum(1).tolist()
batch_size, step_num = tags.size()
scores = torch.FloatTensor([0]).to(self.device)
for b in range(batch_size):
cur_tag = tags[b][0].item()
scores += self.alpha_0[cur_tag][0] + emits[b][0][cur_tag]
for step in range(1, step_num):
pre_tag = cur_tag
cur_tag = tags[b][step].item()
if step < sent_len[b]:
scores += (self.transitions[pre_tag][cur_tag] + emits[b][step][cur_tag])
else:
break
return scores
@staticmethod
def viterbi(max_scores, max_score_pre, mask):
sent_lenth = mask.sum(1).tolist()
best_paths = []
batch_size = mask.size(0)
for b in range(batch_size):
cur_path = []
_, last_max_node = torch.max(max_scores[sent_lenth[b]-1][b], 0, keepdim=True)
last_max_node = last_max_node.item()
cur_path.append(last_max_node)
for t in range(sent_lenth[b]-2, -1, -1):
last_max_node = max_score_pre[t][b][last_max_node]
last_max_node = last_max_node.item()
cur_path.append(last_max_node)
cur_path = cur_path[::-1]
best_paths.append(cur_path)
return best_paths
def get_arg(self, inps):
sent, mask, _ = inps
feats = self._get_lstm_feature(sent, mask)
_, max_scores, max_scores_pre = self.forward(feats, mask)
best_paths = self.viterbi(max_scores, max_scores_pre, mask)
return best_paths
def get_loss(self, inps):
sent, mask, args = inps
feats = self._get_lstm_feature(sent, mask)
forward_score, _, _ = self.forward(feats, mask)
gold_score = self.score_path(feats, args, mask)
return forward_score - gold_score
train---epoch: 5, learn rate: 0.001000, global step: 1009
loss: -382.26562500
macro arg---P: 0.560069, R: 0.111111, F: 0.079778
---------------------------------------
train---epoch: 5, learn rate: 0.001000, global step: 1010
loss: -1400.46484375
macro arg---P: 0.559974, R: 0.111111, F: 0.079770
---------------------------------------
train---epoch: 5, learn rate: 0.001000, global step: 1011
loss: 1773.91503906
macro arg---P: 0.559594, R: 0.111111, F: 0.079735
---------------------------------------
train---epoch: 5, learn rate: 0.001000, global step: 1012
loss: 807.89257812
macro arg---P: 0.559650, R: 0.111111, F: 0.079740
---------------------------------------
train---epoch: 5, learn rate: 0.001000, global step: 1013
loss: 1946.49560547
macro arg---P: 0.559642, R: 0.111111, F: 0.079739
---------------------------------------
train---epoch: 5, learn rate: 0.001000, global step: 1014
loss: -3450.06152344
macro arg---P: 0.559863, R: 0.111111, F: 0.079760
---------------------------------------