mickeysjm / synsetmine-pytorch Goto Github PK

class SynsetDataset(Dataset):

    def __init__(self,
                 ....
                 ):
        

    def __len__(self):
        return len(self.train_labels)

    ...

    def __getitem__(self, index):
        sets = self.train_sets[index]
        # word_sets = np.zeros(self.max_set_lengths)
        # word_sets[:len(sets)] = sets
        word_sets = sets
        word_inst = self.train_insts[index]
        label = self.train_labels[index]
        return torch.tensor(word_sets).long(), torch.tensor(word_inst).long(), torch.tensor(label).long()

    def load_word_synset(self, synset_path):
        """
        读取文本同义词集合,c

        :synset_path: the path of synset
        :type synset_path: str
        :return :None
        :rtype: None
        """
        text = read_synset(synset_path)
        random.shuffle(text)
        for line in tqdm(text, desc='loading word synset...'):
            words = sorted([self.word2index[word]for word in line])
            self.positive_sets.append(words)
            self.vocab.extend(words)

        self.vocab = sorted(self.vocab)
        # for sets in self.positive_sets:
        #     self.vocab.extend(sets)


    def sample_pos_neg_data(self, raw_sets, max_length, neg_sample_size):
        """
        same with synsetmine
        """
        sip_triplets = []
        pos_sip_cnt_sum = 0
        neg_sip_cnt_sum = 0

        sample_sets = []
        sample_insts = []
        sample_labels = []

        for subset_size in range(1, max_length + 1):
            for raw_set in raw_sets:  # 每一个都是同义词集
                if len(raw_set) < subset_size:
                    continue
                raw_set_new = raw_set.copy()
                random.shuffle(raw_set_new)
                batch_set = []
                batch_pos = []
                if len(raw_set) == subset_size:  # put the entire full set
                    for _ in range(neg_sample_size + 1):
                        batch_set.append(raw_set)
                        batch_pos.append(random.choice(raw_set))
                else:
                    for _ in range(neg_sample_size + 1):
                        '''
                        subset_size = 1
                        raw_set_new = [a,b,c,d]

                        '''
                        for start_idx in range(0, len(raw_set_new) - subset_size, subset_size + 1):
                            # slide window= subset_size
                            subset = raw_set_new[start_idx:start_idx + subset_size]
                            # 取subset后一个词
                            pos_inst = raw_set_new[start_idx + subset_size]
                            batch_set.append(subset)
                            batch_pos.append(pos_inst)
                        random.shuffle(raw_set_new)

                pos_sip_cnt = int(len(batch_set) / (neg_sample_size + 1))
                pos_sip_cnt_sum += pos_sip_cnt
                neg_sip_cnt = int(pos_sip_cnt * neg_sample_size)
                neg_sip_cnt_sum += neg_sip_cnt

                negative_pool = [
                    ele for ele in self.vocab if ele not in raw_set]
                sample_size = math.gcd(
                    neg_sip_cnt, len(negative_pool))  # 最大公约数
                sample_times = int(neg_sip_cnt / sample_size)

                batch_neg = []
                for _ in range(sample_times):
                    batch_neg.extend(random.sample(
                        negative_pool, sample_size))
                    # 每次抽出相同大小的neg
                for idx, subset in enumerate(batch_set):

    
                    if idx < pos_sip_cnt:
                        pos = batch_pos[idx]

                        # 设置随机
                        # random.shuffle(subset)
                        sample_sets.append(subset)
                        sample_insts.append(pos)
                        sample_labels.append(1)
                        sip_triplets.append((subset, pos, 1))
                    else:
                        neg = batch_neg[idx - pos_sip_cnt]
                        # random.shuffle(subset)
                        sip_triplets.append((subset, neg, 0))
                        sample_sets.append(subset)
                        sample_insts.append(neg)
                        sample_labels.append(0)
                    

        print(len(sip_triplets))
        return sample_sets, sample_insts, sample_labels

  
    def generate_data(self,sample_iter=15,strategy='paper'):
 
        print(len(self.positive_sets))
        for i in range(sample_iter):
            if strategy=='paper':
                # paper strategy
                # best performance 0.27
                sample_sets, sample_insts, sample_labels = self.sample_pos_neg_data(
                    self.positive_sets, self.max_set_lengths, self.neg_sample_size)

            self.train_sets.extend(sample_sets)
            self.train_insts.extend(sample_insts)
            self.train_labels.extend(sample_labels)
        return self.train_sets, self.train_insts, self.train_labels