bert细节适配：添加词表之外的单词和标点符号的处理细节
由于bert中主要为中文，所以词表中英文单词比较少，但是一般英文单词如果简单的直接使用tokenize函数，往往在一些序列预测问题上存在一些对齐问题，或者很多不存在的单词或符号没办法处理而直接使用 unk 替换了，某些英文单词或符号失去了单词的预训练效果，所以采用以下一种更缓和的方式，来进行ＢＥＲＴ的适配，可以提高模型在中英文文本下，预训练模型的效果
通过重写Tokenize类 ①处理vocab中不存在的标点符号，使用替代方式 ②不存在的单词，正向匹配 ##后缀 的词，一定程度上有接近语义或词性
不过segment_id需要自己计算，一般单句就是全部为0的列表了，即 segment_ids = [0] * len(token_ids)
# coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections import re import unicodedata import six import tensorflow as tf def load_vocab(vocab_file): """Loads a vocabulary file into a dictionary.""" vocab = collections.OrderedDict() index = 0 with tf.gfile.GFile(vocab_file, "r") as reader: for token in reader.readlines(): if not token: break token = token.strip() vocab[token] = index index += 1 return vocab def convert_by_vocab(vocab, items): """Converts a sequence of [tokens|ids] using the vocab.""" output = [] for item in items: output.append(vocab[item]) return output def convert_tokens_to_ids(vocab, tokens): return convert_by_vocab(vocab, tokens) def convert_ids_to_tokens(inv_vocab, ids): return convert_by_vocab(inv_vocab, ids) def whitespace_tokenize(text): """Runs basic whitespace cleaning and splitting on a piece of text.""" text = text.strip() if not text: return [] tokens = text.split() return tokens class BasicTokenizer(object): """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200, do_lower_case=True): """Constructs a BasicTokenizer. Args: do_lower_case: Whether to lower case the input. """ self.do_lower_case = do_lower_case self.vocab = vocab self.unk_token = unk_token self.max_input_chars_per_word = max_input_chars_per_word def tokenize(self, text): """Tokenizes a piece of text.""" text = self._clean_text(text) # This was added on November 1st, 2018 for the multilingual and Chinese # models. This is also applied to the English models now, but it doesn't # matter since the English models were not trained on any Chinese data # and generally don't have any Chinese data in them (there are Chinese # characters in the vocabulary because Wikipedia does have some Chinese # words in the English Wikipedia.). text = self._tokenize_chinese_chars(text) orig_tokens = whitespace_tokenize(text) split_tokens = [] for token in orig_tokens: if self.do_lower_case: token = token.lower() token = self._run_strip_accents(token) split_tokens.extend(self._run_split_on_punc(token)) output_tokens = whitespace_tokenize(" ".join(split_tokens)) return output_tokens # 处理不在词表中的词 def wordpiecetoken(self, tokens): output_tokens = [] for token in tokens: if token in self.vocab: output_tokens.append(token) continue chars = list(token) # 如果超出长度，则用ｕｎｋ if len(chars) > self.max_input_chars_per_word: output_tokens.append(self.unk_token) continue is_bad = False start = 0 sub_tokens = [] end = len(chars) while end > 1: cur_substr = None while start < end: substr = "".join(chars[start:end]) if start > 0: substr = "##" + substr if substr in self.vocab: cur_substr = substr break # end -= 1 start += 1 if cur_substr is not None: sub_tokens.append(cur_substr) break end -= 1 if cur_substr is None: is_bad = True if is_bad: output_tokens.append(self.unk_token) else: output_tokens.extend(sub_tokens) return output_tokens # 处理vocab中不存在的标点符号，使用替代方式 def deal_punctuation(self, c): r = None if c == '“': r = '"' elif c == '”': r = '"' elif c == '“': r = '"' elif c == '“': r = '"' elif c == '—': r = '-' elif c == '…': r = '...' elif c == '……': r = '...' else: r = c return r def deal_punctuations(self, tokens): R = [] for token in tokens: token = self.deal_punctuation(token) R.append(token) return R def encode(self, text, add_cls_sep=True): tokens = self.tokenize(text) # print(len(tokens)) # print(tokens) tokens = self.deal_punctuations(tokens) wordpiecetokens = self.wordpiecetoken(tokens) print(wordpiecetokens) token_ids = convert_tokens_to_ids(self.vocab, wordpiecetokens) if add_cls_sep: token_ids.insert(0,self.vocab['[CLS]']) token_ids.append(self.vocab['[SEP]']) return token_ids def _run_strip_accents(self, text): """Strips accents from a piece of text.""" text = unicodedata.normalize("NFD", text) output = [] for char in text: cat = unicodedata.category(char) if cat == "Mn": continue output.append(char) return "".join(output) def _run_split_on_punc(self, text): """Splits punctuation on a piece of text.""" chars = list(text) i = 0 start_new_word = True output = [] while i < len(chars): char = chars[i] if _is_punctuation(char): output.append([char]) start_new_word = True else: if start_new_word: output.append([]) start_new_word = False output[-1].append(char) i += 1 return ["".join(x) for x in output] def _tokenize_chinese_chars(self, text): """Adds whitespace around any CJK character.""" output = [] for char in text: cp = ord(char) if self._is_chinese_char(cp): output.append(" ") output.append(char) output.append(" ") else: output.append(char) return "".join(output) def _is_chinese_char(self, cp): """Checks whether CP is the codepoint of a CJK character.""" # This defines a "chinese character" as anything in the CJK Unicode block: # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) # # Note that the CJK Unicode block is NOT all Japanese and Korean characters, # despite its name. The modern Korean Hangul alphabet is a different block, # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. if ((cp >= 0x4E00 and cp <= 0x9FFF) or # (cp >= 0x3400 and cp <= 0x4DBF) or # (cp >= 0x20000 and cp <= 0x2A6DF) or # (cp >= 0x2A700 and cp <= 0x2B73F) or # (cp >= 0x2B740 and cp <= 0x2B81F) or # (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) or # (cp >= 0x2F800 and cp <= 0x2FA1F)): # return True return False def _clean_text(self, text): """Performs invalid character removal and whitespace cleanup on text.""" output = [] for char in text: cp = ord(char) if cp == 0 or cp == 0xfffd or _is_control(char): continue if _is_whitespace(char): output.append(" ") else: output.append(char) return "".join(output) def _is_whitespace(char): """Checks whether `chars` is a whitespace character.""" # \t, \n, and \r are technically contorl characters but we treat them # as whitespace since they are generally considered as such. if char == " " or char == "\t" or char == "\n" or char == "\r": return True cat = unicodedata.category(char) if cat == "Zs": return True return False def _is_control(char): """Checks whether `chars` is a control character.""" # These are technically control characters but we count them as whitespace # characters. if char == "\t" or char == "\n" or char == "\r": return False cat = unicodedata.category(char) if cat in ("Cc", "Cf"): return True return False def _is_punctuation(char): """Checks whether `chars` is a punctuation character.""" cp = ord(char) # We treat all non-letter/number ASCII as punctuation. # Characters such as "^", "$", and "`" are not in the Unicode # Punctuation class but we treat them as punctuation anyways, for # consistency. if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): return True cat = unicodedata.category(char) if cat.startswith("P"): return True return False def main(): import os here = os.path.dirname(os.path.abspath(__file__)) vocab = load_vocab(os.path.join(here, 'chinese_L-12_H-768_A-12','vocab.txt')) bsc = BasicTokenizer(vocab=vocab) s = bsc.encode('今天和ｂｏyfriend出去散步,!１我们心情很那NIce人也很BEAtiFUlly…对吗') # print('lens', len(s)) print(s) # ['今', '天', '和', '##end', '出', '去', '散', '步', ',', '!', '１', '我', '们', '心', '情', '很', '那', 'nice', '人', '也', '很', '##lly', '...', '对', '吗'] #[101, 791, 1921, 1469, 11652, 1139, 1343, 3141, 3635, 117, 106, 8029, 2769, 812, 2552, 2658, 2523, 6929, 10192, 782, 738, 2523, 9456, 8106, 2190, 1408, 102] if __name__ == "__main__": main()