def review_to_wordlist(review, remove_stopwords = False): # 1. HTML review_test = BeautifulSoup(review).get_text() # 2. non-letteers review_only = re.sub("[^a-zA-Z]", " ", review_test) # 3. lower and split words review_lower = review_only.lower().split() # 4. Word2vec no need to delete stopword and number if remove_stopwords: stops = set(stopwords.words("english")) words = [w for w in review_lower if not w in stops] # 5. return words list return (review_lower) tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') def review_to_sentences(review, tokenizer, remove_stopwords = False): # 1. 段落->n个句子 raw_sentences = tokenizer.tokenize(review.strip()) # 2. 句子->单词 sentences = [] for raw_sentence in raw_sentences: # If a sentence is empty, skip it if len(raw_sentence) > 0: # Otherwise, call review_to_wordlist to get a list of words sentences.append( review_to_wordlist( raw_sentence, \ remove_stopwords )) return sentences
结束后的结果应该为list中，每个单词为单独元素 3. 处理数据
1 2 3 4 5
sentences = [] for review in train["review"]: sentences += review_to_sentences(review, tokenizer) for review in unlabeled_train["review"]: sentences += review_to_sentences(review, tokenizer)
