正文共10052个字，4张图，预计阅读时间26分钟。

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
import re
import time
from bs4 import BeautifulSoup
req_header = {  
 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  
 'Accept-Language':'zh - CN, zh;q = 0.8, en;q = 0.6',    
'Connection':'keep - alive',    
'Cookie':'_hc.v = "\"4e10f82f-cdb5-4fa1-ba89-262a394be3d1.1490604667\""; PHOENIX_ID = 0a01084a - 15b1299fd35 - 163c9ff; s_ViewType = 10; JSESSIONID = F255BB7A08A17AFC8F2E3701599B3193; aburl = 1; cy = 6; cye = suzhou',  
 'Upgrade-Insecure-Requests':'1',  
 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}
 history = {} #记录链接是否已经爬取过，防止重定向
for subjectid in range(1,220000):    
pageid = 1    
cur_url = "http://bgm.tv/subject/" + str(subjectid)+ "/comments"    
while(True):        

mark = 0        
cur_url = cur_url + "?page=" + str(pageid)        
if cur_url in history: # 是否爬取过            
break        
else:            
history[cur_url
 ] = 1      
 try:            
r = requests.get(cur_url, headers=req_header, timeout=10)        
except:            
print(subjectid)            
time.sleep(5)          
 try:                
r = requests.get(cur_url, headers=req_header, timeout=10)            
except:                
break
#爬两次还爬不到就放弃换下一个        
content = BeautifulSoup(r.content, 'lxml')        
base_url_index = r.url.find("?")      
 cur_url = r.url[0:base_url_index]        
try:           

 title = content.find("a",href=re.compile(r"/subject*")).text        
except:            
break      
 if title == "": #爬取到的页面有问题，比如404            
break      
 # 因为把每个词条都作为文件名，所以有些特殊字符不能作为文件名      
 title = title.replace("/","-").replace(":","-").replace("\""," ").replace("<", "(").replace(">", ")").replace("?","-").replace("*","-").replace("\\","-").replace("|","-").replace("\n","-")              
 # 把爬取到的条目下的评论都拿出来放到条目文件里      
 with open("directory/"+title, "a", encoding="utf-8")as f:          
 for item in content.find("div", {"id": "comment_box"}).find_all("div", {"class":"item clearit"}):                item = item.find("div",{"class": re.compile(r"text_main_*")})              
 try:                    
print(item.find("span")["class"])                
except:                  
break                
f.write(item.find("span")["class"][0] + '\n')                
f.write(item.text.strip())                
mark = 1              
 f.write("\n===========================\n")      
 pageid += 1        
if mark == 0:            
break

   time.sleep(0.3) # 睡一会，别爬太快

#!/usr/bin/env python # -*- coding: utf-8 -*- import osimport re out = open("bangumi_sentiment", "w", encoding="utf-8")for cur_file in os.listdir("directory"):     with open("directory/"+cur_file, "r", encoding="utf-8") as f:        comments = f.read().split("===========================")         for comment in comments:            if comment == "":                 continue            start_index = comment.find("sstars")            score = comment[start_index+6:8].strip()             try:                #找到日期末尾位置                content_index = re.search("\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}  ",comment).end() except:                 continue out.write(score+" ") out.write(" ".join(comment[content_index:].split("\n"))+"\n")

#!/usr/bin/env python# -*- coding: utf-8 -*- import logging import multiprocessing import sysfrom gensim.models import Word2Vec from gensim.models.word2vec import LineSentencedef train(inp, model_name, size=100, word_frequency_threshold=5):    logging.basicConfig(format = '%(asctime)s : %(levelname)s : %(message)s', level = logging.INFO)    print("Begin to train model ...")    model = Word2Vec(LineSentence(inp),    size = size,    window = 10,    hs = 1,    sg = 1,  # skip-gram    min_count = word_frequency_threshold,    workers = multiprocessing.cpu_count()/2)  #CPU数量    model.save_word2vec_format(model_name, binary=False) # save a model in order to be checkedif __name__ == "__main__":    if len(sys.argv) <3:        print("please input input file and model file")    inp = sys.argv[1]    model = sys.argv[2]    word_size = 100    word_threshold = 3    if len(sys.argv) > 3:        word_size = int(sys.argv[3]) #词向量维度        word_threshold = int(sys.argv[4]) #每个字最少出现的次数    train(inp, model, word_size, word_threshold)

# 设置placeholderword_ids = tf.placeholder(tf.int32, shape=[None, None],name="word_ids")  # batch size, max length of sentence in batchsequence_lengths = tf.placeholder(tf.int32, shape=[None], name="sequence_length")   # shape = batch sizelabels = tf.placeholder(tf.int32, shape=[None], name="labels")   # only one dimensiondropout = tf.placeholder(dtype=tf.float32, shape=[], name="dropout") lr = tf.placeholder(dtype=tf.float32, shape=[], name="lr")

# 设置embedding层with tf.variable_scope("words"):    _word_embeddings = tf.Variable(embeddings, name="_word_embeddings", dtype=tf.float32,                                   trainable=trainable)    word_embeddings = tf.nn.embedding_lookup(_word_embeddings, word_ids,                                             name="word_embeddings")    word_embeddings = tf.nn.dropout(word_embeddings, dropout)# 设置模型with tf.variable_scope("bi-lstm"):    lstm_cell = tf.contrib.rnn.LSTMCell(hidden_size)    _, (output_state_fw, output_state_bw) = tf.nn.bidirectional_dynamic_rnn(lstm_cell,     lstm_cell, word_embeddings,     sequence_length=sequence_lengths,     dtype=tf.float32)    output = tf.concat((output_state_fw[-1], output_state_bw[-1]), axis=-1)    output = tf.nn.dropout(output, dropout) # 输出部分在双向lstm的最后一层合并后加一个全连接层，全连接层后接一个softmax层with tf.variable_scope("proj"):    W = tf.get_variable("W", shape=[2 * hidden_size, nlabels],                        dtype=tf.float32)    b = tf.get_variable("b", shape=[nlabels], dtype=tf.float32,                        initializer=tf.zeros_initializer())    output = tf.reshape(output, [-1, 2 * hidden_size])    pred = tf.matmul(output, W) + b    logits = pred labels_pred = tf.cast(tf.argmax(logits, axis=-1), tf.int32) losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) loss = tf.reduce_mean(losses) # batch的平均losswith tf.variable_scope("train_step"):    optimizer = tf.train.AdamOptimizer(lr)    train_op = optimizer.minimize(loss)

# 分类模型def create_model_classify(max_features, nlabels, embeddings=None, embedding_dim=200, hidden_dim=300):    model = Sequential()    if embeddings is not None:        model.add(Embedding(input_dim=max_features,output_dim=embedding_dim, dropout=0.2, weights=[embeddings], mask_zero=True))    else:        model.add(Embedding(max_features, embedding_dim=embedding_dim, dropout=0.2))    model.add(Bidirectional(LSTM(hidden_dim, dropout_W=0.2, dropout_U=0.2, return_sequences=False)))  # try using a GRU instead, for fun    model.add(Dense(nlabels))    model.add(Activation('softmax'))    model.compile(loss='categorical_crossentropy',                  optimizer='adam',                  metrics=['accuracy',]) # availabel metrics https://keras.io/metrics/    return model# 回归模型def create_model_regress(max_features, embeddings=None, embedding_dim=200, hidden_dim=300):    model = Sequential()    if embeddings is not None:        model.add(Embedding(input_dim=max_features,output_dim=embedding_dim, dropout=0.2, weights=[embeddings], mask_zero=True))    else:        model.add(Embedding(max_features, embedding_dim=embedding_dim, dropout=0.2))    model.add(        Bidirectional(LSTM(hidden_dim, dropout_W=0.2, dropout_U=0.2),                      merge_mode='concat'))      model.compile(loss='mean_squared_error',                  optimizer='sgd',                  metrics=['mae','acc' ])  # availabel metrics https://keras.io/metrics/idden_dim= return model

#keras实现from keras.models import Model org_model = load_model("result/model.weights/acc_XXX.model") layer_name = 'bidirectional_1'intermediate_layer_model = Model(input=org_model.input,   # keras 2.0  inputs  outputs output=org_model.get_layer(layer_name).output) intermediate_output = intermediate_layer_model.predict(np.asarray(word_ids))