import requests,time
from fake_useragent import UserAgent

comment_urls = []def get_title_id():    '''爬取戰(zhàn)疫情首頁(yè)的每個(gè)主題的ID'''    for page in range(1,3):# 這是控制ajax通道的量        headers = {            'User-Agent' : UserAgent().chrome #chrome瀏覽器隨機(jī)代理        }        time.sleep(2)        # 該鏈接通過抓包獲得        api_url = 'https://m.weibo.cn/api/feed/trendtop?containerid=102803_ctg1_600059_-_ctg1_600059&page=' + str(page)        print (api_url)        rep = requests.get(url=api_url, headers=headers)        for json in rep.json()['data']['statuses']:            comment_url = 'https://m.weibo.cn/detail/' + json['id']            print (comment_url)            comment_urls.append(comment_url)get_title_id()

# -*- coding: utf-8 -*-import requests,random,reimport timeimport osimport csvimport sysimport jsonimport importlibfrom fake_useragent import UserAgentfrom lxml import etree

importlib.reload(sys)startTime = time.time() #記錄起始時(shí)間

#--------------------------------------------文件存儲(chǔ)-----------------------------------------------------path = os.getcwd() + '/weiboComments.csv'csvfile = open(path, 'a', newline='', encoding = 'utf-8-sig')writer = csv.writer(csvfile)#csv頭部writer.writerow(('話題鏈接','話題內(nèi)容','樓主ID', '樓主昵稱', '樓主性別','發(fā)布日期',                 '發(fā)布時(shí)間', '轉(zhuǎn)發(fā)量','評(píng)論量','點(diǎn)贊量', '評(píng)論者ID', '評(píng)論者昵稱',                 '評(píng)論者性別', '評(píng)論日期', '評(píng)論時(shí)間','評(píng)論內(nèi)容')) 

#設(shè)置headesheaders = {    'Cookie': '_T_WM=22822641575; H5_wentry=H5; backURL=https%3A%2F%2Fm.weibo.cn%2F; ALF=1584226439; MLOGIN=1; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5RJaVYrb.BEuOvUQ8Ca2OO5JpX5K-hUgL.FoqESh-7eKzpShM2dJLoIp7LxKML1KBLBKnLxKqL1hnLBoMceoBfeh2EeKBN; SCF=AnRSOFp6QbWzfH1BqL4HB8my8eWNC5C33KhDq4Ko43RUIzs6rjJC49kIvz5_RcOJV2pVAQKvK2UbAd1Uh6j0pyo.; SUB=_2A25zQaQBDeRhGeBM71cR8SzNzzuIHXVQzcxJrDV6PUJbktAKLXD-kW1NRPYJXhsrLRnku_WvhsXi81eY0FM2oTtt; SUHB=0mxU9Kb_Ce6s6S; SSOLoginState=1581634641; WEIBOCN_FROM=1110106030; XSRF-TOKEN=dc7c27; M_WEIBOCN_PARAMS=oid%3D4471980021481431%26luicode%3D20000061%26lfid%3D4471980021481431%26uicode%3D20000061%26fid%3D4471980021481431',    'Referer': 'https://m.weibo.cn/detail/4312409864846621',    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36',    'X-Requested-With': 'XMLHttpRequest'}

#-----------------------------------爬取戰(zhàn)疫情首頁(yè)的每個(gè)主題的ID------------------------------------------comments_ID = []def get_title_id():    for page in range(1,21):  #每個(gè)頁(yè)面大約有18個(gè)話題        headers = {            'User-Agent' : UserAgent().chrome #chrome瀏覽器隨機(jī)代理        }        time.sleep(1)        #該鏈接通過抓包獲得        api_url = 'https://m.weibo.cn/api/feed/trendtop?containerid=102803_ctg1_600059_-_ctg1_600059&page=' + str(page)        print(api_url)        rep = requests.get(url=api_url, headers=headers)        #獲取ID值并寫入列表comment_ID中        for json in rep.json()['data']['statuses']:            comment_ID = json['id']             comments_ID.append(comment_ID)

#-----------------------------------爬取戰(zhàn)疫情每個(gè)主題的詳情頁(yè)面------------------------------------------         def spider_title(comment_ID):    try:        article_url = 'https://m.weibo.cn/detail/'+ comment_ID        print ('article_url = ', article_url)        html_text = requests.get(url=article_url, headers=headers).text        #話題內(nèi)容        find_title = re.findall('.*?'text': '(.*?)',.*?', html_text)[0]        title_text = re.sub('<(S*?)[^>]*>.*?|<.*? />', '', find_title) #正則匹配掉html標(biāo)簽        print ('title_text = ', title_text)        #樓主ID        title_user_id = re.findall('.*?'id': (.*?),.*?', html_text)[1]        print ('title_user_id = ', title_user_id)        #樓主昵稱        title_user_NicName = re.findall('.*?'screen_name': '(.*?)',.*?', html_text)[0]        print ('title_user_NicName = ', title_user_NicName)        #樓主性別        title_user_gender = re.findall('.*?'gender': '(.*?)',.*?', html_text)[0]        print ('title_user_gender = ', title_user_gender)        #發(fā)布時(shí)間        created_title_time = re.findall('.*?'created_at': '(.*?)'.*?', html_text)[0].split(' ')        #日期        if 'Mar' in created_title_time:            title_created_YMD = '{}/{}/{}'.format(created_title_time[-1], '03', created_title_time[2])        elif 'Feb' in created_title_time:            title_created_YMD = '{}/{}/{}'.format(created_title_time[-1], '02', created_title_time[2])        elif 'Jan' in created_title_time:            title_created_YMD = '{}/{}/{}'.format(created_title_time[-1], '01', created_title_time[2])        else:            print ('該時(shí)間不在疫情范圍內(nèi)，估計(jì)數(shù)據(jù)有誤！URL = ')            pass        print ('title_created_YMD = ', title_created_YMD)        #發(fā)布時(shí)間        add_title_time = created_title_time[3]        print ('add_title_time = ', add_title_time)        #轉(zhuǎn)發(fā)量        reposts_count = re.findall('.*?'reposts_count': (.*?),.*?', html_text)[0]        print ('reposts_count = ', reposts_count)        #評(píng)論量        comments_count = re.findall('.*?'comments_count': (.*?),.*?', html_text)[0]        print ('comments_count = ', comments_count)        #點(diǎn)贊量        attitudes_count = re.findall('.*?'attitudes_count': (.*?),.*?', html_text)[0]        print ('attitudes_count = ', attitudes_count)           comment_count = int(int(comments_count) / 20) #每個(gè)ajax一次加載20條數(shù)據(jù)        position1 = (article_url, title_text, title_user_id, title_user_NicName,title_user_gender, title_created_YMD, add_title_time, reposts_count, comments_count, attitudes_count, ' ', ' ', ' ', ' ',' ', ' ')        #寫入數(shù)據(jù)        writer.writerow((position1))        return comment_count    except:        pass



#-------------------------------------------------抓取評(píng)論信息---------------------------------------------------#comment_ID話題編號(hào)def get_page(comment_ID, max_id, id_type):    params = {        'max_id': max_id,        'max_id_type': id_type    }    url = ' https://m.weibo.cn/comments/hotflow?id={}&mid={}&max_id'.format(comment_ID, comment_ID)    try:        r = requests.get(url, params=params, headers=headers)        if r.status_code == 200:            return r.json()    except requests.ConnectionError as e:        print('error', e.args)        pass

#-------------------------------------------------抓取評(píng)論item最大值---------------------------------------------------def parse_page(jsondata):    if jsondata:        items = jsondata.get('data')        item_max_id = {}        item_max_id['max_id'] = items['max_id']        item_max_id['max_id_type'] = items['max_id_type']        return item_max_id

#-------------------------------------------------抓取評(píng)論信息---------------------------------------------------def write_csv(jsondata):    for json in jsondata['data']['data']:        #用戶ID        user_id = json['user']['id']        # 用戶昵稱        user_name = json['user']['screen_name']        # 用戶性別,m表示男性，表示女性        user_gender = json['user']['gender']        #獲取評(píng)論        comments_text = json['text']        comment_text = re.sub('<(S*?)[^>]*>.*?|<.*? />', '', comments_text) #正則匹配掉html標(biāo)簽        # 評(píng)論時(shí)間        created_times = json['created_at'].split(' ')        if 'Feb' in created_times:            created_YMD = '{}/{}/{}'.format(created_times[-1], '02', created_times[2])        elif 'Jan' in created_times:            created_YMD = '{}/{}/{}'.format(created_times[-1], '01', created_times[2])        else:            print ('該時(shí)間不在疫情范圍內(nèi)，估計(jì)數(shù)據(jù)有誤！')            pass        created_time = created_times[3] #評(píng)論時(shí)間時(shí)分秒        #if len(comment_text) != 0:        position2 = (' ', ' ', ' ', ' ',' ', ' ', ' ', ' ', ' ', ' ', user_id, user_name, user_gender, created_YMD, created_time, comment_text)        writer.writerow((position2))#寫入數(shù)據(jù)        #print (user_id, user_name, user_gender, created_YMD, created_time)    



#-------------------------------------------------主函數(shù)---------------------------------------------------def main():    count_title = len(comments_ID)    for count, comment_ID in enumerate(comments_ID):        print ('正在爬取第%s個(gè)話題，一共找到個(gè)%s話題需要爬取'%(count+1, count_title))        #maxPage獲取返回的最大評(píng)論數(shù)量        maxPage = spider_title(comment_ID)        print ('maxPage = ', maxPage)        m_id = 0        id_type = 0        if maxPage != 0: #小于20條評(píng)論的不需要循環(huán)            try:                #用評(píng)論數(shù)量控制循環(huán)                for page in range(0, maxPage):                    #自定義函數(shù)-抓取網(wǎng)頁(yè)評(píng)論信息                    jsondata = get_page(comment_ID, m_id, id_type)
                    #自定義函數(shù)-寫入CSV文件                    write_csv(jsondata)
                    #自定義函數(shù)-獲取評(píng)論item最大值                    results = parse_page(jsondata)                    time.sleep(1)                    m_id = results['max_id']                    id_type = results['max_id_type']                          except:                pass        print ('--------------------------分隔符---------------------------')    csvfile.close() 
if __name__ == '__main__':
    #獲取話題ID    get_title_id()
    #主函數(shù)操作    main()
    #計(jì)算使用時(shí)間    endTime = time.time()    useTime = (endTime-startTime) / 60    print('該次所獲的信息一共使用%s分鐘'%useTime)

# coding=utf-8
from pyecharts import options as optsfrom pyecharts.charts import WordCloudfrom pyecharts.globals import SymbolType

# 數(shù)據(jù)words = [    ('背包問題', 10000),    ('大整數(shù)', 6181),    ('Karatsuba乘法算法', 4386),    ('窮舉搜索', 4055),    ('傅里葉變換', 2467),    ('狀態(tài)樹遍歷', 2244),    ('剪枝', 1868),    ('Gale-shapley', 1484),    ('最大匹配與匈牙利算法', 1112),    ('線索模型', 865),    ('關(guān)鍵路徑算法', 847),    ('最小二乘法曲線擬合', 582),    ('二分逼近法', 555),    ('牛頓迭代法', 550),    ('Bresenham算法', 462),    ('粒子群優(yōu)化', 366),    ('Dijkstra', 360),    ('A*算法', 282),    ('負(fù)極大極搜索算法', 273),    ('估值函數(shù)', 265)]

# 渲染圖def wordcloud_base() -> WordCloud:    c = (        WordCloud()        .add('', words, word_size_range=[20, 100], shape='diamond')  # SymbolType.ROUND_RECT        .set_global_opts(title_opts=opts.TitleOpts(title='WordCloud詞云'))    )    return c

# 生成圖wordcloud_base().render('詞云圖.html')

# coding=utf-8import jiebaimport reimport timefrom collections import Counter

#------------------------------------中文分詞------------------------------------cut_words = ''all_words = ''f = open('C-class-fenci.txt', 'w')for line in open('C-class.txt', encoding='utf-8'):    line.strip('\n')    seg_list = jieba.cut(line,cut_all=False)    # print(' '.join(seg_list))    cut_words = (' '.join(seg_list))    f.write(cut_words)    all_words += cut_wordselse:    f.close()

# 輸出結(jié)果all_words = all_words.split()print(all_words)

# 詞頻統(tǒng)計(jì)c = Counter()for x in all_words:    if len(x)>1 and x != '\r\n':        c[x] += 1

# 輸出詞頻最高的前10個(gè)詞print('\n詞頻統(tǒng)計(jì)結(jié)果：')for (k,v) in c.most_common(10):    print('%s:%d'%(k,v))

# 存儲(chǔ)數(shù)據(jù)name = time.strftime('%Y-%m-%d') + '-fc.csv'fw = open(name, 'w', encoding='utf-8')i = 1for (k,v) in c.most_common(len(c)):    fw.write(str(i)+','+str(k)+','+str(v)+'\n')    i = i + 1else:    print('Over write file!')    fw.close()

#------------------------------------詞云分析------------------------------------from pyecharts import options as optsfrom pyecharts.charts import WordCloudfrom pyecharts.globals import SymbolType

# 生成數(shù)據(jù) word = [('A',10), ('B',9), ('C',8)] 列表+Tuplewords = []for (k,v) in c.most_common(1000):    # print(k, v)    words.append((k,v))

# 渲染圖def wordcloud_base() -> WordCloud:    c = (        WordCloud()        .add('', words, word_size_range=[20, 100], shape=SymbolType.ROUND_RECT)        .set_global_opts(title_opts=opts.TitleOpts(title='全國(guó)新型冠狀病毒疫情詞云圖'))    )    return c

# 生成圖wordcloud_base().render('疫情詞云圖.html')

pip install snownlp

# -*- coding: utf-8 -*-from snownlp import SnowNLPs = SnowNLP(u'這本書質(zhì)量真不太好！')

print(u'\n中文分詞:')print( ' '.join(s.words))

print(u'\n詞性標(biāo)注:')print(s.tags)for k in s.tags:    print(k)

print(u'\n情感分?jǐn)?shù):')print(s.sentiments)

print(u'\n轉(zhuǎn)換拼音:')print(s.pinyin)

print(u'\n輸出前4個(gè)關(guān)鍵詞:')print(s.keywords(4))for k in s.keywords(4):    print(k)

print(u'\n輸出關(guān)鍵句子:')print(s.summary(1))for k in s.summary(1):    print(k)

print(u'\n輸出tf和idf:')print(s.tf)print(s.idf)

n = SnowNLP(u'「繁體字」「繁體中文」的叫法在臺(tái)灣亦很常見。')print(u'\n繁簡(jiǎn)體轉(zhuǎn)換:')print(n.han)

# -*- coding: utf-8 -*-from snownlp import SnowNLPs1 = SnowNLP(u'我今天很開心')print(u's1情感分?jǐn)?shù):')print(s1.sentiments)

s2 = SnowNLP(u'我今天很沮喪')print(u's2情感分?jǐn)?shù):')print(s2.sentiments)

s3 = SnowNLP(u'大傻瓜，你脾氣真差，動(dòng)不動(dòng)就打人')print(u's3情感分?jǐn)?shù):')print(s3.sentiments)

# -*- coding: utf-8 -*-from snownlp import SnowNLPimport codecsimport os

source = open('data.txt','r', encoding='utf-8')line = source.readlines()sentimentslist = []for i in line:    s = SnowNLP(i)    print(s.sentiments)    sentimentslist.append(s.sentiments)

import matplotlib.pyplot as pltimport numpy as npplt.hist(sentimentslist, bins = np.arange(0, 1, 0.01), facecolor = 'g')plt.xlabel('Sentiments Probability')plt.ylabel('Quantity')plt.title('Analysis of Sentiments')plt.show()

# -*- coding: utf-8 -*-from snownlp import SnowNLPimport codecsimport os

source = open('data.txt','r', encoding='utf-8')line = source.readlines()sentimentslist = []for i in line:    s = SnowNLP(i)    print(s.sentiments)    sentimentslist.append(s.sentiments)

import matplotlib.pyplot as pltimport numpy as npplt.plot(np.arange(0, 356, 1), sentimentslist, 'k-')plt.xlabel('Number')plt.ylabel('Sentiment')plt.title('Analysis of Sentiments')plt.show()