Python爬虫之对CSDN榜单进行分析

from requests import get
from bs4 import BeautifulSoup as bs
from win32com.shell import shell
from datetime import datetime as dt
from time import perf_counter
from re import findall
 
def lenB(str):
    t = 0
    for s in str:
        if '\u4e00' <= s <= '\u9fef': t += 1
    return t
 
def reportTime():
    d = dt.today()
    return f'{d.year}-{d.month:02}-{d.day:02} {d.hour:02}:{d.minute:02}'
 
def reportData(Index, num=0):
    count = [0,0,0,0,0,0]
    for i in Index:
        if i[0] <= num : count[0]+=1
        if i[1] <= num : count[1]+=1
        if i[2] <= num : count[2]+=1
        if sum(i[:2]) <= num : count[3]+=1
        if sum(i[:3]) <= num : count[4]+=1
        if i[-1]: count[-1]+=1
    return count
 
def reportData2(Index):
    count = [0,0,0,0,0,0]
    out1,out2 = (200,20,20),(100,10,10)
    for i in Index:
        if i[0] >= out1[0] : count[0]+=1
        if i[0] >= out2[0] : count[1]+=1
        if i[1] >= out1[1] : count[2]+=1
        if i[1] >= out2[1] : count[3]+=1
        if i[2] >= out1[2] : count[4]+=1
        if i[2] >= out2[2] : count[5]+=1
    return count
 
def reportString(count):
    outStr = f'浏览为0的：\t{count[0]}\n评论为0的：\t'
    outStr += f'{count[1]}\n收藏为0的：\t{count[2]}\n'
    outStr += f'浏览评论0的：\t{count[3]}\n'
    outStr += f'三指标都0的：\t{count[4]}\n'
    return outStr
 
def reportString2(count):
    outStr = f'浏览个位数的：\t{count[0]}\n'
    outStr += f'评论个位数的：\t{count[1]}\n'
    outStr += f'收藏个位数的：\t{count[2]}\n'
    outStr += f'无封面题图的：\t{count[-1]}\n'
    return outStr
 
def reportString3(count,idx=0):
    out1,out2 = (200,20,20),(100,10,10)
    if idx==0: out = out1
    else: idx=1; out = out2
    outStr = f'浏览>={out[0]}的：\t{count[idx]}\n'
    outStr += f'评论>={out[1]}的：\t{count[idx+2]}\n'
    outStr += f'收藏>={out[2]}的：\t{count[idx+4]}\n'
    return outStr
 
def dictFromList(Values,num=5):
    tags,tagDict = [],dict()
    if num==5:
        for tag in Values:
            tags.extend(list(tag[num]))
    else:
        for tag in Values:
            tags.append(tag[num])
    Tags = list(set(tags))
    for t in Tags:
        tagDict[str(t)] = tags.count(t)
    return tagDict
 
def matchRange(value):
    if 0=1 and kwi>0:
            outStr3 += f'{kws:<{10-lenB(kws)}}: {kwi}\n'
    '''
    #字典形式可以排序
    kwDict = dict()
    for i,kw in enumerate(keywords): kwDict[kw] = kwIndex[i]
    for item in sorted(kwDict.items(),key=lambda k:k[1],reverse=True):
        if not item[1]: print();break
        print(f'{item[0]:<{10-lenB(item[0])}}: {item[1]}')
    '''
    print(outStr1 + '\n' + outStr1_2 + '\n' + outStr1_3)
    print(outStr2 + '\n' + outStr2_2 + '\n' + outStr2_3)
 
    #爬取Top100资料
    Values = []
    print('开始爬取Top100网页，请稍候(大约1分钟)...')
    timer = perf_counter()
    for idx,url in enumerate(articles):
        res = get(url,headers={'User-Agent':agent})
        res.encoding='uft-8'
        soup = bs(res.text,'html.parser')
        value = []
        for i in range(3,8):
            if i!=4: value.append(rank[idx][i])
        try: #码龄
            ctxt = 'personal-home-page personal-home-years'
            html = soup.find('span',class_=ctxt)
            value.append(int(findall(r'\d+',html.text)[0]))
        except: value.append(-1)
        try: #标签
            ctxt = 'tags-box artic-tag-box'
            html = soup.find('div',class_=ctxt)
            html = html.find_all('a')
            value.append(set([i.text.strip().lower().title() for i in html]))
        except: value.append(set())
        html = soup.find_all('dl')
        for dl in range(1,len(html)):
            try: dlval = html[dl]['title']
            except: break
            if '级' in dlval: dlval = dlval[:dlval.find('级')]
            if '暂无排名' in dlval: dlval = -1
            value.append(int(dlval))
        if len(value)==16: value.append(False)
        else: value.append(True)
        Values.append(value)
    print(f'耗时：{perf_counter()-timer}s')
    datafile = desktop + r'\【CSDN排行榜Top100】data'+reportTime()[:10]+'.txt'
    #收集到的数据全部写入文件*data*.txt
    failVals = 0
    with open(datafile,'w',encoding='utf-8') as f:
        f.write(f'#报告日期：{reportTime()}\n')
        for val in Values:
            if val[-1]:
                failVals += 1
                continue
            f.writelines('|'.join([str(i) for i in val]))
            f.write('\n')
    Values = [val for val in Values if not val[-1]]
    print(f'下载失败数：{failVals}\n')
    '''value列表各元素的索引值分别代表：
    0~ 5：昵称 用户名 标题 链接 码龄 文章标签
    6~10：原创 周排名 总排名 访问 等级
    11~15：积分 粉丝 获赞 评论 收藏
    16：  失败标记=False'''
    #输出内容
    tagsRank,tagDict = [],dictFromList(Values)
    for item in sorted(tagDict.items(),key=lambda t:t[1],reverse=True):
        tagsRank.append(f'{item[0]:<{22-lenB(item[0])}}: {item[1]}')
        if len(tagsRank)>=100: break
    outStr4 = lambda x:f'\n【Top100文章标签排名(Top{x})】\n'
    
    outStr5 = '\n【Top100博客等级分布】\n\n'
    valDict = dictFromList(Values,10)
    for key,value in valDict.items():
        outStr5 += f'等级{key:>2}: {value}\n'
    print(outStr5, file=tofile)
    
    outStr6 = '\n【Top100作者码龄分布】\n\n'
    valDict = dictFromList(Values,4)
    for key,value in valDict.items():
        outStr6 += f'码龄{key:>2}: {value}\n'
    print(outStr6, file=tofile)
 
    outStr7 = '\n【Top100作者周排名分布】\n\n'
    valDict = dictFromList(Values,7)
    weekRank = [0 for _ in range(10)]
    for key,value in valDict.items():
        weekRank[matchRange(int(key))] += 1
    textRank = ('周榜前100名','第100~200名','第200~500名','第500~1千名','第1千~5千名',
                '第5千~1万名','第1万~2万名','第2万~5万名','第5万~10万名','10万名之后')
    for i,txt in enumerate(textRank):
        outStr7 += f'{txt:<{14-lenB(txt)}}: {weekRank[i]}\n'
    print(outStr7, file=tofile)
    #更多指标可用以分析或者画折线图柱形图等等，指标索引见上面的注释
    print(outStr5 + outStr6 + outStr7)
    print(f'更多详情见桌面文件：\n{filename[1:]}')
    endStr = '='*26
    endStr += f'\n报告日期：{reportTime()[:10]}'
    endStr += f'\n报告时间：{reportTime()[11:]}'
    endStr += f'\n程序耗时：{round(perf_counter(),2)}s\n'
    print(outStr3, file=tofile)
    print(outStr4(100), file=tofile)
    for t in tagsRank: print(t, file=tofile)
    print(endStr, file=tofile)
    tofile.close()
    print(endStr)
 
''' 运行环境： Windows7 + Python3.8.8 '''
''' Written by Hann, 2021.10.25 '''

import matplotlib.pyplot as plt
 
plt.figure().canvas.set_window_title('CSDN Top100 热度衰减示意图')
plt.title('Top100 Attenuation Diagram')
		     
hots = [h[3] for h in index]
x = [i+1 for i in range(len(hots))]
plt.plot(x, hots)
		     
plt.show()

import matplotlib.pyplot as plt
 
plt.figure().canvas.set_window_title('CSDN Top100 博客等级分布示意图')
 
Level = [i[10] for i in Values]
x = [i+1 for i in range(len(Level))]
plt.scatter(x, Level,marker='o')
		     
plt.show()