Python selenium爬取微信公众号文章代码详解

éæ±ï¼

æ³é读微信å¬ä¼å·åå²æç« ï¼ä½æ¯æ¯æ¬¡æ¾åçå¾å°æ¹ä¸æ¹ä¾¿ã

æè·¯ï¼

1ã使ç¨seleniumæå¼å¾®ä¿¡å¬ä¼å·åå²æç« ï¼å¹¶æ»å¨å·æ°å°æåºé¨ï¼è·åå°ææåå²æç« urlsã

2ã对urlsè¿è¡éå访é®ï¼å¹¶è¿è¡ä¸è½½å°æ¬å°ã

å®ç°

1ãæå¼å¾®ä¿¡å®¢æ·ç«¯ï¼ç¹å»æ个微信å¬ä¼å·->è¿å¥å¬ä¼å·->æå¼åå²æç« é¾æ¥ï¼ä½¿ç¨æµè§å¨æå¼ï¼ï¼å¹¶éè¿å¼åèå·¥å·è·åå°cookiesï¼ä¿å­ä¸ºexcelã

2ãå¯å¨webdriverï¼å¹¶æ·»å ç¸åºcookiesã

browser = webdriver.Chrome()
wait = WebDriverWait(browser,10)
# é便访é®ä¸ä¸ªå°åï¼ç¶åæè½è®¾ç½®cookies
browser.get('https://httpbin.org/get')
# æ·»å cookies,df为ä¿å­çexcel cookies
for i in range(len(df)):
  cookie_dict = {
          "domain": df.loc[i,'DomaiN'], 
          'name': df.loc[i,'Name'],
          'value': str(df.loc[i,'Value']),
          "expires": df.loc[i,"Expires/Max-Age"],
          'path': '/',}
  browser.add_cookie(cookie_dict)
browser.get(weixin_url)

3ãæ§å¶æµè§å¨ä¸ç§»å¨

è§å¯page_sourceï¼å¯ä»¥åç°ï¼æç« å°æåºé¨çå¤æ­æ¯ã

<div class="loadmore with_line" style="display: none;" id="js_nomore">
    <div class="tips_wrp">
      <span class="tips js_no_more_msg" style="display: none;">å·²æ æ´å¤</span>
      <span class="tips js_need_add_contact" style="display: none;">å³æ³¨å¬ä¼å¸å·ï¼æ¥æ¶æ´å¤æ¶æ¯</span>
    </div>
  </div>

使ç¨driveræ§å¶JSã

%%time
# éè¿å¤æ­å·²æ æ´å¤çstyle,æ¥å¤æ­æ¯å¦å°æåºé¨ï¼æç»æ§è¡å°æåºé¨
no_more_msg_style = 'display: none;'
while True:
  wait.until(EC.presence_of_element_located((By.XPATH,'//span[@class="tips js_no_more_msg" and text()="å·²æ æ´å¤"]')))
  no_more= browser.find_element_by_xpath('//span[@class="tips js_no_more_msg" and text()="å·²æ æ´å¤"]')
  now_style = no_more.get_attribute('style')
  if str(now_style).find(no_more_msg_style) == -1:
    # 说æå·²ç»å è½½å®äº
    break
  else:
    # åé¡¿ä¸ä¼ï¼ç­å¾æµè§å¨å è½½
    time.sleep(5)
    # éè¿JSï¼æ§è¡å°æåºé¨
    browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')

4ãå³é®ä¿¡æ¯è·åã

æ ¹æ®html,åæå¾åºæç« urlå¤å¨<div msgid="1000000026">中ã

<div class="weui_msg_card js_card" msgid="1000000026">
      <div class="weui_msg_card_hd">2017å¹´1æ13æ¥</div>
      <div class="weui_msg_card_bd">
         <!-- å¾æ -->
             <!-- æ®éå¾æ -->
            <div id="WXAPPMSG1000000026" class="weui_media_box appmsg js_appmsg" hrefs="http://mp.weixin.qq.com/s?__biz=MzI5MDQ4NzU5MA==&mid=2247483748&idx=1&sn=e804e638484794181a27c094f81be8e1&chksm=ec1e6d2ddb69e43bd3e1f554c2d0cedb37f099252f122cee1ac5052b589b56f428b2c304de8e&scene=38#wechat_redirect" data-t="0">
              <span class="weui_media_hd js_media" style="background-image:url(http://mmbiz.qpic.cn/mmbiz_jpg/XibhQ5tjv6dG9B4GF1C9MGBJO5AR2wvjCL9LgdcFgAdEgyU8wZFuDXoH9O9dNvafwK3RibCjUyiarIlUDlkxbcyfQ/640?wx_fmt=jpeg)" data-s="640" hrefs="http://mp.weixin.qq.com/s?__biz=MzI5MDQ4NzU5MA==&mid=2247483748&idx=1&sn=e804e638484794181a27c094f81be8e1&chksm=ec1e6d2ddb69e43bd3e1f554c2d0cedb37f099252f122cee1ac5052b589b56f428b2c304de8e&scene=38#wechat_redirect" data-type="APPMSG">
              </span>
              <div class="weui_media_bd js_media" data-type="APPMSG">
                <h4 class="weui_media_title" hrefs="http://mp.weixin.qq.com/s?__biz=MzI5MDQ4NzU5MA==&mid=2247483748&idx=1&sn=e804e638484794181a27c094f81be8e1&chksm=ec1e6d2ddb69e43bd3e1f554c2d0cedb37f099252f122cee1ac5052b589b56f428b2c304de8e&scene=38#wechat_redirect">
                  æ¿è®¤èªå·±æ¯é¾æ°æä»ä¹é
                </h4>
                <p class="weui_media_desc">æ·éå·²ç»è¶³å¤æ²éï¼è°¢ç»éå¾·ç»æ¶</p>
                <p class="weui_media_extra_info">2017å¹´1æ13æ¥</p>
              </div>
            </div> 
      </div>
    </div>

æç« ç±»å主è¦å为ï¼

<div class="weui_media_bd js_media" data-type="APPMSG">
<div class="weui_media_bd js_media" data-type="TEXT">

ææ ååè¿è¡ååã

æç»å®ç°ï¼

%%time
result = []
errlist = []
# åå¾å°å¶ä¸­ä¸ä¸ª
el_divs = browser.find_elements_by_xpath('//div[@class="weui_msg_card_list"]/div[@class="weui_msg_card js_card"]')
i = 0
for div in el_divs:
  date = title = url = yuanchuang = ''
  try:
    date = div.find_element_by_xpath('.//div[@class="weui_msg_card_hd"]').get_attribute('innerHTML')
    el_content = div.find_element_by_xpath('.//div[@class="weui_media_bd js_media"]')
    if el_content.get_attribute('data-type') == 'APPMSG':
      el = el_content.find_element_by_xpath('./h4[@class="weui_media_title"]')
      title = el.text
      url = el.get_attribute('hrefs')
      xb = el_content.find_element_by_xpath('./p[@class="weui_media_extra_info"]').text
      yuanchuang = 'åå' if xb.find('åå') != -1 else ''
    elif el_content.get_attribute('data-type') == 'TEXT':
      title = 'éæ'
      url = el_content.find_element_by_xpath('./div').text
      yuanchuang = 'åå'
    else:
      # å¶ä»æªè½è¯å«çç±»å
      errlist.append([i,div.get_attribute('innerHTML')])
  except NoSuchElementException:
    errlist.append([i,div.get_attribute('innerHTML')])
  print(str(i),':',date,title,url,yuanchuang)
  result.append([date,title,yuanchuang,url])
  i = i + 1

5ãå°å¾å°urlä¿å­å°excel

dfout = pd.DataFrame(result, columns=['æ¥æ', 'æ é¢', 'åå', 'å°å'])
with pd.ExcelWriter(savename) as writer:
dfout.to_excel(writer,index=False,sheet_name = 'Sheet1')

æç»ä¿å­å½¢å¼

6ãå¨éåæåçé¾æ¥å°åï¼é个requetsä¿å­ï¼å³å¯å¾å°ãç»å»ºæèåå½¢å¼çæç« ï¼å¯åè

éå°çåï¼

1ãfind_element_by_xpath éè¦éä¸ NoSuchElementException 使ç¨ï¼å¦åéå°æªæ¾å°çèç¹å°±ä¼åºéï¼æåfind_elements_by_xpath æ¥é²æ­¢æ¾ä¸å°ç¸å³èç¹ï¼ç»æåç°ï¼æ§è¡é度å¼å¸¸çæ¢ï¼éè¦æ¥æ¾åå ã

2ãcookies使ç¨çæ¶åæ¯äººä¸ºè·åï¼å¦æ太é¿æ¶é´ä¸ç¨ï¼éè¦éæ°è·åãå¯ä»¥èèç»åpyautoguiæ¥æ§å¶weixin客æ·ç«¯æ¥è¿è¡è·åãï¼

3ãæ建çæ¶åï¼æååå¸è¯è¡ï¼æåçæç« ç±»å没æå好å¤æ­ï¼ç»ææ§è¡æ¶é´å¾ä¹ãå好å¼å¸¸æè·ï¼åéæ­¥åæé误çèç¹é®é¢ã

以ä¸å°±æ¯æ¬æçå¨é¨å容ï¼å¸æ对大家ç学习ææ帮å©ï¼ä¹å¸æ大家å¤å¤æ¯æèæ¬ä¹å®¶ã