1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | #!/usr/bin/env python 获取单个信息 # coding=utf-8 import datetime from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException import selenium.webdriver.support.ui as ui browser = webdriver.Firefox() def is_visible(locator, timeout = 10): try: ui.WebDriverWait(browser, timeout).until(EC.visibility_of_element_located((By.XPATH, locator))) return True except TimeoutException: return False browser.get("http://zssom.sysu.edu.cn/zh-hans/teacher/377") is_visible('/html/body/div[2]/div[2]/div[1]') html = browser.page_source content = BeautifulSoup(html, "lxml") description = content.find(attrs={"name":"description"})['content'] print(description) |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | #以下还有问题 请小心 无法运行 #以下为中山医实例 此处向中山大学致以崇高的敬意与祝福 欢迎提意见 #!/usr/bin/env python 如有侵权 亲联系 删文章哦 # coding=utf-8 import datetime from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException import selenium.webdriver.support.ui as ui browser = webdriver.Firefox() def is_visible(locator, timeout = 10): try: ui.WebDriverWait(browser, timeout).until(EC.visibility_of_element_located((By.XPATH, locator))) return True except TimeoutException: return False #编辑 url i =350 while i <381: i=i+1 a = str(i) b = "http://zssom.sysu.edu.cn/zh-hans/teacher/" url= b+a browser.get(url) if is_visible('/html/body/div[2]/div[2]/div[1]'): html = browser.page_source content = BeautifulSoup(html, "lxml") #获取老师信息内容 description = content.find(attrs={"name":"description"})['content'] print(description) else: print("获取内容为空") else: print("over") |
tips
1.用selenium 模拟浏览器行为
2.拼接字符串 设计url
3.bs4 获取 节点数据