一、
bs4安装与使用
'''''''''安装解析器:pip3 install lxml安装解析库:pip3 install bs4'''html_doc = """The Dormouse's story $37
Once upon a time there were three little sisters; and their names wereElsie,Lacie andTillie;and they lived at the bottom of a well.
...
"""from bs4 import BeautifulSoup# python自带的解析库# soup = BeautifulSoup(html_doc, 'html.parser')# 调用bs4得到一个soup对象soup = BeautifulSoup(html_doc, 'lxml')# bs4对象print(soup)# bs4类型print(type(soup))# 美化功能html = soup.prettify()print(html) 二、bs4解析库之遍历文档树
'''''''''安装解析器:pip3 install lxml安装解析库:pip3 install bs4'''html_doc = """The Dormouse's story $37
Once upon a time there were three little sisters; and their names wereElsie,Lacie andTillie;and they lived at the bottom of a well.
...
"""from bs4 import BeautifulSoup# python自带的解析库# soup = BeautifulSoup(html_doc, 'html.parser')# 调用bs4得到一个soup对象soup = BeautifulSoup(html_doc, 'lxml')# bs4对象print(soup)# bs4类型print(type(soup))# 美化功能html = soup.prettify()print(html) 三、bs4解析库之遍历文档树
html_doc = """The Dormouse's story $37
Once upon a time there were three little sisters; and their names wereElsie,Lacie andTillie;and they lived at the bottom of a well.
...
"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html_doc, 'lxml')# print(soup)# print(type(soup))# 遍历文档树# 1、直接使用 *****print(soup.html)print(type(soup.html))print(soup.a)print(soup.p)# 2、获取标签的名称print(soup.a.name)# 3、获取标签的属性 *****print(soup.a.attrs) # 获取a标签中所有的属性print(soup.a.attrs['href'])# 4、获取标签的文本内容 *****print(soup.p.text) # $37# 5、嵌套选择print(soup.html.body.p)# 6、子节点、子孙节点print(soup.p.children) # 返回迭代器对象print(list(soup.p.children)) # [$37]# 7、父节点、祖先节点print(soup.b.parent)print(soup.b.parents)print(list(soup.b.parents))# 8、兄弟节点 （sibling: 兄弟姐妹）print(soup.a)# 获取下一个兄弟节点print(soup.a.next_sibling)# 获取下一个的所有兄弟节点,返回的是一个生成器print(soup.a.next_siblings)print(list(soup.a.next_siblings))# 获取上一个兄弟节点print(soup.a.previous_sibling)# 获取上一个的所有兄弟节点，返回的是一个生成器print(list(soup.a.previous_siblings)) 四、bs4之搜索文档树
'''''''''find: 找第一个find_all: 找所有标签查找与属性查找:name 属性匹配 name 标签名 attrs 属性查找匹配 text 文本匹配 标签: - 字符串过滤器 字符串全局匹配 - 正则过滤器 re模块匹配 - 列表过滤器 列表内的数据匹配 - bool过滤器 True匹配 - 方法过滤器 用于一些要的属性以及不需要的属性查找。 属性: - class_ - id'''html_doc = """The Dormouse's story $37
Once upon a time there were three little sisters; and their names wereElsieLacie andTillieand they lived at the bottom of a well.
...
"""from bs4 import BeautifulSoupsoup = BeautifulSoup(html_doc, 'lxml')# name 标签名# attrs 属性查找匹配# text 文本匹配# find与find_all搜索文档'''字符串过滤器'''p = soup.find(name='p')p_s = soup.find_all(name='p')print(p)print(p_s)# name + attrsp = soup.find(name='p', attrs={"id": "p"})print(p)# name + texttag = soup.find(name='title', text="The Dormouse's story")print(tag)# name + attrs + texttag = soup.find(name='a', attrs={"class": "sister"}, text="Elsie")print(tag)'''- 正则过滤器re模块匹配'''import re# name# 根据re模块匹配带有a的节点a = soup.find(name=re.compile('a'))print(a)a_s = soup.find_all(name=re.compile('a'))print(a_s)# attrsa = soup.find(attrs={"id": re.compile('link')})print(a)# - 列表过滤器# 列表内的数据匹配print(soup.find(name=['a', 'p', 'html', re.compile('a')]))print(soup.find_all(name=['a', 'p', 'html', re.compile('a')]))# - bool过滤器# True匹配print(soup.find(name=True, attrs={"id": True}))# - 方法过滤器# 用于一些要的属性以及不需要的属性查找。def have_id_not_class(tag): # print(tag.name) if tag.name == 'p' and tag.has_attr("id") and not tag.has_attr("class"): return tag# print(soup.find_all(name=函数对象))print(soup.find_all(name=have_id_not_class))# 补充知识点:# ida = soup.find(id='link2')print(a)# classp = soup.find(class_='sister')print(p)