python爬取安居客二手房網(wǎng)站數(shù)據(jù)方法分享

# 網(wǎng)頁的請(qǐng)求頭
header = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}
# url鏈接
url = 'https://zhengzhou.anjuke.com/sale/'
response = requests.get(url, headers=header)
print(response.text)

# 通過BeautifulSoup進(jìn)行解析出每個(gè)房源詳細(xì)列表并進(jìn)行打印
soup = BeautifulSoup(response.text, 'html.parser')
result_li = soup.find_all('li', {'class': 'list-item'})
for i in result_li:
 print(i)

# 通過BeautifulSoup進(jìn)行解析出每個(gè)房源詳細(xì)列表并進(jìn)行打印
soup = BeautifulSoup(response.text, 'html.parser')
result_li = soup.find_all('li', {'class': 'list-item'})
# 進(jìn)行循環(huán)遍歷其中的房源詳細(xì)列表
for i in result_li:
 # 由于BeautifulSoup傳入的必須為字符串，所以進(jìn)行轉(zhuǎn)換
 page_url = str(i)
 soup = BeautifulSoup(page_url, 'html.parser')
 # 由于通過class解析的為一個(gè)列表，所以只需要第一個(gè)參數(shù)
 result_href = soup.find_all('a', {'class': 'houseListTitle'})[0]
 print(result_href.attrs['href'])

# 進(jìn)行下一頁的爬取
result_next_page = soup.find_all('a', {'class': 'aNxt'})
if len(result_next_page) != 0:
 print(result_next_page[0].attrs['href'])
else:
 print('沒有下一頁了')

import requests
from bs4 import BeautifulSoup

# 網(wǎng)頁的請(qǐng)求頭
header = {
 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}

def get_page(url):
 response = requests.get(url, headers=header)

 # 通過BeautifulSoup進(jìn)行解析出每個(gè)房源詳細(xì)列表并進(jìn)行打印
 soup = BeautifulSoup(response.text, 'html.parser')
 result_li = soup.find_all('li', {'class': 'list-item'})

 # 進(jìn)行下一頁的爬取
 result_next_page = soup.find_all('a', {'class': 'aNxt'})
 if len(result_next_page) != 0:
 # 函數(shù)進(jìn)行遞歸
 get_page(result_next_page[0].attrs['href'])
 else:
 print('沒有下一頁了')

 # 進(jìn)行循環(huán)遍歷其中的房源詳細(xì)列表
 for i in result_li:
 # 由于BeautifulSoup傳入的必須為字符串，所以進(jìn)行轉(zhuǎn)換
 page_url = str(i)
 soup = BeautifulSoup(page_url, 'html.parser')
 # 由于通過class解析的為一個(gè)列表，所以只需要第一個(gè)參數(shù)
 result_href = soup.find_all('a', {'class': 'houseListTitle'})[0]
 # 先不做分析，等一會(huì)進(jìn)行詳細(xì)頁面函數(shù)完成后進(jìn)行調(diào)用
 print(result_href.attrs['href'])


if __name__ == '__main__':
 # url鏈接
 url = 'https://zhengzhou.anjuke.com/sale/'
 # 頁面爬取函數(shù)調(diào)用
 get_page(url)

import requests
from bs4 import BeautifulSoup

# 網(wǎng)頁的請(qǐng)求頭
header = {
 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}

def get_page(url):
 response = requests.get(url, headers=header)

 # 通過BeautifulSoup進(jìn)行解析出每個(gè)房源詳細(xì)列表并進(jìn)行打印
 soup_idex = BeautifulSoup(response.text, 'html.parser')
 result_li = soup_idex.find_all('li', {'class': 'list-item'})

 # 進(jìn)行循環(huán)遍歷其中的房源詳細(xì)列表
 for i in result_li:
 # 由于BeautifulSoup傳入的必須為字符串，所以進(jìn)行轉(zhuǎn)換
 page_url = str(i)
 soup = BeautifulSoup(page_url, 'html.parser')
 # 由于通過class解析的為一個(gè)列表，所以只需要第一個(gè)參數(shù)
 result_href = soup.find_all('a', {'class': 'houseListTitle'})[0]
 # 詳細(xì)頁面的函數(shù)調(diào)用
 get_page_detail(result_href.attrs['href'])


 # 進(jìn)行下一頁的爬取
 result_next_page = soup_idex.find_all('a', {'class': 'aNxt'})
 if len(result_next_page) != 0:
 # 函數(shù)進(jìn)行遞歸
 get_page(result_next_page[0].attrs['href'])
 else:
 print('沒有下一頁了')

# 進(jìn)行字符串中空格，換行，tab鍵的替換及刪除字符串兩邊的空格刪除
def my_strip(s):
 return str(s).replace(" ", "").replace("
", "").replace("	", "").strip()
# 由于頻繁進(jìn)行BeautifulSoup的使用，封裝一下，很雞肋
def my_Beautifulsoup(response):
 return BeautifulSoup(str(response), 'html.parser')



# 詳細(xì)頁面的爬取
def get_page_detail(url):
 response = requests.get(url, headers=header)
 if response.status_code == 200:
 soup = BeautifulSoup(response.text, 'html.parser')
 # 標(biāo)題什么的一大堆，哈哈
 result_title = soup.find_all('h3', {'class': 'long-title'})[0]
 result_price = soup.find_all('span', {'class': 'light info-tag'})[0]
 result_house_1 = soup.find_all('p', {'class': 'first-col detail-col'})
 result_house_2 = soup.find_all('p', {'class': 'second-col detail-col'})
 result_house_3 = soup.find_all('p', {'class': 'third-col detail-col'})
 soup_1 = my_Beautifulsoup(result_house_1)
 soup_2 = my_Beautifulsoup(result_house_2)
 soup_3 = my_Beautifulsoup(result_house_3)
 result_house_tar_1 = soup_1.find_all('dd')
 result_house_tar_2 = soup_2.find_all('dd')
 result_house_tar_3 = soup_3.find_all('dd')
 '''
 文博公寓，省實(shí)驗(yàn)中學(xué)，首付只需70萬，大三房，誠心賣，價(jià)可談 270萬
 宇泰文博公寓 金水－花園路－文博東路4號(hào) 2010年 普通住宅
 3室2廳2衛(wèi) 140平方米 南北 中層(共32層)
 精裝修 19285元/m? 81.00萬
 '''
 print(my_strip(result_title.text), my_strip(result_price.text))
 print(my_strip(result_house_tar_1[0].text),
 my_strip(my_Beautifulsoup(result_house_tar_1[1]).find_all('p')[0].text),
 my_strip(result_house_tar_1[2].text), my_strip(result_house_tar_1[3].text))
 print(my_strip(result_house_tar_2[0].text), my_strip(result_house_tar_2[1].text),
 my_strip(result_house_tar_2[2].text), my_strip(result_house_tar_2[3].text))
 print(my_strip(result_house_tar_3[0].text), my_strip(result_house_tar_3[1].text),
 my_strip(result_house_tar_3[2].text))

if __name__ == '__main__':
 # url鏈接
 url = 'https://zhengzhou.anjuke.com/sale/'
 # 頁面爬取函數(shù)調(diào)用
 get_page(url)