#coding utf-8
import requests
import re
import pandas
from bs4 import BeautifulSoup
user_agent = 'User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)'
headers = {'User-Agent':user_agent}
#定义函数,得到每日报的链接,并以列表形式返回
def get_newsurl():
newsurl=[]
url1='http://www.p2p001.com/licai/index/id/147/p/'
num=1
url2='.html'
while num<=22:
url=url1+str(num)+url2
try:
r1=requests.get(url,headers=headers)
except:
print ('wrong %s' % url)
else:
s1=BeautifulSoup(r1.text,'lxml')
for x in s1.find_all(href=re.compile('licai/shownews')):
newsurl.append(x['href'])
num=num+1
return newsurl
#定义函数,得到的数据,以字典形式返回
def get_info():
url='http://www.p2p001.com'
date=[]
zonghe=[]
one=[]
one_three=[]
three_six=[]
six_twelve=[]
twelve_most=[]
for y in get_newsurl():
try:
main_url=url+y
r2=requests.get(main_url,headers=headers)
except:
print ('wrong %s' % main_url)
else:
s2=BeautifulSoup(r2.text,'lxml')
date.append(s2.find(text=re.compile('统计日期'))[5:])
rate=s2.find_all('td')
zonghe.append(rate[10].string)
one.append(rate[11].string)
one_three.append(rate[12].string)
three_six.append(rate[13].string)
six_twelve.append(rate[14].string)
twelve_most.append(rate[15].string)
p={'Date':date,
'综合':zonghe,
'1个月':one,
'1-3个月':one_three,
'3-6个月':three_six,
'6-12个月':six_twelve,
'12个月及以上':twelve_most}
return p
#pandas存储数据
p=pd.DataFrame(get_info())
p.to_csv('f://1//rate1.csv',index=False,
columns=['Date','综合','1个月','1-3个月','3-6个月','6-12个月','12个月及以上'],
header=['Date','综合','1个月','1-3个月','3-6个月','6-12个月','12个月及以上'])