python3小demo

总结常用的功能小实例,快速学习并掌握python技能

　　总结常用的功能小实例,快速学习并掌握python技能

1.墨迹天气

import requests
from lxml.html import etree
import json
import time # 导入模块
class MoJiWeather():
def city_name(self): # 定义一个输入城市名称的函数
cityname = str(input("输入城市名称："))
return cityname
def search_city(city_name):# 搜索这个城市
index_url = "http://tianqi.moji.com/api/citysearch/%s"%city_name # 构造查询相应城市天气的url
response = requests.get(index_url)
response.encoding = "utf-8"
try:# 异常捕获
city_id = json.loads(response.text).get('city_list')[0].get('cityId')# 通过上面的url获取城市的id
city_url = "http://tianqi.moji.com/api/redirect/%s"%str(city_id) # 通过城市id获取城市天气
return city_url
except:
print('城市名输入错误')
exit()
def parse(city_url):# 解析函数
response = requests.get(city_url)
response.encoding = 'utf-8'
html = etree.HTML(response.text)
current_city = html.xpath("//div[@class='search_default']/em/text()")[0]# 下面都是利用xpath解析的
print('当前城市：'+current_city)
current_kongqi = html.xpath("//div[@class='left']/div[@class='wea_alert clearfix']/ul/li/a/em/text()")[0]
print('空气质量：'+current_kongqi)
current_wendu = html.xpath("//div[@class='left']/div[@class='wea_weather clearfix']/em/text()")[0]
print('当前温度：'+current_wendu+'℃')
current_weather = html.xpath("//div[@class='wea_weather clearfix']/b/text()")[0]
print('天气状况：' + current_weather)
current_shidu = html.xpath("//div[@class='left']/div[@class='wea_about clearfix']/span/text()")[0]
print('当前湿度：'+current_shidu)
current_fengji = html.xpath("//div[@class='left']/div[@class='wea_about clearfix']/em/text()")[0]
print('当前风速：'+current_fengji)
jingdian = html.xpath("//div[@class='right']/div[@class='near'][2]/div[@class='item clearfix']/ul/li/a/text()")
print('附近景点：')
for j in jingdian:
print('\t\t'+j)
if __name__ == '__main__':
print("欢迎使用墨迹天气查询系统")
city_name = MoJiWeather.city_name(1)
city_url = MoJiWeather.search_city(city_name)
MoJiWeather.parse(city_url)
print("谢谢使用本查询系统")
input("按任意键退出...")

 

2.Tiobe排行榜

import json
from lxml import etree
from lxml.etree import ParseError
import requests
from requests.exceptions import RequestException

'''
lxml实例应用
'''

'''
获取页面数据
'''
def one_to_page(url):
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'
}

try:
res = requests.get(url,headers=headers)
body = res.text #获取网页内容
except RequestException as e:
print('request is error',e)

try:
html = etree.HTML(body,etree.HTMLParser())
# tr 下的所有子孙节点(只获取文本数,图片资源不获取)
result = html.xpath('//table[contains(@class,"table-top20")]/tbody/tr//text()')

pos = 0
for i in range(20):
if i == 0:
yield result[i:5]
else:
yield result[pos:pos+5]
pos += 5

except ParseError as e:
print(e.position)


'''
写入文件
'''
def write_file(data):
for item in data:
sul = {
'2018年6月排行': item[0],
'2017年6排行': item[1],
'开发语言': item[2],
'评级': item[3],
'变化率': item[4]
}
# with 更好处理异常情况,进行文件的关闭后续工作
with open('test.txt','a',encoding='utf-8') as f:
f.write(json.dumps(sul,ensure_ascii=False)+'\n')
f.close()
print(sul)
return None

'''
主程序
'''
def main():
url = 'https://www.tiobe.com/tiobe-index/'
data = one_to_page(url)
ret = write_file(data)
if ret == None:
print('ok')

if __name__ == '__main__':
main()

 

3.新闻列表

'''
墨迹天气文章爬虫
'''
import requests
import json
from lxml.html import etree
from lxml.etree import ParseError

'''
解析页面内容
'''
def parseHtml(content):
try:
html = etree.HTML(content,etree.HTMLParser())
# one = html.xpath('//ul[@class="advisory_list_item"]//text()')
one = html.xpath('//ul[@class="advisory_list_item"]//li/a/@href')
print(one)
exit(0)
LOOP = 8
pos = 0
for i in range(20):
if i == 0:
yield one[0:LOOP]
else:
yield one[pos:pos+LOOP]
pos += LOOP

except ParseError as e:
print(e.position)

'''
写入文件
'''
def write_log(data):
for item in data:
msg = {
'发文时间':item[3],
'文章标题':item[5]
}
with open('moji.log','a',encoding='utf-8') as f:
f.write(json.dumps(msg,ensure_ascii=False)+'\n')
f.close()
print(msg)
return None
'''
主程序
'''
def main():

for page in range(1,73):
url = 'https://tianqi.moji.com/news/list/moji/{}'.format(page)
res = requests.get(url)
res.encoding = 'utf-8'
content = parseHtml(res.text)
ret = write_log(content)
if ret is None:
print('ok')

if __name__ == '__main__':
main()

 

4.爬取IP

import requests
import re
import random

from bs4 import BeautifulSoup

ua_list = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
"Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 45.0.2454.101Safari / 537.36"
]


def ip_parse_xici(page):
"""

:param page: 采集的页数
:return:
"""
ip_list = []
for pg in range(1, int(page)):
url = 'http://www.xicidaili.com/nn/' + str(pg)
user_agent = random.choice(ua_list)
my_headers = {
'Accept': 'text/html, application/xhtml+xml, application/xml;',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Referer': 'http: // www.xicidaili.com/nn',
'User-Agent': user_agent
}
try:
r = requests.get(url, headers=my_headers)
soup = BeautifulSoup(r.text, 'html.parser')
except requests.exceptions.ConnectionError:
print('ConnectionError')
else:
data = soup.find_all('td')
# 定义IP和端口Pattern规则
ip_compile = re.compile(r'(\d+\.\d+\.\d+\.\d+)') # 匹配IP
port_compile = re.compile(r'(\d+)') # 匹配端口
ips = re.findall(ip_compile, str(data)) # 获取所有IP

ports = re.findall(port_compile, str(data)) # 获取所有端口
check_api = "http://ip.taobao.com/service/getIpInfo2.php?ip="

for i in range(len(ips)):
if i < len(ips):
ip = ips[i]
api = check_api + ip
api_headers = {
'User-Agent': user_agent
}
try:
response = requests.get(url=api, headers=api_headers, timeout=2)
print("ip：%s 可用" % ip)
except Exception as e:
print("此ip %s 已失效：%s" % (ip, e))
del ips[i]
del ports[i]
ips_usable = ips
ip_list += [':'.join(n) for n in zip(ips_usable, ports)] # 列表生成式
print('第{}页ip采集完成'.format(pg))
print(ip_list)


if __name__ == '__main__':
xici_pg = input("请输入需要采集的页数：")
ip_parse_xici(page=xici_pg)