'''
# demo1 GET
import urllib.request as request
# urlopen
response = request.urlopen(url)
#print(response.read().decode('utf-8')) #解码 如中文 下载页面则不需要解码
#print(type(response)) #类型
print(response.status)
print(response.getheaders())
print(response.getheader('Server'))
'''
''' demo2 POST
import urllib.parse
import urllib.request
data = bytes(urllib.parse.urlencode({'word':'hello'}),encoding='utf-8')
response = urllib.request.urlopen('http://httpbin.org/post',data=data,timeout=1)
print(response.read())
'''
'''
# demo3
import socket
import urllib.request
import urllib.error
try:
response = urllib.request.urlopen('http://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
if isintance(e.reason,socket.timeout):
print('Time Out')
'''
'''
# demo4
from urllib import request
url = r'https://www.python.org'#转义
req = request.Request(url) #此方法可自定义其它参数 可对反爬
response = request.urlopen(req)
print(response.read().decode('utf-8'))
'''
'''
# demo5
# get cookie
import http.cookiejar, urllib.request
cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
for item in cookie:
print(item.name+"="+item.value)
'''
'''
# demo6 get cookie and save
import http.cookiejar, urllib.request
filename = 'cookie.txt'
cookie = http.cookiejar.MozillaCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)
'''
'''
# demo7 get cookie and save
import http.cookiejar, urllib.request
filename = 'cookie.txt'
cookie = http.cookiejar.LWPCookieJar()
# 加载本地cookie
cookie.load(filename,ignore_discard=True, ignore_expires=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))
'''
'''
# demo8 user-agent
from urllib import request
import random
import re
url = r'http://www.baidu.com'
agent1 = 'a'
agent2 = 'b'
agent = [agent1,agent2]
header = {
"user-agent": random.choice(agent)
}
#print(random.choice(agent))
req = request.Request(url,headers=header)
response = request.urlopen(req)
'''
'''
#demo9 自定义 opener
from urllib import request
#构建HTTP处理对象专门HTTP请求
handler = request.HTTPHandler()
#创建自定义opener
opener = request.build_opener(handler)
#创建自定义对象
req = request.Request(url)
#发送请求获取响应
#reponse = opener.open(req).read().decode() #只能发一次请求
#设置全局opener 完成多次请求 用urlopen发送请求也可以自定义opener
request.install_opener(opener)
reponse = request.urlopen(req).read()
print(reponse)
'''
'''
#demo10 proxy
from urllib import request
import random
#多个代理IP及端口 免费/收费
proxylist = [
{"http",ip1},
{"http",ip2},
{"http",ip...},
]
proxy = random.choice(proxylist)
#构建代理对象
handler = request.ProxyHandler(proxy)
#创建自定义opener
opener = request.build_opener(handler)
req = request.Request(url)
res = opener.open(req).read().decode()
'''
'''
#demo 11 GET请求
from urllib import request,parse
#分析网址传值 注意中文编码
wd = {"wd":"中文"}
#构造URL编码
wdd = parse.urlencode(wd)
url = "http://www.baidu.com/s?"+wdd
#同上
'''
'''
#requests 基本用法
import requests
#content二进制适合获取视频，图片 text字符串会中文乱码
response = requests.get('http://www.baidu.com').content.decode()
#与上同样
response = requests.request('get','http://www.baidu.com').content.decode()
print(response)
'''
'''
#传参
import requests
wd = {"wd":"中文"}
url = r'http://www.baidu.com/s?'
#伪浏览器
agent1 = 'a'
agent2 = 'b'
agent = [agent1,agent2]
header = {
"user-agent": random.choice(agent)
}
#get params=wd 中文省转义
response = requests.get(url,params=wd,headers=header)
#post
formdata = {
"":""
...
}
response = requests.post(url,headers=header,data=formdata)
print(response.json())
#多个代理IP及端口 免费/收费
proxylist = {
"http",ip1,
"https",ip2,
"http",ip...
}
proxy = random.choice(proxylist)
#proxies 单个多个都行
response = requests.get(url,proxies=proxylist)
print(response)
'''
'''
#获取cookie
import requests
req = requests.get("http://www.baidu.com")
#获取返回cookiejar对象
cookiejar = req.cookies
#将cookiejar转成字典
cookiedict = requests.utils.dict_from_cookiejar(cookiejar)
print(cookiedict)
'''
'''
#session 模拟登录
import requests
header = {
"user-agent": "xxx"
}
ses = requests.session()
data = {"user":"xxx","pwd":"xxx"}
#获取COOKIE
ses.post(url,data=data)
#请求页面
response = ses.get(newurl)
print(response)
'''
