PyProject/proxyIP/Req.py at master · javaGe/PyProject · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# -*- coding=utf-8 -*-
import random

import requests
import time

from proxyIP import get_IP
from proxyIP import check_IP
from meizituSpider import user_agent
import re


class Req():
    def __init__(self):
        '''
        初始化代理IP列表和用户代理
        '''
        # get_IP.getIP(1)
        # check_IP.check()
        self.iplist = []  # 初始化一个IP列表
        with open('valid_ip.csv', 'r', encoding='utf-8') as r:
            lines = r.readlines()  # 读取所有行
            for line in lines:
                ip = re.sub(r'\n', '', line) #去除换行
                if ip == '':
                    continue
                self.iplist.append(ip) #将有效IP加到列表中
                # print(self.iplist)
        self.user_agent_list = user_agent.UA  # 用户代理列表

    def get(self, url, timeout, proxy=None, num_req=6):
        '''
        :param url: 请求链接
        :param timeout:  超时时间
        :param proxy:  代理IP 默认为None
        :param num_req: 请求次数
        :return:
        '''
        UA = random.choice(self.user_agent_list)  # 获取随机用户代理
        headers = {'User-Agent': UA}  # 构造代理头
        if proxy == None:  # 代理为空时
            try:
                return requests.get(url, headers=headers, timeout=timeout)
            except:  # 请求出现异常时，继续按照设定的次数进行请求
                if num_req > 0:
                    time.sleep(5)
                    print('获取页面出错，剩余请求次数%s次' % (num_req))
                    return self.get(url, timeout, num_req=num_req - 1)  # 继续调用请求
                else:
                    print('使用代理。。。')
                    time.sleep(5)
                    IP = ''.join(str(random.choice(self.iplist)).strip())
                    proxy = {'http': IP}
                    return self.get(url, timeout, proxy)
        else:  # 代理不为空时
            try:
                # IP = ''.join(str(random.choice(self.iplist)).strip())
                # proxy = {'http': IP}
                return requests.get(url, headers=headers, proxys=proxy, timeout=timeout)
            except: #更换代理
                if num_req > 0:
                    time.sleep(5)
                    IP = ''.join(str(random.choice(self.iplist)).strip())
                    proxy = {"http": IP}
                    print('剩余请求%s次' %(num_req))
                    print('更换代理为：%s' %(IP))
                    return self.get(url, timeout, proxy, num_req-1)
                else: #退出代理
                    print('网站封了你，代理也不好使，自己看着办。。。')
                    return

request = Req() #实例化
# print(dw.get('http://mzitu.com', 3))