From 865a13e06cbb0baf28d4d7e44dd065b49b8eff08 Mon Sep 17 00:00:00 2001 From: enuiai Date: Mon, 19 Nov 2018 22:56:50 +0800 Subject: [PATCH 001/139] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E6=95=88?= =?UTF-8?q?=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/crawler.py | 96 +------------------------------------------- 1 file changed, 1 insertion(+), 95 deletions(-) diff --git a/proxypool/crawler.py b/proxypool/crawler.py index 273df049..4f5f5692 100644 --- a/proxypool/crawler.py +++ b/proxypool/crawler.py @@ -23,15 +23,7 @@ def get_proxies(self, callback): print('成功获取到代理', proxy) proxies.append(proxy) return proxies - - # def crawl_daxiang(self): - # url = 'http://vtp.daxiangdaili.com/ip/?tid=559363191592228&num=50&filter=on' - # html = get_page(url) - # if html: - # urls = html.split('\n') - # for url in urls: - # yield url - + def crawl_daili66(self, page_count=4): """ 获取代理66 @@ -51,47 +43,6 @@ def crawl_daili66(self, page_count=4): port = tr.find('td:nth-child(2)').text() yield ':'.join([ip, port]) - def crawl_proxy360(self): - """ - 获取Proxy360 - :return: 代理 - """ - start_url = 'http://www.proxy360.cn/Region/China' - print('Crawling', start_url) - html = get_page(start_url) - if html: - doc = pq(html) - lines = doc('div[name="list_proxy_ip"]').items() - for line in lines: - ip = line.find('.tbBottomLine:nth-child(1)').text() - port = line.find('.tbBottomLine:nth-child(2)').text() - yield ':'.join([ip, port]) - - def crawl_goubanjia(self): - """ - 获取Goubanjia - :return: 代理 - """ - start_url = 'http://www.goubanjia.com/free/gngn/index.shtml' - html = get_page(start_url) - if html: - doc = pq(html) - tds = doc('td.ip').items() - for td in tds: - td.find('p').remove() - yield td.text().replace(' ', '') - - def crawl_ip181(self): - start_url = 'http://www.ip181.com/' - html = get_page(start_url) - ip_address = re.compile('\s*(.*?)\s*(.*?)') - # \s* 匹配空格，起到换行作用 - re_ip_address = ip_address.findall(html) - for address,port in re_ip_address: - result = address + ':' + port - yield result.replace(' ', '') - - def crawl_ip3366(self): for page in range(1, 4): start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page) @@ -102,42 +53,6 @@ def crawl_ip3366(self): for address, port in re_ip_address: result = address+':'+ port yield result.replace(' ', '') - - - def crawl_kxdaili(self): - for i in range(1, 11): - start_url = 'http://www.kxdaili.com/ipList/{}.html#ip'.format(i) - html = get_page(start_url) - ip_address = re.compile('\s*(.*?)\s*(.*?)') - # \s* 匹配空格，起到换行作用 - re_ip_address = ip_address.findall(html) - for address, port in re_ip_address: - result = address + ':' + port - yield result.replace(' ', '') - - - def crawl_premproxy(self): - for i in ['China-01','China-02','China-03','China-04','Taiwan-01']: - start_url = 'https://premproxy.com/proxy-by-country/{}.htm'.format(i) - html = get_page(start_url) - if html: - ip_address = re.compile('(.*?)') - re_ip_address = ip_address.findall(html) - for address_port in re_ip_address: - yield address_port.replace(' ','') - - def crawl_xroxy(self): - for i in ['CN','TW']: - start_url = 'http://www.xroxy.com/proxylist.php?country={}'.format(i) - html = get_page(start_url) - if html: - ip_address1 = re.compile("title='View this Proxy details'>\s*(.*).*") - re_ip_address1 = ip_address1.findall(html) - ip_address2 = re.compile("title='Select proxies with port number .*'>(.*)") - re_ip_address2 = ip_address2.findall(html) - for address,port in zip(re_ip_address1,re_ip_address2): - address_port = address+':'+port - yield address_port.replace(' ','') def crawl_kuaidaili(self): for i in range(1, 4): @@ -206,15 +121,6 @@ def crawl_iphai(self): address_port = address+':'+port yield address_port.replace(' ','') - def crawl_89ip(self): - start_url = 'http://www.89ip.cn/apijk/?&tqsl=1000&sxa=&sxb=&tta=&ports=&ktip=&cf=1' - html = get_page(start_url) - if html: - find_ips = re.compile('(\d+\.\d+\.\d+\.\d+:\d+)', re.S) - ip_ports = find_ips.findall(html) - for address_port in ip_ports: - yield address_port - def crawl_data5u(self): start_url = 'http://www.data5u.com/free/gngn/index.shtml' headers = { From e2675c0a3fc50422e41e17e1e677da4734a09e02 Mon Sep 17 00:00:00 2001 From: DemSephiroth <1206760823@qq.com> Date: Fri, 19 Jul 2019 18:06:45 +0800 Subject: [PATCH 002/139] Update db.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本redis还原注释掉的原代码 --- proxypool/db.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/proxypool/db.py b/proxypool/db.py index bf108f77..fbbd3f5c 100644 --- a/proxypool/db.py +++ b/proxypool/db.py @@ -27,7 +27,9 @@ def add(self, proxy, score=INITIAL_SCORE): print('代理不符合规范', proxy, '丢弃') return if not self.db.zscore(REDIS_KEY, proxy): - return self.db.zadd(REDIS_KEY, score, proxy) + #return self.db.zadd(REDIS_KEY, score, proxy) + #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 + return self.db.zadd(REDIS_KEY, {proxy:score}) def random(self): """ @@ -53,7 +55,9 @@ def decrease(self, proxy): score = self.db.zscore(REDIS_KEY, proxy) if score and score > MIN_SCORE: print('代理', proxy, '当前分数', score, '减1') - return self.db.zincrby(REDIS_KEY, proxy, -1) + #return self.db.zincrby(REDIS_KEY, proxy, -1) + #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 + return self.db.zincrby(REDIS_KEY, -1, proxy) else: print('代理', proxy, '当前分数', score, '移除') return self.db.zrem(REDIS_KEY, proxy) @@ -73,7 +77,9 @@ def max(self, proxy): :return: 设置结果 """ print('代理', proxy, '可用，设置为', MAX_SCORE) - return self.db.zadd(REDIS_KEY, MAX_SCORE, proxy) + #return self.db.zadd(REDIS_KEY, MAX_SCORE, proxy) + #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 + return self.db.zadd(REDIS_KEY, {proxy:MAX_SCORE}) def count(self): """ From a98c2b7a95f355ac3417d130ffafba5078e77107 Mon Sep 17 00:00:00 2001 From: muyun001 <1219258598@qq.com> Date: Tue, 17 Dec 2019 09:28:03 +0800 Subject: [PATCH 003/139] =?UTF-8?q?=E4=BF=AE=E6=94=B9example=E4=B8=BE?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example.py b/examples/example.py index 8a50b7d5..27a2cf5b 100644 --- a/examples/example.py +++ b/examples/example.py @@ -8,7 +8,7 @@ def get_proxy(): - r = requests.get('http://127.0.0.1:5000/get') + r = requests.get('http://127.0.0.1:5555/random') proxy = BeautifulSoup(r.text, "lxml").get_text() return proxy @@ -24,6 +24,6 @@ def main(): html = crawl('http://docs.jinkan.org/docs/flask/', proxy) print(html) + if __name__ == '__main__': main() - From 39bcf6b48f3eab2c122bd9ef3d74d34baad0a98b Mon Sep 17 00:00:00 2001 From: Germey Date: Wed, 19 Feb 2020 04:52:39 +0800 Subject: [PATCH 004/139] update --- examples/example.py | 29 ------ examples/proxytest.py | 15 --- proxypool/api.py | 17 ++-- proxypool/crawler.py | 147 ----------------------------- proxypool/crawlers/__init__.py | 10 ++ proxypool/crawlers/base.py | 27 ++++++ proxypool/crawlers/daili66.py | 31 +++++++ proxypool/crawlers/ip3366.py | 31 +++++++ proxypool/crawlers/iphai.py | 34 +++++++ proxypool/db.py | 164 ++++++++++++++++++--------------- proxypool/error.py | 10 +- proxypool/getter.py | 54 +++++++---- proxypool/importer.py | 22 ----- proxypool/proxy.py | 22 +++++ proxypool/setting.py | 102 ++++++++++++-------- proxypool/utils.py | 59 ++++++++---- 16 files changed, 397 insertions(+), 377 deletions(-) delete mode 100644 examples/example.py delete mode 100644 examples/proxytest.py delete mode 100644 proxypool/crawler.py create mode 100644 proxypool/crawlers/__init__.py create mode 100644 proxypool/crawlers/base.py create mode 100644 proxypool/crawlers/daili66.py create mode 100644 proxypool/crawlers/ip3366.py create mode 100644 proxypool/crawlers/iphai.py delete mode 100644 proxypool/importer.py create mode 100644 proxypool/proxy.py diff --git a/examples/example.py b/examples/example.py deleted file mode 100644 index 27a2cf5b..00000000 --- a/examples/example.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import sys -import requests -from bs4 import BeautifulSoup - -dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, dir) - - -def get_proxy(): - r = requests.get('http://127.0.0.1:5555/random') - proxy = BeautifulSoup(r.text, "lxml").get_text() - return proxy - - -def crawl(url, proxy): - proxies = {'http': proxy} - r = requests.get(url, proxies=proxies) - return r.text - - -def main(): - proxy = get_proxy() - html = crawl('http://docs.jinkan.org/docs/flask/', proxy) - print(html) - - -if __name__ == '__main__': - main() diff --git a/examples/proxytest.py b/examples/proxytest.py deleted file mode 100644 index 6d809125..00000000 --- a/examples/proxytest.py +++ /dev/null @@ -1,15 +0,0 @@ -import requests -from proxypool.setting import TEST_URL - -proxy = '96.9.90.90:8080' - -proxies = { - 'http': 'http://' + proxy, - 'https': 'https://' + proxy, -} - -print(TEST_URL) -response = requests.get(TEST_URL, proxies=proxies, verify=False) -if response.status_code == 200: - print('Successfully') - print(response.text) \ No newline at end of file diff --git a/proxypool/api.py b/proxypool/api.py index 9d9885ec..712a5df1 100644 --- a/proxypool/api.py +++ b/proxypool/api.py @@ -1,42 +1,41 @@ from flask import Flask, g - from .db import RedisClient +from .setting import API_HOST, API_PORT __all__ = ['app'] app = Flask(__name__) - def get_conn(): + """ + get redis client object + :return: + """ if not hasattr(g, 'redis'): g.redis = RedisClient() return g.redis - @app.route('/') def index(): return '

Welcome to Proxy Pool System

' - @app.route('/random') def get_proxy(): """ - Get a proxy + get a random proxy :return: 随机代理 """ conn = get_conn() return conn.random() - @app.route('/count') def get_counts(): """ - Get the count of proxies + get the count of proxies :return: 代理池总量 """ conn = get_conn() return str(conn.count()) - if __name__ == '__main__': - app.run() + app.run(host=API_HOST, port=API_PORT, threaded=True) diff --git a/proxypool/crawler.py b/proxypool/crawler.py deleted file mode 100644 index 4f5f5692..00000000 --- a/proxypool/crawler.py +++ /dev/null @@ -1,147 +0,0 @@ -import json -import re -from .utils import get_page -from pyquery import PyQuery as pq - - -class ProxyMetaclass(type): - def __new__(cls, name, bases, attrs): - count = 0 - attrs['__CrawlFunc__'] = [] - for k, v in attrs.items(): - if 'crawl_' in k: - attrs['__CrawlFunc__'].append(k) - count += 1 - attrs['__CrawlFuncCount__'] = count - return type.__new__(cls, name, bases, attrs) - - -class Crawler(object, metaclass=ProxyMetaclass): - def get_proxies(self, callback): - proxies = [] - for proxy in eval("self.{}()".format(callback)): - print('成功获取到代理', proxy) - proxies.append(proxy) - return proxies - - def crawl_daili66(self, page_count=4): - """ - 获取代理66 - :param page_count: 页码 - :return: 代理 - """ - start_url = 'http://www.66ip.cn/{}.html' - urls = [start_url.format(page) for page in range(1, page_count + 1)] - for url in urls: - print('Crawling', url) - html = get_page(url) - if html: - doc = pq(html) - trs = doc('.containerbox table tr:gt(0)').items() - for tr in trs: - ip = tr.find('td:nth-child(1)').text() - port = tr.find('td:nth-child(2)').text() - yield ':'.join([ip, port]) - - def crawl_ip3366(self): - for page in range(1, 4): - start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page) - html = get_page(start_url) - ip_address = re.compile('\s*(.*?)\s*(.*?)') - # \s * 匹配空格，起到换行作用 - re_ip_address = ip_address.findall(html) - for address, port in re_ip_address: - result = address+':'+ port - yield result.replace(' ', '') - - def crawl_kuaidaili(self): - for i in range(1, 4): - start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i) - html = get_page(start_url) - if html: - ip_address = re.compile('(.*?)') - re_ip_address = ip_address.findall(html) - port = re.compile('(.*?)') - re_port = port.findall(html) - for address,port in zip(re_ip_address, re_port): - address_port = address+':'+port - yield address_port.replace(' ','') - - def crawl_xicidaili(self): - for i in range(1, 3): - start_url = 'http://www.xicidaili.com/nn/{}'.format(i) - headers = { - 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'Cookie':'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRjYzc5MmM1MTBiMDMzYTUzNTZjNzA4NjBhNWRjZjliBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUp6S2tXT3g5a0FCT01ndzlmWWZqRVJNek1WanRuUDBCbTJUN21GMTBKd3M9BjsARg%3D%3D--2a69429cb2115c6a0cc9a86e0ebe2800c0d471b3', - 'Host':'www.xicidaili.com', - 'Referer':'http://www.xicidaili.com/nn/3', - 'Upgrade-Insecure-Requests':'1', - } - html = get_page(start_url, options=headers) - if html: - find_trs = re.compile('(.*?)', re.S) - trs = find_trs.findall(html) - for tr in trs: - find_ip = re.compile('(\d+\.\d+\.\d+\.\d+)') - re_ip_address = find_ip.findall(tr) - find_port = re.compile('(\d+)') - re_port = find_port.findall(tr) - for address,port in zip(re_ip_address, re_port): - address_port = address+':'+port - yield address_port.replace(' ','') - - def crawl_ip3366(self): - for i in range(1, 4): - start_url = 'http://www.ip3366.net/?stype=1&page={}'.format(i) - html = get_page(start_url) - if html: - find_tr = re.compile('(.*?)', re.S) - trs = find_tr.findall(html) - for s in range(1, len(trs)): - find_ip = re.compile('(\d+\.\d+\.\d+\.\d+)') - re_ip_address = find_ip.findall(trs[s]) - find_port = re.compile('(\d+)') - re_port = find_port.findall(trs[s]) - for address,port in zip(re_ip_address, re_port): - address_port = address+':'+port - yield address_port.replace(' ','') - - def crawl_iphai(self): - start_url = 'http://www.iphai.com/' - html = get_page(start_url) - if html: - find_tr = re.compile('(.*?)', re.S) - trs = find_tr.findall(html) - for s in range(1, len(trs)): - find_ip = re.compile('\s+(\d+\.\d+\.\d+\.\d+)\s+', re.S) - re_ip_address = find_ip.findall(trs[s]) - find_port = re.compile('\s+(\d+)\s+', re.S) - re_port = find_port.findall(trs[s]) - for address,port in zip(re_ip_address, re_port): - address_port = address+':'+port - yield address_port.replace(' ','') - - def crawl_data5u(self): - start_url = 'http://www.data5u.com/free/gngn/index.shtml' - headers = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', - 'Cache-Control': 'max-age=0', - 'Connection': 'keep-alive', - 'Cookie': 'JSESSIONID=47AA0C887112A2D83EE040405F837A86', - 'Host': 'www.data5u.com', - 'Referer': 'http://www.data5u.com/free/index.shtml', - 'Upgrade-Insecure-Requests': '1', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36', - } - html = get_page(start_url, options=headers) - if html: - ip_address = re.compile('
  • (\d+\.\d+\.\d+\.\d+)
    • .*?
  • (\d+)
    • ', re.S) - re_ip_address = ip_address.findall(html) - for address, port in re_ip_address: - result = address + ':' + port - yield result.replace(' ', '') - - - \ No newline at end of file diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py new file mode 100644 index 00000000..bb690937 --- /dev/null +++ b/proxypool/crawlers/__init__.py @@ -0,0 +1,10 @@ +from .daili66 import Daili66Crawler +from .ip3366 import IP3366Crawler +from .iphai import IPHaiCrawler + + +__all__ = [ + Daili66Crawler, + IP3366Crawler, + IP3366Crawler +] diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py new file mode 100644 index 00000000..84a0c94c --- /dev/null +++ b/proxypool/crawlers/base.py @@ -0,0 +1,27 @@ +from retrying import retry +import requests +from loguru import logger + + +class BaseCrawler(object): + urls = [] + + @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None) + def fetch(self, url, **kwargs): + try: + response = requests.get(url, **kwargs) + if response.status_code == 200: + return response.text + except requests.ConnectionError: + return + + @logger.catch + def crawl(self): + """ + crawl main method + """ + for url in self.urls: + logger.info(f'fetching {url}') + html = self.fetch(url) + for proxy in self.parse(html): + yield proxy diff --git a/proxypool/crawlers/daili66.py b/proxypool/crawlers/daili66.py new file mode 100644 index 00000000..c91e64d2 --- /dev/null +++ b/proxypool/crawlers/daili66.py @@ -0,0 +1,31 @@ +from pyquery import PyQuery as pq +from proxypool.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler + + +BASE_URL = 'http://www.636ip.cn/{page}.html' +MAX_PAGE = 5 + +class Daili66Crawler(BaseCrawler): + """ + daili66 crawler, http://www.66ip.cn/1.html + """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] + + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('.containerbox table tr:gt(0)').items() + for tr in trs: + host = tr.find('td:nth-child(1)').text() + port = int(tr.find('td:nth-child(2)').text()) + yield Proxy(host=host, port=port) + +if __name__ == '__main__': + crawler = Daili66Crawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/ip3366.py b/proxypool/crawlers/ip3366.py new file mode 100644 index 00000000..d9673892 --- /dev/null +++ b/proxypool/crawlers/ip3366.py @@ -0,0 +1,31 @@ +from proxypool.crawlers.base import BaseCrawler +from proxypool.proxy import Proxy +import re + + +MAX_PAGE = 5 +BASE_URL = 'http://www.ip3366.net/free/?stype=1&page={page}' + +class IP3366Crawler(BaseCrawler): + """ + ip3366 crawler, http://www.ip3366.net/ + """ + urls = [BASE_URL.format(page=i) for i in range(1, 8)] + + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + ip_address = re.compile('\s*(.*?)\s*(.*?)') + # \s * 匹配空格，起到换行作用 + re_ip_address = ip_address.findall(html) + for address, port in re_ip_address: + proxy = Proxy(host=address.strip(), port=int(port.strip())) + yield proxy + +if __name__ == '__main__': + crawler = IP3366Crawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/iphai.py b/proxypool/crawlers/iphai.py new file mode 100644 index 00000000..59d892ad --- /dev/null +++ b/proxypool/crawlers/iphai.py @@ -0,0 +1,34 @@ +from proxypool.crawlers.base import BaseCrawler +from proxypool.proxy import Proxy +import re + + +BASE_URL = 'http://www.iphai.com/' + +class IPHaiCrawler(BaseCrawler): + """ + iphai crawler, http://www.iphai.com/ + """ + urls = [BASE_URL] + + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + find_tr = re.compile('(.*?)', re.S) + trs = find_tr.findall(html) + for s in range(1, len(trs)): + find_ip = re.compile('\s+(\d+\.\d+\.\d+\.\d+)\s+', re.S) + re_ip_address = find_ip.findall(trs[s]) + find_port = re.compile('\s+(\d+)\s+', re.S) + re_port = find_port.findall(trs[s]) + for address, port in zip(re_ip_address, re_port): + proxy = Proxy(host=address.strip(), port=int(port.strip())) + yield proxy + +if __name__ == '__main__': + crawler = IPHaiCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/db.py b/proxypool/db.py index fbbd3f5c..3b47ce85 100644 --- a/proxypool/db.py +++ b/proxypool/db.py @@ -1,111 +1,131 @@ import redis from proxypool.error import PoolEmptyError +from proxypool.proxy import Proxy from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY -from proxypool.setting import MAX_SCORE, MIN_SCORE, INITIAL_SCORE +from proxypool.setting import PROXY_SCORE_MAX, PROXY_SCORE_MIN, PROXY_SCORE_INIT from random import choice -import re +from typing import List +from loguru import logger + +from proxypool.utils import is_valid_proxy, convert_proxy_or_proxies + + +REDIS_CLIENT_VERSION = redis.__version__ +IS_REDIS_VERSION_2 = REDIS_CLIENT_VERSION.startswith('2.') class RedisClient(object): - def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD): + """ + redis connection client of proxypool + """ + + def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, **kwargs): """ - 初始化 - :param host: Redis 地址 - :param port: Redis 端口 - :param password: Redis密码 + init redis client + :param host: redis host + :param port: redis port + :param password: redis password """ - self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=True) + self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=True, **kwargs) - def add(self, proxy, score=INITIAL_SCORE): + def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: """ - 添加代理，设置分数为最高 - :param proxy: 代理 - :param score: 分数 - :return: 添加结果 + add proxy and set it to init score + :param proxy: proxy, ip:port, like 8.8.8.8:88 + :param score: int score + :return: result """ - if not re.match('\d+\.\d+\.\d+\.\d+\:\d+', proxy): - print('代理不符合规范', proxy, '丢弃') + if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): + logger.info(f'invalid proxy {proxy}, throw it') return if not self.db.zscore(REDIS_KEY, proxy): - #return self.db.zadd(REDIS_KEY, score, proxy) - #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 - return self.db.zadd(REDIS_KEY, {proxy:score}) + if IS_REDIS_VERSION_2: + return self.db.zadd(REDIS_KEY, score, proxy) + return self.db.zadd(REDIS_KEY, {proxy: score}) - def random(self): - """ - 随机获取有效代理，首先尝试获取最高分数代理，如果不存在，按照排名获取，否则异常 - :return: 随机代理 - """ - result = self.db.zrangebyscore(REDIS_KEY, MAX_SCORE, MAX_SCORE) - if len(result): - return choice(result) - else: - result = self.db.zrevrange(REDIS_KEY, 0, 100) - if len(result): - return choice(result) - else: - raise PoolEmptyError + def random(self) -> Proxy: + """ + get random proxy + firstly try to get proxy with max score + if not exists, try to get proxy by rank + if not exists, raise error + :return: proxy, like 8.8.8.8:8 + """ + # try to get proxy with max score + proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + if len(proxies): + return convert_proxy_or_proxies(choice(proxies)) + # else get proxy by rank + proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + if len(proxies): + return convert_proxy_or_proxies(choice(proxies)) + # else raise error + raise PoolEmptyError - def decrease(self, proxy): - """ - 代理值减一分，小于最小值则删除 - :param proxy: 代理 - :return: 修改后的代理分数 - """ - score = self.db.zscore(REDIS_KEY, proxy) - if score and score > MIN_SCORE: - print('代理', proxy, '当前分数', score, '减1') - #return self.db.zincrby(REDIS_KEY, proxy, -1) - #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 - return self.db.zincrby(REDIS_KEY, -1, proxy) + def decrease(self, proxy: Proxy) -> int: + """ + decrease score of proxy, if small than PROXY_SCORE_MIN, delete it + :param proxy: proxy + :return: new score + """ + score = self.db.zscore(REDIS_KEY, proxy.string()) + # current score is larger than PROXY_SCORE_MIN + if score and score > PROXY_SCORE_MIN: + logger.info(f'{proxy.string()} current score {score}, decrease 1') + if IS_REDIS_VERSION_2: + return self.db.zincrby(REDIS_KEY, proxy.string(), -1) + return self.db.zincrby(REDIS_KEY, -1, proxy.string()) + # otherwise delete proxy else: - print('代理', proxy, '当前分数', score, '移除') + logger.info(f'{proxy.string()} current score {score}, remove') return self.db.zrem(REDIS_KEY, proxy) - def exists(self, proxy): + def exists(self, proxy: Proxy) -> bool: """ - 判断是否存在 - :param proxy: 代理 - :return: 是否存在 + if proxy exists + :param proxy: proxy + :return: if exists, bool """ - return not self.db.zscore(REDIS_KEY, proxy) == None + return not self.db.zscore(REDIS_KEY, proxy.string()) == None - def max(self, proxy): + def max(self, proxy: Proxy) -> int: """ - 将代理设置为MAX_SCORE - :param proxy: 代理 - :return: 设置结果 + set proxy to max score + :param proxy: proxy + :return: new score """ - print('代理', proxy, '可用，设置为', MAX_SCORE) - #return self.db.zadd(REDIS_KEY, MAX_SCORE, proxy) - #更新为redis3.0+版本，解决redis3.0更新后的报错，如用旧版本还原上方代码 - return self.db.zadd(REDIS_KEY, {proxy:MAX_SCORE}) + logger.info(f'{proxy.string()} is valid, set to {PROXY_SCORE_MAX}') + if IS_REDIS_VERSION_2: + return self.db.zadd(REDIS_KEY, PROXY_SCORE_MAX, proxy.string()) + return self.db.zadd(REDIS_KEY, {proxy.string(): PROXY_SCORE_MAX}) - def count(self): + def count(self) -> int: """ - 获取数量 - :return: 数量 + get count of proxies + :return: count, int """ return self.db.zcard(REDIS_KEY) - def all(self): + def all(self) -> List[Proxy]: """ - 获取全部代理 - :return: 全部代理列表 + get all proxies + :return: list of proxies """ - return self.db.zrangebyscore(REDIS_KEY, MIN_SCORE, MAX_SCORE) + return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) - def batch(self, start, stop): + def batch(self, start, end) -> List[Proxy]: """ - 批量获取 - :param start: 开始索引 - :param stop: 结束索引 - :return: 代理列表 + get batch of proxies + :param start: start index + :param end: end index + :return: list of proxies """ - return self.db.zrevrange(REDIS_KEY, start, stop - 1) + return convert_proxy_or_proxies(self.db.zrevrange(REDIS_KEY, start, end - 1)) if __name__ == '__main__': conn = RedisClient() - result = conn.batch(680, 688) + result = conn.batch(1, 10) print(result) + result = conn.random() + print(result, type(result)) diff --git a/proxypool/error.py b/proxypool/error.py index ca19569e..22ac1df3 100644 --- a/proxypool/error.py +++ b/proxypool/error.py @@ -1,7 +1,7 @@ class PoolEmptyError(Exception): - - def __init__(self): - Exception.__init__(self) - def __str__(self): - return repr('代理池已经枯竭') + """ + proxypool is used out + :return: + """ + return repr('no proxy in proxypool') diff --git a/proxypool/getter.py b/proxypool/getter.py index 7af9b5c0..5787e19f 100644 --- a/proxypool/getter.py +++ b/proxypool/getter.py @@ -1,30 +1,44 @@ -from proxypool.tester import Tester from proxypool.db import RedisClient -from proxypool.crawler import Crawler -from proxypool.setting import * -import sys +from proxypool.setting import PROXY_NUMBER_MAX +from proxypool.crawlers import __all__ as crawlers_cls + class Getter(): + """ + getter of proxypool + """ + + def __init__(self): + """ + init db and crawlers + """ self.redis = RedisClient() - self.crawler = Crawler() + self.crawlers_cls = crawlers_cls + self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] + - def is_over_threshold(self): + def is_full(self): """ - 判断是否达到了代理池限制 + if proxypool if full + return: bool """ - if self.redis.count() >= POOL_UPPER_THRESHOLD: - return True - else: - return False + return self.redis.count() >= PROXY_NUMBER_MAX + def run(self): - print('获取器开始执行') - if not self.is_over_threshold(): - for callback_label in range(self.crawler.__CrawlFuncCount__): - callback = self.crawler.__CrawlFunc__[callback_label] - # 获取代理 - proxies = self.crawler.get_proxies(callback) - sys.stdout.flush() - for proxy in proxies: - self.redis.add(proxy) + """ + run crawlers to get proxy + :return: + """ + if self.is_full(): + return + for crawler in self.crawlers: + for proxy in crawler.crawl(): + print('proxy', proxy) + self.redis.add(proxy) + + +if __name__ == '__main__': + getter = Getter() + getter.run() \ No newline at end of file diff --git a/proxypool/importer.py b/proxypool/importer.py deleted file mode 100644 index d9651bb2..00000000 --- a/proxypool/importer.py +++ /dev/null @@ -1,22 +0,0 @@ -from proxypool.db import RedisClient - -conn = RedisClient() - - -def set(proxy): - result = conn.add(proxy) - print(proxy) - print('录入成功' if result else '录入失败') - - -def scan(): - print('请输入代理, 输入exit退出读入') - while True: - proxy = input() - if proxy == 'exit': - break - set(proxy) - - -if __name__ == '__main__': - scan() diff --git a/proxypool/proxy.py b/proxypool/proxy.py new file mode 100644 index 00000000..543ad7bc --- /dev/null +++ b/proxypool/proxy.py @@ -0,0 +1,22 @@ +from attr import attrs, attr + + +@attrs +class Proxy(object): + """ + proxy schema + """ + host = attr(type=str, default=None) + port = attr(type=int, default=None) + + def __str__(self): + return f'{self.host}:{self.port}' + + def string(self): + return self.__str__() + + +if __name__ == '__main__': + proxy = Proxy(host='8.8.8.8', port=8888) + print('proxy', proxy) + print('proxy', proxy.string()) diff --git a/proxypool/setting.py b/proxypool/setting.py index 3e4ed901..97173950 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -1,40 +1,62 @@ -# Redis数据库地址 -REDIS_HOST = '127.0.0.1' - -# Redis端口 -REDIS_PORT = 6379 - -# Redis密码，如无填None -REDIS_PASSWORD = None - -REDIS_KEY = 'proxies' - -# 代理分数 -MAX_SCORE = 100 -MIN_SCORE = 0 -INITIAL_SCORE = 10 - -VALID_STATUS_CODES = [200, 302] - -# 代理池数量界限 -POOL_UPPER_THRESHOLD = 50000 - -# 检查周期 -TESTER_CYCLE = 20 -# 获取周期 -GETTER_CYCLE = 300 - -# 测试API，建议抓哪个网站测哪个 -TEST_URL = 'http://www.baidu.com' - -# API配置 -API_HOST = '0.0.0.0' -API_PORT = 5555 - -# 开关 -TESTER_ENABLED = True -GETTER_ENABLED = True -API_ENABLED = True - -# 最大批测试量 -BATCH_TEST_SIZE = 10 +from environs import Env +from proxypool.utils import parse_redis_connection_string + + +env = Env() +env.read_env() + +# definition of environments +DEV_MODE, TEST_MODE, PROD_MODE = 'dev', 'test', 'prod' +APP_ENV = env.str('APP_ENV', DEV_MODE).lower() +APP_DEBUG = env.bool('APP_DEBUG', True if APP_ENV == DEV_MODE else False) +APP_DEV = IS_DEV = APP_ENV == DEV_MODE +APP_PROD = IS_PROD = APP_DEV == PROD_MODE +APP_TEST = IS_TEST = APP_ENV = TEST_MODE + +# redis host +REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1') +# redis port +REDIS_PORT = env.int('REDIS_PORT', 6379) +# redis password, if no password, set it to None +REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) +# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port +# REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) + +# if REDIS_CONNECTION_STRING: +# REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING) + +# redis hash table key name +REDIS_KEY = env.str('REDIS_HOST', 'proxies') + +# definition of proxy scores +PROXY_SCORE_MAX = 100 +PROXY_SCORE_MIN = 0 +PROXY_SCORE_INIT = 10 + +# definition of proxy number +PROXY_NUMBER_MAX = 50000 +PROXY_NUMBER_MIN = 0 + +# definition of tester cycle, it will test every CYCLE_TESTER second +CYCLE_TESTER = env.int('CYCLE_TESTER', 20) +# definition of getter cycle, it will get proxy every CYCLE_GETTER second +CYCLE_GETTER = env.int('CYCLE_GETTER', 100) + +# definition of tester +TEST_URL = env.str('TEST_URL', 'http://www.baidu.com') +TEST_TIMEOUT = env.int('TEST_TIMEOUT', 10) +TEST_BATCH = env.int('TEST_BATCH', 20) +# TEST_HEADERS = env.json('TEST_HEADERS', { +# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', +# }) +TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) + +# definition of api +API_HOST = env.str('API_HOST', '0.0.0.0') +API_PORT = env.int('API_PORT', 5555) +API_THREADED = env.bool('API_THREADED', True) + +# flags of enable +ENABLE_TESTER = env.bool('ENABLE_TESTER', True) +ENABLE_GETTER = env.bool('ENABLE_GETTER', True) +ENABLE_API = env.bool('ENABLE_API', True) diff --git a/proxypool/utils.py b/proxypool/utils.py index 16fa2c5a..3ac88798 100644 --- a/proxypool/utils.py +++ b/proxypool/utils.py @@ -1,27 +1,50 @@ import requests from requests.exceptions import ConnectionError +import re -base_headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', - 'Accept-Encoding': 'gzip, deflate, sdch', - 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7' -} +from proxypool.proxy import Proxy -def get_page(url, options={}): +def parse_redis_connection_string(connection_string): """ - 抓取代理 - :param url: - :param options: + parse a redis connection string, for example: + redis://[password]@host:port + rediss://[password]@host:port + :param connection_string: :return: """ - headers = dict(base_headers, **options) - print('正在抓取', url) - try: - response = requests.get(url, headers=headers) - print('抓取成功', url, response.status_code) - if response.status_code == 200: - return response.text - except ConnectionError: - print('抓取失败', url) + result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string) + return result.group(2), int(result.group(3)), (result.group(1) or None) if result \ + else ('localhost', 6379, None) + + +def is_valid_proxy(data): + """ + is data is valid proxy format + :param data: + :return: + """ + return re.match('\d+\.\d+\.\d+\.\d+\:\d+', data) + + +def convert_proxy_or_proxies(data): + """ + convert list of str to valid proxies or proxy + :param data: + :return: + """ + print(data) + if not data: return None + if isinstance(data, list): + result = [] + for item in data: + # skip invalid item + item = item.strip() + if not is_valid_proxy(item): continue + host, port = item.split(':') + result.append(Proxy(host=host, port=int(port))) + return result + if isinstance(data, str) and is_valid_proxy(data): + host, port = data.split(':') + return Proxy(host=host, port=int(port)) From d4fca11146b5487de10f3ca1e4636bbc572941c6 Mon Sep 17 00:00:00 2001 From: Germey Date: Wed, 19 Feb 2020 23:42:05 +0800 Subject: [PATCH 005/139] update structure --- .gitignore | 3 +- LICENSE | 201 -------------------- README.md | 80 +++++--- importer.py | 4 - proxy provider.txt | 7 - proxypool/.gitignore | 134 +++++++++++++ proxypool/crawlers/__init__.py | 2 +- proxypool/crawlers/base.py | 3 +- proxypool/crawlers/daili66.py | 4 +- proxypool/crawlers/ip3366.py | 2 +- proxypool/crawlers/iphai.py | 2 +- proxypool/exceptions/__init__.py | 1 + proxypool/{error.py => exceptions/empty.py} | 2 +- proxypool/processors/__init__.py | 0 proxypool/{ => processors}/getter.py | 10 +- proxypool/{api.py => processors/server.py} | 24 ++- proxypool/processors/tester.py | 74 +++++++ proxypool/scheduler.py | 98 +++++++--- proxypool/schemas/__init__.py | 1 + proxypool/{ => schemas}/proxy.py | 8 + proxypool/setting.py | 18 +- proxypool/storages/__init__.py | 0 proxypool/{db.py => storages/redis.py} | 21 +- proxypool/tester.py | 61 ------ proxypool/utils/__init__.py | 0 proxypool/utils/parse.py | 13 ++ proxypool/{utils.py => utils/proxy.py} | 20 +- requirements.txt | 14 +- run.py | 22 +-- 29 files changed, 429 insertions(+), 400 deletions(-) delete mode 100644 LICENSE delete mode 100644 importer.py delete mode 100644 proxy provider.txt create mode 100644 proxypool/.gitignore create mode 100644 proxypool/exceptions/__init__.py rename proxypool/{error.py => exceptions/empty.py} (78%) create mode 100644 proxypool/processors/__init__.py rename proxypool/{ => processors}/getter.py (88%) rename proxypool/{api.py => processors/server.py} (59%) create mode 100644 proxypool/processors/tester.py create mode 100644 proxypool/schemas/__init__.py rename proxypool/{ => schemas}/proxy.py (75%) create mode 100644 proxypool/storages/__init__.py rename proxypool/{db.py => storages/redis.py} (90%) delete mode 100644 proxypool/tester.py create mode 100644 proxypool/utils/__init__.py create mode 100644 proxypool/utils/parse.py rename proxypool/{utils.py => utils/proxy.py} (59%) diff --git a/.gitignore b/.gitignore index 416be2aa..7f21799f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.pyc *.db venv -/.idea \ No newline at end of file +/.idea +*.log \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 8dada3ed..00000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index 42340928..95773ac8 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,76 @@ # ProxyPool -## 安装 +## Requirements -### 安装Python +* Docker -至少Python3.5以上 + or -### 安装Redis +* Python: >=3.6 +* Redis +* Environment: Virtual Env -安装好之后将Redis服务开启 +## Run with Docker -### 配置代理池 - -``` -cd proxypool +```shell script +docker-compose up ``` -进入proxypool目录，修改settings.py文件 +## Run without Docker -PASSWORD为Redis密码，如果为空，则设置为None +Here are steps to run ProxyPool. -#### 安装依赖 +### Install Redis -``` -pip3 install -r requirements.txt +You need to install Redis locally or get a Redis server firstly. + +Next set Redis environment: + +```shell script +export REDIS_HOST='localhost' +export REDIS_PORT=6379 +export REDIS_PASSWORD='foobar' ``` -#### 打开代理池和API +Also you can just set the Redis Connection String: +```shell script +export REDIS_CONNECTION_STRING='redis://[password]@host:port' ``` -python3 run.py + +You can choose one method of above to set Redis environment. + +### Clone ProxyPool + +```shell script +https://github.com/Python3WebSpider/ProxyPool +cd ProxyPool ``` -## 获取代理 +### Install Requirements + +```shell script +pip3 install -r requirements.txt +``` +### Run ProxyPool -利用requests获取方法如下 +You can run all of the processors including Getter、Tester、 +Server: -```python -import requests +```shell script +python3 run.py +``` -PROXY_POOL_URL = 'http://localhost:5555/random' +or run with args to run specific processor: -def get_proxy(): - try: - response = requests.get(PROXY_POOL_URL) - if response.status_code == 200: - return response.text - except ConnectionError: - return None +```shell script +python3 run.py --processor getter +python3 run.py --processor tester +python3 run.py --processor server ``` + +### Usage + +After running the ProxyPool, you can visit +[http://localhost:5555/random](http://localhost:5555/) to access random proxy. \ No newline at end of file diff --git a/importer.py b/importer.py deleted file mode 100644 index 22b2ed30..00000000 --- a/importer.py +++ /dev/null @@ -1,4 +0,0 @@ -from proxypool.importer import scan - -if __name__ == '__main__': - scan() \ No newline at end of file diff --git a/proxy provider.txt b/proxy provider.txt deleted file mode 100644 index 063c1b91..00000000 --- a/proxy provider.txt +++ /dev/null @@ -1,7 +0,0 @@ -代理： -https://proxy.mimvp.com/free.php?proxy=in_hp -http://www.coobobo.com/free-http-proxy -http://ip.zdaye.com/ -http://www.mayidaili.com/free/anonymous/%E9%AB%98%E5%8C%BF -http://http.taiyangruanjian.com/ -http://http.zhimaruanjian.com/ \ No newline at end of file diff --git a/proxypool/.gitignore b/proxypool/.gitignore new file mode 100644 index 00000000..9f263786 --- /dev/null +++ b/proxypool/.gitignore @@ -0,0 +1,134 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.idea/ +*.log \ No newline at end of file diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py index bb690937..0b6ca544 100644 --- a/proxypool/crawlers/__init__.py +++ b/proxypool/crawlers/__init__.py @@ -6,5 +6,5 @@ __all__ = [ Daili66Crawler, IP3366Crawler, - IP3366Crawler + IPHaiCrawler ] diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 84a0c94c..1c204bb1 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -14,7 +14,7 @@ def fetch(self, url, **kwargs): return response.text except requests.ConnectionError: return - + @logger.catch def crawl(self): """ @@ -24,4 +24,5 @@ def crawl(self): logger.info(f'fetching {url}') html = self.fetch(url) for proxy in self.parse(html): + logger.info(f'fetched proxy {proxy.string()}') yield proxy diff --git a/proxypool/crawlers/daili66.py b/proxypool/crawlers/daili66.py index c91e64d2..965e0998 100644 --- a/proxypool/crawlers/daili66.py +++ b/proxypool/crawlers/daili66.py @@ -1,9 +1,9 @@ from pyquery import PyQuery as pq -from proxypool.proxy import Proxy +from proxypool.schemas.proxy import Proxy from proxypool.crawlers.base import BaseCrawler -BASE_URL = 'http://www.636ip.cn/{page}.html' +BASE_URL = 'http://www.664ip.cn/{page}.html' MAX_PAGE = 5 class Daili66Crawler(BaseCrawler): diff --git a/proxypool/crawlers/ip3366.py b/proxypool/crawlers/ip3366.py index d9673892..e85863aa 100644 --- a/proxypool/crawlers/ip3366.py +++ b/proxypool/crawlers/ip3366.py @@ -1,5 +1,5 @@ from proxypool.crawlers.base import BaseCrawler -from proxypool.proxy import Proxy +from proxypool.schemas.proxy import Proxy import re diff --git a/proxypool/crawlers/iphai.py b/proxypool/crawlers/iphai.py index 59d892ad..57638a2b 100644 --- a/proxypool/crawlers/iphai.py +++ b/proxypool/crawlers/iphai.py @@ -1,5 +1,5 @@ from proxypool.crawlers.base import BaseCrawler -from proxypool.proxy import Proxy +from proxypool.schemas.proxy import Proxy import re diff --git a/proxypool/exceptions/__init__.py b/proxypool/exceptions/__init__.py new file mode 100644 index 00000000..b54b1e85 --- /dev/null +++ b/proxypool/exceptions/__init__.py @@ -0,0 +1 @@ +from .empty import PoolEmptyException \ No newline at end of file diff --git a/proxypool/error.py b/proxypool/exceptions/empty.py similarity index 78% rename from proxypool/error.py rename to proxypool/exceptions/empty.py index 22ac1df3..255c7fbf 100644 --- a/proxypool/error.py +++ b/proxypool/exceptions/empty.py @@ -1,4 +1,4 @@ -class PoolEmptyError(Exception): +class PoolEmptyException(Exception): def __str__(self): """ proxypool is used out diff --git a/proxypool/processors/__init__.py b/proxypool/processors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proxypool/getter.py b/proxypool/processors/getter.py similarity index 88% rename from proxypool/getter.py rename to proxypool/processors/getter.py index 5787e19f..4bd59a4f 100644 --- a/proxypool/getter.py +++ b/proxypool/processors/getter.py @@ -1,14 +1,14 @@ -from proxypool.db import RedisClient +from loguru import logger +from proxypool.storages.redis import RedisClient from proxypool.setting import PROXY_NUMBER_MAX from proxypool.crawlers import __all__ as crawlers_cls -class Getter(): +class Getter(object): """ getter of proxypool """ - def __init__(self): """ init db and crawlers @@ -17,7 +17,6 @@ def __init__(self): self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] - def is_full(self): """ if proxypool if full @@ -25,7 +24,7 @@ def is_full(self): """ return self.redis.count() >= PROXY_NUMBER_MAX - + @logger.catch def run(self): """ run crawlers to get proxy @@ -35,7 +34,6 @@ def run(self): return for crawler in self.crawlers: for proxy in crawler.crawl(): - print('proxy', proxy) self.redis.add(proxy) diff --git a/proxypool/api.py b/proxypool/processors/server.py similarity index 59% rename from proxypool/api.py rename to proxypool/processors/server.py index 712a5df1..e87f82f5 100644 --- a/proxypool/api.py +++ b/proxypool/processors/server.py @@ -1,11 +1,13 @@ from flask import Flask, g -from .db import RedisClient -from .setting import API_HOST, API_PORT +from proxypool.storages.redis import RedisClient +from proxypool.setting import API_HOST, API_PORT, API_THREADED + __all__ = ['app'] app = Flask(__name__) + def get_conn(): """ get redis client object @@ -15,27 +17,35 @@ def get_conn(): g.redis = RedisClient() return g.redis + @app.route('/') def index(): + """ + get home page, you can define your own templates + :return: + """ return '

    Welcome to Proxy Pool System

    ' + @app.route('/random') def get_proxy(): """ get a random proxy - :return: 随机代理 + :return: get a random proxy """ conn = get_conn() - return conn.random() + return conn.random().string() + @app.route('/count') -def get_counts(): +def get_count(): """ get the count of proxies - :return: 代理池总量 + :return: count, int """ conn = get_conn() return str(conn.count()) + if __name__ == '__main__': - app.run(host=API_HOST, port=API_PORT, threaded=True) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py new file mode 100644 index 00000000..6df2f5bf --- /dev/null +++ b/proxypool/processors/tester.py @@ -0,0 +1,74 @@ +import asyncio +import aiohttp +from loguru import logger +from proxypool.schemas import Proxy +from proxypool.storages.redis import RedisClient +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS +from aiohttp import ClientProxyConnectionError, ServerDisconnectedError +from asyncio import TimeoutError + + +EXCEPTIONS = ( + ClientProxyConnectionError, + ConnectionRefusedError, + TimeoutError, + ServerDisconnectedError +) + + +class Tester(object): + """ + tester for testing proxies in queue + """ + + def __init__(self): + """ + init redis + """ + self.redis = RedisClient() + self.loop = asyncio.get_event_loop() + + async def test(self, proxy: Proxy): + """ + test single proxy + :param proxy: Proxy object + :return: + """ + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: + try: + logger.debug(f'testing {proxy.string()}') + async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, + allow_redirects=False) as response: + if response.status in TEST_VALID_STATUS: + self.redis.max(proxy) + logger.debug(f'proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease(proxy) + logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + except EXCEPTIONS: + self.redis.decrease(proxy) + logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + + @logger.catch + def run(self): + """ + test main method + :return: + """ + # event loop of aiohttp + logger.info('stating tester...') + count = self.redis.count() + logger.debug(f'{count} proxies to test') + for i in range(0, count, TEST_BATCH): + # start end end offset + start, end = i, min(i + TEST_BATCH, count) + logger.debug(f'testing proxies from {start} to {end} indices') + proxies = self.redis.batch(start, end) + tasks = [self.test(proxy) for proxy in proxies] + # run tasks using event loop + self.loop.run_until_complete(asyncio.wait(tasks)) + + +if __name__ == '__main__': + tester = Tester() + tester.run() diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index 4ca55eed..3045341b 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -1,50 +1,92 @@ import time -from multiprocessing import Process -from proxypool.api import app -from proxypool.getter import Getter -from proxypool.tester import Tester -from proxypool.db import RedisClient -from proxypool.setting import * +import multiprocessing +from proxypool.processors.server import app +from proxypool.processors.getter import Getter +from proxypool.processors.tester import Tester +from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, API_THREADED, API_PORT, ENABLE_SERVER, \ + ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS +from loguru import logger + + +if IS_WINDOWS: + multiprocessing.freeze_support() + +tester_process, getter_process, server_process = None, None, None class Scheduler(): - def schedule_tester(self, cycle=TESTER_CYCLE): + """ + scheduler + """ + + def run_tester(self, cycle=CYCLE_TESTER): """ - 定时测试代理 + run tester """ tester = Tester() + loop = 0 while True: - print('测试器开始运行') + logger.debug(f'tester loop {loop} start...') tester.run() + loop += 1 time.sleep(cycle) - def schedule_getter(self, cycle=GETTER_CYCLE): + def run_getter(self, cycle=CYCLE_GETTER): """ - 定时获取代理 + run getter """ getter = Getter() + loop = 0 while True: - print('开始抓取代理') + logger.debug(f'getter loop {loop} start...') getter.run() + loop += 1 time.sleep(cycle) - def schedule_api(self): + def run_server(self): """ - 开启API + run server for api """ - app.run(API_HOST, API_PORT) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) def run(self): - print('代理池开始运行') - - if TESTER_ENABLED: - tester_process = Process(target=self.schedule_tester) - tester_process.start() - - if GETTER_ENABLED: - getter_process = Process(target=self.schedule_getter) - getter_process.start() - - if API_ENABLED: - api_process = Process(target=self.schedule_api) - api_process.start() + global tester_process, getter_process, server_process + try: + logger.info('starting proxypool...') + if ENABLE_TESTER: + tester_process = multiprocessing.Process(target=self.run_tester) + logger.info(f'starting tester, pid {tester_process.pid}...') + tester_process.start() + + if ENABLE_GETTER: + getter_process = multiprocessing.Process(target=self.run_getter) + logger.info(f'starting getter, pid{getter_process.pid}...') + getter_process.start() + + if ENABLE_SERVER: + server_process = multiprocessing.Process(target=self.run_server) + logger.info(f'starting server, pid{server_process.pid}...') + server_process.start() + + tester_process.join() + getter_process.join() + server_process.join() + except KeyboardInterrupt: + logger.info('received keyboard interrupt signal') + tester_process.terminate() + getter_process.terminate() + server_process.terminate() + finally: + # must call join method before calling is_alive + tester_process.join() + getter_process.join() + server_process.join() + logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}') + logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}') + logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}') + logger.info('proxy terminated') + + +if __name__ == '__main__': + scheduler = Scheduler() + scheduler.run() diff --git a/proxypool/schemas/__init__.py b/proxypool/schemas/__init__.py new file mode 100644 index 00000000..699f6dc2 --- /dev/null +++ b/proxypool/schemas/__init__.py @@ -0,0 +1 @@ +from .proxy import Proxy \ No newline at end of file diff --git a/proxypool/proxy.py b/proxypool/schemas/proxy.py similarity index 75% rename from proxypool/proxy.py rename to proxypool/schemas/proxy.py index 543ad7bc..8be3fb34 100644 --- a/proxypool/proxy.py +++ b/proxypool/schemas/proxy.py @@ -10,9 +10,17 @@ class Proxy(object): port = attr(type=int, default=None) def __str__(self): + """ + to string, for print + :return: + """ return f'{self.host}:{self.port}' def string(self): + """ + to string + :return: : + """ return self.__str__() diff --git a/proxypool/setting.py b/proxypool/setting.py index 97173950..7f205ac0 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -1,10 +1,19 @@ +import platform +from os.path import dirname, abspath, join from environs import Env -from proxypool.utils import parse_redis_connection_string +from loguru import logger env = Env() env.read_env() +# definition of flags +IS_WINDOWS = platform.system().lower() == 'windows' + +# definition of dirs +ROOT_DIR = dirname(dirname(abspath(__file__))) +LOG_DIR = join(ROOT_DIR, env.str('LOG_DIR', 'logs')) + # definition of environments DEV_MODE, TEST_MODE, PROD_MODE = 'dev', 'test', 'prod' APP_ENV = env.str('APP_ENV', DEV_MODE).lower() @@ -26,7 +35,7 @@ # REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING) # redis hash table key name -REDIS_KEY = env.str('REDIS_HOST', 'proxies') +REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') # definition of proxy scores PROXY_SCORE_MAX = 100 @@ -59,4 +68,7 @@ # flags of enable ENABLE_TESTER = env.bool('ENABLE_TESTER', True) ENABLE_GETTER = env.bool('ENABLE_GETTER', True) -ENABLE_API = env.bool('ENABLE_API', True) +ENABLE_SERVER = env.bool('ENABLE_SERVER', True) + +logger.add(env.str('LOG_RUNTIME_FILE', 'runtime.log'), level='DEBUG', rotation='1 week', retention='20 days') +logger.add(env.str('LOG_ERROR_FILE', 'error.log'), level='ERROR', rotation='1 week') diff --git a/proxypool/storages/__init__.py b/proxypool/storages/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proxypool/db.py b/proxypool/storages/redis.py similarity index 90% rename from proxypool/db.py rename to proxypool/storages/redis.py index 3b47ce85..06971cea 100644 --- a/proxypool/db.py +++ b/proxypool/storages/redis.py @@ -1,13 +1,12 @@ import redis -from proxypool.error import PoolEmptyError -from proxypool.proxy import Proxy -from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY -from proxypool.setting import PROXY_SCORE_MAX, PROXY_SCORE_MIN, PROXY_SCORE_INIT +from proxypool.exceptions import PoolEmptyException +from proxypool.schemas.proxy import Proxy +from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ + PROXY_SCORE_INIT from random import choice from typing import List from loguru import logger - -from proxypool.utils import is_valid_proxy, convert_proxy_or_proxies +from proxypool.utils.proxy import is_valid_proxy, convert_proxy_or_proxies REDIS_CLIENT_VERSION = redis.__version__ @@ -38,7 +37,7 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): logger.info(f'invalid proxy {proxy}, throw it') return - if not self.db.zscore(REDIS_KEY, proxy): + if not self.exists(proxy): if IS_REDIS_VERSION_2: return self.db.zadd(REDIS_KEY, score, proxy) return self.db.zadd(REDIS_KEY, {proxy: score}) @@ -60,7 +59,7 @@ def random(self) -> Proxy: if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error - raise PoolEmptyError + raise PoolEmptyException def decrease(self, proxy: Proxy) -> int: """ @@ -86,7 +85,7 @@ def exists(self, proxy: Proxy) -> bool: :param proxy: proxy :return: if exists, bool """ - return not self.db.zscore(REDIS_KEY, proxy.string()) == None + return not self.db.zscore(REDIS_KEY, proxy.string()) is None def max(self, proxy: Proxy) -> int: """ @@ -125,7 +124,5 @@ def batch(self, start, end) -> List[Proxy]: if __name__ == '__main__': conn = RedisClient() - result = conn.batch(1, 10) - print(result) result = conn.random() - print(result, type(result)) + print(result) diff --git a/proxypool/tester.py b/proxypool/tester.py deleted file mode 100644 index b43b59f9..00000000 --- a/proxypool/tester.py +++ /dev/null @@ -1,61 +0,0 @@ -import asyncio -import aiohttp -import time -import sys -try: - from aiohttp import ClientError -except: - from aiohttp import ClientProxyConnectionError as ProxyConnectionError -from proxypool.db import RedisClient -from proxypool.setting import * - - -class Tester(object): - def __init__(self): - self.redis = RedisClient() - - async def test_single_proxy(self, proxy): - """ - 测试单个代理 - :param proxy: - :return: - """ - conn = aiohttp.TCPConnector(verify_ssl=False) - async with aiohttp.ClientSession(connector=conn) as session: - try: - if isinstance(proxy, bytes): - proxy = proxy.decode('utf-8') - real_proxy = 'http://' + proxy - print('正在测试', proxy) - async with session.get(TEST_URL, proxy=real_proxy, timeout=15, allow_redirects=False) as response: - if response.status in VALID_STATUS_CODES: - self.redis.max(proxy) - print('代理可用', proxy) - else: - self.redis.decrease(proxy) - print('请求响应码不合法 ', response.status, 'IP', proxy) - except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError): - self.redis.decrease(proxy) - print('代理请求失败', proxy) - - def run(self): - """ - 测试主函数 - :return: - """ - print('测试器开始运行') - try: - count = self.redis.count() - print('当前剩余', count, '个代理') - for i in range(0, count, BATCH_TEST_SIZE): - start = i - stop = min(i + BATCH_TEST_SIZE, count) - print('正在测试第', start + 1, '-', stop, '个代理') - test_proxies = self.redis.batch(start, stop) - loop = asyncio.get_event_loop() - tasks = [self.test_single_proxy(proxy) for proxy in test_proxies] - loop.run_until_complete(asyncio.wait(tasks)) - sys.stdout.flush() - time.sleep(5) - except Exception as e: - print('测试器发生错误', e.args) diff --git a/proxypool/utils/__init__.py b/proxypool/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proxypool/utils/parse.py b/proxypool/utils/parse.py new file mode 100644 index 00000000..ceee566d --- /dev/null +++ b/proxypool/utils/parse.py @@ -0,0 +1,13 @@ +import re + +def parse_redis_connection_string(connection_string): + """ + parse a redis connection string, for example: + redis://[password]@host:port + rediss://[password]@host:port + :param connection_string: + :return: + """ + result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string) + return result.group(2), int(result.group(3)), (result.group(1) or None) if result \ + else ('localhost', 6379, None) \ No newline at end of file diff --git a/proxypool/utils.py b/proxypool/utils/proxy.py similarity index 59% rename from proxypool/utils.py rename to proxypool/utils/proxy.py index 3ac88798..00302b35 100644 --- a/proxypool/utils.py +++ b/proxypool/utils/proxy.py @@ -1,22 +1,6 @@ -import requests -from requests.exceptions import ConnectionError +from proxypool.schemas import Proxy import re -from proxypool.proxy import Proxy - - -def parse_redis_connection_string(connection_string): - """ - parse a redis connection string, for example: - redis://[password]@host:port - rediss://[password]@host:port - :param connection_string: - :return: - """ - result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string) - return result.group(2), int(result.group(3)), (result.group(1) or None) if result \ - else ('localhost', 6379, None) - def is_valid_proxy(data): """ @@ -33,9 +17,9 @@ def convert_proxy_or_proxies(data): :param data: :return: """ - print(data) if not data: return None + # if list of proxies if isinstance(data, list): result = [] for item in data: diff --git a/requirements.txt b/requirements.txt index 42e86f0f..2688f78a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ -aiohttp>=1.3.3 -Flask>=0.11.1 -redis>=2.10.5 -requests>=2.13.0 -pyquery>=1.2.17 +environs==7.2.0 +Flask==1.0.3 +attrs==19.1.0 +retrying==1.3.3 +aiohttp==3.6.2 +requests==2.22.0 +loguru==0.3.2 +pyquery==1.4.0 +attr==0.3.1 diff --git a/run.py b/run.py index 50f89f0c..ec7c75ef 100644 --- a/run.py +++ b/run.py @@ -1,17 +1,15 @@ from proxypool.scheduler import Scheduler -import sys -import io +import argparse -sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') - - -def main(): - try: - s = Scheduler() - s.run() - except: - main() +parser = argparse.ArgumentParser(description='ProxyPool') +parser.add_argument('--processor', type=str, help='processor to run') +args = parser.parse_args() +print('args', args) if __name__ == '__main__': - main() + # if processor set, just run it + if args.processor: + getattr(Scheduler(), f'run_{args.processor}')() + else: + Scheduler().run() From 162542e93e7230068ea38cc1ee43d9c79a52affb Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 00:17:25 +0800 Subject: [PATCH 006/139] update docker compose --- .dockerignore | 131 +++++++++++++++++++++++++++++++++++++++++ Dockerfile | 5 ++ README.md | 16 +++-- docker-compose.yml | 18 ++++++ proxypool/scheduler.py | 9 +++ proxypool/setting.py | 8 ++- requirements.txt | 2 + run.py | 1 - supervisord.conf | 17 ++++++ 9 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 supervisord.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..803baf5e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,131 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d3e7fa86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.6 +WORKDIR /app +COPY . . +RUN pip install -r requirements.txt +CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/README.md b/README.md index 95773ac8..4ce0c99a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,20 @@ # ProxyPool +![](https://img.shields.io/badge/python-3.6%2B-brightgreen) + +Welcome to ProxyPool. + ## Requirements -* Docker +Below is a list of requirements for ProxyPool. + +* Docker +* Docker-Compose or * Python: >=3.6 * Redis -* Environment: Virtual Env ## Run with Docker @@ -70,7 +76,9 @@ python3 run.py --processor tester python3 run.py --processor server ``` -### Usage +## Usage After running the ProxyPool, you can visit -[http://localhost:5555/random](http://localhost:5555/) to access random proxy. \ No newline at end of file +[http://localhost:5555/random](http://localhost:5555/) to access random proxy. + +Just enjoy it. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..c05b84a5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' +services: + redis: + image: redis:alpine + container_name: redis + command: redis-server + ports: + - "6379:6379" + restart: always + proxypool: + build: . + image: 'germey/proxypool' + container_name: proxypool + ports: + - "5555:5555" + restart: always + environment: + REDIS_HOST: redis \ No newline at end of file diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index 3045341b..bec0c595 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -23,6 +23,9 @@ def run_tester(self, cycle=CYCLE_TESTER): """ run tester """ + if not ENABLE_TESTER: + logger.info('tester not enabled, exit') + return tester = Tester() loop = 0 while True: @@ -35,6 +38,9 @@ def run_getter(self, cycle=CYCLE_GETTER): """ run getter """ + if not ENABLE_GETTER: + logger.info('getter not enabled, exit') + return getter = Getter() loop = 0 while True: @@ -47,6 +53,9 @@ def run_server(self): """ run server for api """ + if not ENABLE_SERVER: + logger.info('server not enabled, exit') + return app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) def run(self): diff --git a/proxypool/setting.py b/proxypool/setting.py index 7f205ac0..0f2e64cd 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -3,6 +3,8 @@ from environs import Env from loguru import logger +from proxypool.utils.parse import parse_redis_connection_string + env = Env() env.read_env() @@ -29,10 +31,10 @@ # redis password, if no password, set it to None REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) # redis connection string, like redis://[password]@host:port or rediss://[password]@host:port -# REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) +REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) -# if REDIS_CONNECTION_STRING: -# REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING) +if REDIS_CONNECTION_STRING: + REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING) # redis hash table key name REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') diff --git a/requirements.txt b/requirements.txt index 2688f78a..52e44e61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ requests==2.22.0 loguru==0.3.2 pyquery==1.4.0 attr==0.3.1 +supervisor==4.1.0 +redis==2.10.6 \ No newline at end of file diff --git a/run.py b/run.py index ec7c75ef..e858da90 100644 --- a/run.py +++ b/run.py @@ -5,7 +5,6 @@ parser = argparse.ArgumentParser(description='ProxyPool') parser.add_argument('--processor', type=str, help='processor to run') args = parser.parse_args() -print('args', args) if __name__ == '__main__': # if processor set, just run it diff --git a/supervisord.conf b/supervisord.conf new file mode 100644 index 00000000..c5828e0c --- /dev/null +++ b/supervisord.conf @@ -0,0 +1,17 @@ +[supervisord] +nodaemon=true + +[program:tester] +process_name=tester +command=python3 run.py --processor tester +directory=/app + +[program:getter] +process_name=getter +command=python3 run.py --processor getter +directory=/app + +[program:server] +process_name=server +command=python3 run.py --processor server +directory=/app \ No newline at end of file From 097461d1e23b35f22731f19f68055148157aa848 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 00:24:16 +0800 Subject: [PATCH 007/139] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ce0c99a..22fab1cd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ProxyPool ![](https://img.shields.io/badge/python-3.6%2B-brightgreen) - +![Docker Pulls](https://img.shields.io/docker/pulls/germey/proxypool) Welcome to ProxyPool. ## Requirements From c562a794ed9dda2cffe36d3c5cffea8db008127c Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 01:01:38 +0800 Subject: [PATCH 008/139] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 22fab1cd..a376fe17 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ![](https://img.shields.io/badge/python-3.6%2B-brightgreen) ![Docker Pulls](https://img.shields.io/docker/pulls/germey/proxypool) + Welcome to ProxyPool. ## Requirements From f64b41caa4c52592fe6b6b008b4efa12091c6d57 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 02:01:30 +0800 Subject: [PATCH 009/139] add readme --- README.md | 282 ++++++++++++++++++++++++++++++++++++++----- deployment.yml | 0 examples/__init__.py | 0 examples/usage.py | 39 ++++++ 4 files changed, 294 insertions(+), 27 deletions(-) create mode 100644 deployment.yml create mode 100644 examples/__init__.py create mode 100644 examples/usage.py diff --git a/README.md b/README.md index a376fe17..8a06cccb 100644 --- a/README.md +++ b/README.md @@ -3,73 +3,111 @@ ![](https://img.shields.io/badge/python-3.6%2B-brightgreen) ![Docker Pulls](https://img.shields.io/docker/pulls/germey/proxypool) -Welcome to ProxyPool. +简易高效的代理池，提供如下功能： -## Requirements +* 定时抓取免费代理网站，简易可扩展。 +* 使用 Redis 对代理进行存储并对代理可用性进行排序。 +* 定时测试和筛选，剔除不可用代理，留下可用代理。 +* 提供代理 API，随机取用测试通过的可用代理。 -Below is a list of requirements for ProxyPool. +代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」，建议使用之前阅读。 + +## 使用要求 + +可以通过两种方式来运行代理池，一种方式是使用 Docker（推荐），另一种方式是常规方式运行。 + +### Docker + +如果使用 Docker，则需要安装了如下环境： * Docker * Docker-Compose - or +### 常规方式 + +常规方式要求有 Python 环境、Redis 环境，具体要求如下： * Python: >=3.6 * Redis -## Run with Docker +## Docker 运行 + +如果安装好了 Docker 和 Docker-Compose，只需要一条命令即可运行。 ```shell script docker-compose up ``` -## Run without Docker +运行结果类似如下： + +``` +redis | 1:M 19 Feb 2020 17:09:43.940 * DB loaded from disk: 0.000 seconds +redis | 1:M 19 Feb 2020 17:09:43.940 * Ready to accept connections +proxypool | 2020-02-19 17:09:44,200 CRIT Supervisor is running as root. Privileges were not dropped because no user is specified in the config file. If you intend to run as root, you can set user=root in the config file to avoid this message. +proxypool | 2020-02-19 17:09:44,203 INFO supervisord started with pid 1 +proxypool | 2020-02-19 17:09:45,209 INFO spawned: 'getter' with pid 10 +proxypool | 2020-02-19 17:09:45,212 INFO spawned: 'server' with pid 11 +proxypool | 2020-02-19 17:09:45,216 INFO spawned: 'tester' with pid 12 +proxypool | 2020-02-19 17:09:46,596 INFO success: getter entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +proxypool | 2020-02-19 17:09:46,596 INFO success: server entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +``` + +可以看到 Redis、Getter、Server、Tester 都已经启动成功。 + +这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 + +## 常规方式运行 + +如果不使用 Docker 运行，配置好 Python、Redis 环境之后也可运行，步骤如下。 + +### 安装和配置 Redis -Here are steps to run ProxyPool. +本地安装 Redis、Docker 启动 Redis、远程 Redis 都是可以的，只要能正常连接使用即可。 -### Install Redis +首先可以需要一下环境变量，代理池会通过环境变量读取这些值。 -You need to install Redis locally or get a Redis server firstly. +设置 Redis 的环境变量有两种方式，一种是分别设置 host、port、password，另一种是设置连接字符串，设置方法分别如下： -Next set Redis environment: +设置 host、port、password，如果 password 为空可以设置为空字符串，示例如下： ```shell script export REDIS_HOST='localhost' export REDIS_PORT=6379 -export REDIS_PASSWORD='foobar' +export REDIS_PASSWORD='' ``` -Also you can just set the Redis Connection String: +或者只设置连接字符串： ```shell script export REDIS_CONNECTION_STRING='redis://[password]@host:port' ``` -You can choose one method of above to set Redis environment. +这里连接字符串的格式需要符合 `redis://[password]@host:port` 的格式。 -### Clone ProxyPool +以上两种设置任选其一即可。 -```shell script -https://github.com/Python3WebSpider/ProxyPool -cd ProxyPool -``` +### 安装依赖包 -### Install Requirements +这里强烈推荐使用 Conda 或 VirtualEnv 创建虚拟环境，然后 pip 安装依赖即可： ```shell script pip3 install -r requirements.txt ``` -### Run ProxyPool +### 运行代理池 -You can run all of the processors including Getter、Tester、 -Server: +两种方式运行代理池，一种是 Tester、Getter、Server 全部运行，另一种是按需分别运行。 + +一般来说可以选择全部运行，命令如下： ```shell script python3 run.py ``` -or run with args to run specific processor: +运行之后会启动 Tester、Getter、Server，这时访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 + +或者如果你弄清楚了代理池的架构，可以按需分别运行，命令如下： ```shell script python3 run.py --processor getter @@ -77,9 +115,199 @@ python3 run.py --processor tester python3 run.py --processor server ``` -## Usage +这里 processor 可以指定运行 Tester、Getter 还是 Server。 + +## 使用 + +成功运行之后可以通过 [http://localhost:5555/random](http://localhost:5555/random) 获取一个随机可用代理。 + +可以用程序对接实现，简单的示例展示了获取代理并爬取网页的过程： + +```python +import requests + +proxypool_url = 'http://127.0.0.1:5555/random' +target_url = 'http://httpbin.org/get' + +def get_random_proxy(): + """ + get random proxy from proxypool + :return: proxy + """ + return requests.get(proxypool_url).text.strip() + +def crawl(url, proxy): + """ + use proxy to crawl page + :param url: page url + :param proxy: proxy, such as 8.8.8.8:8888 + :return: html + """ + proxies = {'http': 'http://' + proxy} + return requests.get(url, proxies=proxies).text + + +def main(): + """ + main method, entry point + :return: none + """ + proxy = get_random_proxy() + print('get random proxy', proxy) + html = crawl(target_url, proxy) + print(html) + +if __name__ == '__main__': + main() +``` + +运行结果如下： + +``` +get random proxy 116.196.115.209:8080 +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Host": "httpbin.org", + "User-Agent": "python-requests/2.22.0", + "X-Amzn-Trace-Id": "Root=1-5e4d7140-662d9053c0a2e513c7278364" + }, + "origin": "116.196.115.209", + "url": "https://httpbin.org/get" +} +``` + +可以看到成功获取了代理，并请求 httpbin.org 验证了代理的可用性。 + +## 可配置项 + +代理池可以通过设置环境变量来配置一些参数。 + +### 开关 + +* ENABLE_TESTER：允许 Tester 启动，默认 true +* ENABLE_GETTER：允许 Getter 启动，默认 true +* ENABLE_SERVER：运行 Server 启动，默认 true + +### 环境 + +* APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev +* APP_DEBUG：调试模式，可以设置 true 或 false，默认 true + +### Redis 连接 + +* REDIS_HOST：Redis 的 Host +* REDIS_PORT：Redis 的端口 +* REDIS_PASSWORD：Redis 的密码 +* REDIS_CONNECTION_STRING：Redis 连接字符串 +* REDIS_KEY：Redis 储存代理使用字典的名称 + +### 处理器 + +* CYCLE_TESTER：Tester 运行周期，即间隔多久运行一次测试，默认 20 秒 +* CYCLE_GETTER：Getter 运行周期，即间隔多久运行一次代理获取，默认 100 秒 +* TEST_URL：测试 URL，默认百度 +* TEST_TIMEOUT：测试超时时间，默认 10 秒 +* TEST_BATCH：批量测试数量，默认 20 个代理 +* TEST_VALID_STATUS：测试有效的状态吗 +* API_HOST：代理 Server 运行 Host，默认 0.0.0.0 +* API_PORT：代理 Server 运行端口，默认 5555 +* API_THREADED：代理 Server 是否使用多线程，默认 true + +### 日志 + +* LOG_DIR：日志相对路径 +* LOG_RUNTIME_FILE：运行日志文件名称 +* LOG_ERROR_FILE：错误日志文件名称 + +以上内容均可使用环境变量配置，即在运行前设置对应环境变量值即可，如更改测试地址和 Redis 键名： + +```shell script +export TEST_URL=http://weibo.cn +export REDIS_KEY=proxies:weibo +``` + +即可构建一个专属于微博的代理池，有效的代理都是可以爬取微博的。 + +如果使用 Docker-Compose 启动代理池，则需要在 docker-compose.yml 文件里面指定环境变量，如： + +```shell script +version: '3' +services: + redis: + image: redis:alpine + container_name: redis + command: redis-server + ports: + - "6379:6379" + restart: always + proxypool: + build: . + image: 'germey/proxypool' + container_name: proxypool + ports: + - "5555:5555" + restart: always + environment: + REDIS_HOST: redis + TEST_URL: http://weibo.cn + REDIS_KEY: proxies:weibo +``` + +## 扩展代理 + +代理的爬虫均放置在 proxypool/crawlers 文件夹下，目前对接了有限的爬虫。 + +若扩展一个爬虫，只需要在 crawlers 文件夹下新建一个 Python 文件声明一个 Class 即可。 + +写法规范如下： + +```python +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler + +BASE_URL = 'http://www.664ip.cn/{page}.html' +MAX_PAGE = 5 + +class Daili66Crawler(BaseCrawler): + """ + daili66 crawler, http://www.66ip.cn/1.html + """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('.containerbox table tr:gt(0)').items() + for tr in trs: + host = tr.find('td:nth-child(1)').text() + port = int(tr.find('td:nth-child(2)').text()) + yield Proxy(host=host, port=port) +``` + +在这里只需要定义一个 Crawler 继承 BaseCrawler 即可，然后定义好 urls 变量和 parse 方法即可。 + +* urls 变量即为爬取的代理网站网址列表，可以用程序定义也可写成固定内容。 +* parse 方法接收一个参数即 html，代理网址的 html，在 parse 方法里只需要写好 html 的解析，解析出 host 和 port，并构建 Proxy 对象 yield 返回即可。 + +网页的爬取不需要实现，BaseCrawler 已经有了默认实现，如需更改爬取方式，重写 crawl 方法即可。 + +欢迎大家多多发 Pull Request 贡献 Crawler，使其代理源更丰富强大起来。 + +## 部署 + +本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，执行如下命令即可： + +```shell script +kubectl apply -f deployment.yml +``` -After running the ProxyPool, you can visit -[http://localhost:5555/random](http://localhost:5555/) to access random proxy. +## License -Just enjoy it. \ No newline at end of file +MIT diff --git a/deployment.yml b/deployment.yml new file mode 100644 index 00000000..e69de29b diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/usage.py b/examples/usage.py new file mode 100644 index 00000000..87c73b35 --- /dev/null +++ b/examples/usage.py @@ -0,0 +1,39 @@ +import requests + + +proxypool_url = 'http://127.0.0.1:5555/random' +target_url = 'http://httpbin.org/get' + + +def get_random_proxy(): + """ + get random proxy from proxypool + :return: proxy + """ + return requests.get(proxypool_url).text.strip() + + +def crawl(url, proxy): + """ + use proxy to crawl page + :param url: page url + :param proxy: proxy, such as 8.8.8.8:8888 + :return: html + """ + proxies = {'http': 'http://' + proxy} + return requests.get(url, proxies=proxies).text + + +def main(): + """ + main method, entry point + :return: none + """ + proxy = get_random_proxy() + print('get random proxy', proxy) + html = crawl(target_url, proxy) + print(html) + + +if __name__ == '__main__': + main() From dc674573f2335bfb69affefe618071e1b96e7b7e Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 02:10:21 +0800 Subject: [PATCH 010/139] update readme --- README.md | 17 ++++++++++------- proxypool/crawlers/daili66.py | 3 ++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8a06cccb..ea0fe650 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ ### Docker -如果使用 Docker，则需要安装了如下环境： +如果使用 Docker，则需要安装如下环境： * Docker * Docker-Compose @@ -27,7 +27,7 @@ 常规方式要求有 Python 环境、Redis 环境，具体要求如下： -* Python: >=3.6 +* Python>=3.6 * Redis ## Docker 运行 @@ -89,7 +89,10 @@ export REDIS_CONNECTION_STRING='redis://[password]@host:port' ### 安装依赖包 -这里强烈推荐使用 Conda 或 VirtualEnv 创建虚拟环境，然后 pip 安装依赖即可： +这里强烈推荐使用 [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands) +或 [virtualenv](https://virtualenv.pypa.io/en/latest/user_guide.html) 创建虚拟环境，Python 版本不低于 3.6。 + +然后 pip 安装依赖即可： ```shell script pip3 install -r requirements.txt @@ -121,7 +124,7 @@ python3 run.py --processor server 成功运行之后可以通过 [http://localhost:5555/random](http://localhost:5555/random) 获取一个随机可用代理。 -可以用程序对接实现，简单的示例展示了获取代理并爬取网页的过程： +可以用程序对接实现，下面的示例展示了获取代理并爬取网页的过程： ```python import requests @@ -233,7 +236,7 @@ export REDIS_KEY=proxies:weibo 如果使用 Docker-Compose 启动代理池，则需要在 docker-compose.yml 文件里面指定环境变量，如： -```shell script +```yaml version: '3' services: redis: @@ -256,9 +259,9 @@ services: REDIS_KEY: proxies:weibo ``` -## 扩展代理 +## 扩展代理爬虫 -代理的爬虫均放置在 proxypool/crawlers 文件夹下，目前对接了有限的爬虫。 +代理的爬虫均放置在 proxypool/crawlers 文件夹下，目前对接了有限几个代理的爬虫。 若扩展一个爬虫，只需要在 crawlers 文件夹下新建一个 Python 文件声明一个 Class 即可。 diff --git a/proxypool/crawlers/daili66.py b/proxypool/crawlers/daili66.py index 965e0998..a6e4ca82 100644 --- a/proxypool/crawlers/daili66.py +++ b/proxypool/crawlers/daili66.py @@ -6,13 +6,13 @@ BASE_URL = 'http://www.664ip.cn/{page}.html' MAX_PAGE = 5 + class Daili66Crawler(BaseCrawler): """ daili66 crawler, http://www.66ip.cn/1.html """ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] - def parse(self, html): """ parse html file to get proxies @@ -25,6 +25,7 @@ def parse(self, html): port = int(tr.find('td:nth-child(2)').text()) yield Proxy(host=host, port=port) + if __name__ == '__main__': crawler = Daili66Crawler() for proxy in crawler.crawl(): From 94d5e763074491e525f0857a9d55b5b88e6c9959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 02:13:28 +0800 Subject: [PATCH 011/139] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..89052acc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Germey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 3e7a2cf0f7aeea29d156fbaee6e27d2f4fd615be Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 02:28:03 +0800 Subject: [PATCH 012/139] add k8s file --- README.md | 2 +- deployment.yml | 118 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea0fe650..8f04d6d1 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,6 @@ class Daili66Crawler(BaseCrawler): kubectl apply -f deployment.yml ``` -## License +## LICENSE MIT diff --git a/deployment.yml b/deployment.yml index e69de29b..40497b36 100644 --- a/deployment.yml +++ b/deployment.yml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + name: proxypool +--- +apiVersion: v1 +items: + - apiVersion: v1 + kind: Service + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: proxypool + name: proxypool + namespace: proxypool + spec: + ports: + - name: "5555" + port: 5555 + targetPort: 5555 + selector: + io.kompose.service: proxypool + status: + loadBalancer: {} + - apiVersion: v1 + kind: Service + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: redis + name: redis + namespace: proxypool + spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + io.kompose.service: redis + status: + loadBalancer: {} + - apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: proxypool + name: proxypool + namespace: proxypool + spec: + replicas: 1 + strategy: {} + template: + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: proxypool + spec: + containers: + - env: + - name: REDIS_HOST + value: redis + image: germey/proxypool + name: proxypool + ports: + - containerPort: 5555 + resources: {} + restartPolicy: Always + status: {} + - apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: redis + name: redis + namespace: proxypool + spec: + replicas: 1 + strategy: {} + template: + metadata: + annotations: + kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml + kompose.version: 1.20.0 () + creationTimestamp: null + labels: + io.kompose.service: redis + spec: + containers: + - args: + - redis-server + image: redis:alpine + name: redis + ports: + - containerPort: 6379 + resources: {} + restartPolicy: Always + status: {} +kind: List +metadata: {} + From f568fee31b24aa5328bb356cc684f5089892cae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 12:40:51 +0800 Subject: [PATCH 013/139] Create build.yml --- .github/workflows/build.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..3bc03def --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,17 @@ +name: build +on: + push: + branches: + - master +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v1 + - name: Docker Login + run: docker login -u germey -p ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} + - name: Build the Docker Image + run: docker-compose build + - name: Push the Docker Image + run: docker-compose push From e74dec2be321b5ee44fd69d898b893bb96aaf29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Thu, 20 Feb 2020 12:45:00 +0800 Subject: [PATCH 014/139] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..22003f13 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: Germey + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environments (please complete the following information):** + - OS: [e.g. macOS 10.15.2] + - Python [e.g. Python 3.6] + - Browser [e.g. Chrome 67 ] + +**Additional context** +Add any other context about the problem here. From 3800d31604206ab4d1bb662bce4e0988cb454aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 12:53:18 +0800 Subject: [PATCH 015/139] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8f04d6d1..94221cd9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # ProxyPool +![build](https://github.com/Python3WebSpider/ProxyPool/workflows/build/badge.svg) ![](https://img.shields.io/badge/python-3.6%2B-brightgreen) ![Docker Pulls](https://img.shields.io/docker/pulls/germey/proxypool) From 832a5db6927b57590ae4547c7793f67d45e427a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:04:16 +0800 Subject: [PATCH 016/139] Create package.yml --- .github/workflows/package.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000..d7b8b2ba --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,25 @@ +name: build +on: + push: + branches: + - master + paths-ignore: + - .gitignore + - README.md + - '.github/ISSUE_TEMPLATE/**' +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v1 + - name: Setup Docker + uses: docker-practice/actions-setup-docker@0.0.1 + - name: Docker Login + run: docker login -u Germey -p ${{ secrets.github_token }} docker.pkg.github.com + - name: Build the Docker Image + run: docker-compose build + - name: Tag the Docker Image + run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool + - name: Push the Docker Image + run: docker push docker.pkg.github.com/python3webspider/proxypool From 1ae0c2abc61610347d65cb0a80eaa717435b45d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:05:15 +0800 Subject: [PATCH 017/139] Update package.yml --- .github/workflows/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d7b8b2ba..bbb78fde 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,4 +1,4 @@ -name: build +name: package on: push: branches: From 852720fcc271bf53dd240778e0000440a817960d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:13:43 +0800 Subject: [PATCH 018/139] Create deploy.yml --- .github/workflows/deploy.yml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..3b233849 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,42 @@ +name: deploy +on: + push: + branches: + - master + paths-ignore: + - .gitignore + - README.md + - '.github/ISSUE_TEMPLATE/**' +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + - name: Docker Login + uses: Azure/docker-login@v1 + with: + username: germey + password: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} + - name: Set Kubectl + uses: Azure/k8s-set-context@v1 + with: + kubeconfig: ${{ secrets.KUBE_CONFIG }} + - name: Test Kubectl + run: | + kubectl get nodes + kubectl get svc -n scrape + - name: Generate Build Number + uses: einaregilsson/build-number@v2 + with: + token: ${{ secrets.github_token }} + - name: Get Build Number + run: | + echo $BUILD_NUMBER + - name: Build Push Deploy + run: | + docker-compose build + docker tag germey/proxypool germey/proxypool:$BUILD_NUMBER + docker push germey/proxypool:$BUILD_NUMBER + cat deployment.yml | sed 's/\${TAG}/'"$BUILD_NUMBER"'/g' | kubectl apply -f - + From 9bb24e68c41a407bf30dbbd25593b0235a64690f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:14:55 +0800 Subject: [PATCH 019/139] Update package.yml --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index bbb78fde..41d134d3 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,6 +20,6 @@ jobs: - name: Build the Docker Image run: docker-compose build - name: Tag the Docker Image - run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool + run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool:master - name: Push the Docker Image - run: docker push docker.pkg.github.com/python3webspider/proxypool + run: docker push docker.pkg.github.com/python3webspider/proxypool:master From ee20972ce7a180be0abbdd245f7a45af3f6e962c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:18:53 +0800 Subject: [PATCH 020/139] Update package.yml --- .github/workflows/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 41d134d3..7a86907a 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,6 +20,6 @@ jobs: - name: Build the Docker Image run: docker-compose build - name: Tag the Docker Image - run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool:master + run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool/proxypool:master - name: Push the Docker Image - run: docker push docker.pkg.github.com/python3webspider/proxypool:master + run: docker push docker.pkg.github.com/python3webspider/proxypool/proxypool:master From f0ca5d9a461bb8776ddaeb2f96a42905f10cf791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 13:23:14 +0800 Subject: [PATCH 021/139] Update build.yml --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bc03def..84773352 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,10 @@ on: push: branches: - master + paths-ignore: + - .gitignore + - README.md + - '.github/ISSUE_TEMPLATE/**' jobs: build: runs-on: ubuntu-latest From 190a84b49a002b573d5bdafdc1c53aefccf00d18 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 13:25:59 +0800 Subject: [PATCH 022/139] update tag --- deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.yml b/deployment.yml index 40497b36..a26c441c 100644 --- a/deployment.yml +++ b/deployment.yml @@ -73,7 +73,7 @@ items: - env: - name: REDIS_HOST value: redis - image: germey/proxypool + image: germey/proxypool:{TAG} name: proxypool ports: - containerPort: 5555 From ce66b5e7bbbf3ff8087b036de28d180a1a725c8a Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 13:31:01 +0800 Subject: [PATCH 023/139] update deployment yml and readme --- README.md | 2 +- deployment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f04d6d1..f7ab9552 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ class Daili66Crawler(BaseCrawler): 本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，执行如下命令即可： ```shell script -kubectl apply -f deployment.yml +cat deployment.yml | sed 's/\${TAG}/latest/g' | kubectl apply -f - ``` ## LICENSE diff --git a/deployment.yml b/deployment.yml index a26c441c..24d1f5ac 100644 --- a/deployment.yml +++ b/deployment.yml @@ -73,7 +73,7 @@ items: - env: - name: REDIS_HOST value: redis - image: germey/proxypool:{TAG} + image: germey/proxypool:${TAG} name: proxypool ports: - containerPort: 5555 From fd91d07ad65d13bc243090844fdf01f989b7041b Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 14:14:37 +0800 Subject: [PATCH 024/139] update ingress and redis port --- ingress.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ingress.yml diff --git a/ingress.yml b/ingress.yml new file mode 100644 index 00000000..e69de29b From 2d26b90f60ab7d58e9a2fa346d91de739703e6e1 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 14:34:44 +0800 Subject: [PATCH 025/139] update --- deployment.yml | 7 +++++-- ingress.yml | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/deployment.yml b/deployment.yml index 24d1f5ac..f1159a68 100644 --- a/deployment.yml +++ b/deployment.yml @@ -59,6 +59,7 @@ items: namespace: proxypool spec: replicas: 1 + revisionHistoryLimit: 0 strategy: {} template: metadata: @@ -73,6 +74,8 @@ items: - env: - name: REDIS_HOST value: redis + - name: REDIS_PORT + value: 6379 image: germey/proxypool:${TAG} name: proxypool ports: @@ -93,6 +96,7 @@ items: namespace: proxypool spec: replicas: 1 + revisionHistoryLimit: 0 strategy: {} template: metadata: @@ -114,5 +118,4 @@ items: restartPolicy: Always status: {} kind: List -metadata: {} - +metadata: {} \ No newline at end of file diff --git a/ingress.yml b/ingress.yml index e69de29b..ae41e441 100644 --- a/ingress.yml +++ b/ingress.yml @@ -0,0 +1,22 @@ +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: ingress-universal-proxypool + namespace: proxypool + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + tls: + - hosts: + - universal.proxypool.cuiqingcai.com + secretName: tls-proxypool-cuiqingcai-com + rules: + - host: universal.proxypool.cuiqingcai.com + http: + paths: + - backend: + serviceName: proxypool + servicePort: 5555 + path: / \ No newline at end of file From 85dccebf674f2cf72be31f02030d6a843258394c Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 20 Feb 2020 14:40:05 +0800 Subject: [PATCH 026/139] update redis port --- deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.yml b/deployment.yml index f1159a68..6cfa0505 100644 --- a/deployment.yml +++ b/deployment.yml @@ -75,7 +75,7 @@ items: - name: REDIS_HOST value: redis - name: REDIS_PORT - value: 6379 + value: '6379' image: germey/proxypool:${TAG} name: proxypool ports: From d35eb8a8e17950be56a0010821790bf3cc8d9880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Thu, 20 Feb 2020 14:44:51 +0800 Subject: [PATCH 027/139] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b68fea1f..ecf9ef57 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # ProxyPool ![build](https://github.com/Python3WebSpider/ProxyPool/workflows/build/badge.svg) +![deploy](https://github.com/Python3WebSpider/ProxyPool/workflows/deploy/badge.svg) ![](https://img.shields.io/badge/python-3.6%2B-brightgreen) ![Docker Pulls](https://img.shields.io/docker/pulls/germey/proxypool) From faa39ffcfbe6bfab7d612ffc02a6bcd4ef091ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=93=E6=80=A1=E7=84=B6?= <38436053+samzhuoyiran@users.noreply.github.com> Date: Fri, 21 Feb 2020 16:10:15 +0800 Subject: [PATCH 028/139] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=85=B3?= =?UTF-8?q?=E4=BA=8EBASE=5FURL=E5=9F=9F=E5=90=8D=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=9A=84retrying=E6=A8=A1=E5=9D=97=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/crawlers/daili66.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/crawlers/daili66.py b/proxypool/crawlers/daili66.py index a6e4ca82..09a3ee45 100644 --- a/proxypool/crawlers/daili66.py +++ b/proxypool/crawlers/daili66.py @@ -3,7 +3,7 @@ from proxypool.crawlers.base import BaseCrawler -BASE_URL = 'http://www.664ip.cn/{page}.html' +BASE_URL = 'http://www.66ip.cn/{page}.html' MAX_PAGE = 5 From 5e09f65a54e4c8ed89691bde9d10e8571fae8f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 21 Feb 2020 22:10:45 +0800 Subject: [PATCH 029/139] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index ecf9ef57..c9fb7ae8 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,13 @@ class Daili66Crawler(BaseCrawler): cat deployment.yml | sed 's/\${TAG}/latest/g' | kubectl apply -f - ``` +## 待开发 + +- [ ] 前端页面管理 +- [ ] 使用情况统计分析 + +如有一起开发的兴趣可以在 Issue 留言，非常感谢！ + ## LICENSE MIT From b6498ec724df310c878e4e0a5f24401b00697826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 21 Feb 2020 22:14:11 +0800 Subject: [PATCH 030/139] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c9fb7ae8..52bc1106 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ 代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」，建议使用之前阅读。 +## 运行示例 + +API Server 可以见[部署样例](https://universal.proxypool.cuiqingcai.com/)，随机代理[取用地址](https://universal.proxypool.cuiqingcai.com/random)，代理源比较少，仅供演示。 + +本样例为 GitHub Actions + Kubernetes 自动部署 master 分支代码结果。 + ## 使用要求 可以通过两种方式来运行代理池，一种方式是使用 Docker（推荐），另一种方式是常规方式运行。 From b94ee7b02cd1eb2d6a0b1f08cdba95356cd10284 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:08:48 +0800 Subject: [PATCH 031/139] add import from module --- Dockerfile | 1 + deployment.yml | 20 ++++++++++++++++++++ docker-compose.yml | 2 ++ proxypool/crawlers/__init__.py | 20 ++++++++++++-------- proxypool/setting.py | 1 - 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3e7fa86..7f45f202 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,5 @@ FROM python:3.6 WORKDIR /app COPY . . RUN pip install -r requirements.txt +VOLUME ["/app/proxypool"] CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/deployment.yml b/deployment.yml index 6cfa0505..0db479fb 100644 --- a/deployment.yml +++ b/deployment.yml @@ -5,6 +5,19 @@ metadata: name: proxypool --- apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: proxypool + namespace: proxypool +spec: + storageClassName: azure-file + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 items: - apiVersion: v1 kind: Service @@ -81,7 +94,14 @@ items: ports: - containerPort: 5555 resources: {} + volumeMounts: + - mountPath: "/app/proxypool" + name: proxypool restartPolicy: Always + volumes: + - name: proxypool + persistentVolumeClaim: + claimName: proxypool status: {} - apiVersion: extensions/v1beta1 kind: Deployment diff --git a/docker-compose.yml b/docker-compose.yml index c05b84a5..ad49da85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,5 +14,7 @@ services: ports: - "5555:5555" restart: always + volumes: + - /tmp/proxypool:/app/proxypool environment: REDIS_HOST: redis \ No newline at end of file diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py index 0b6ca544..a626b146 100644 --- a/proxypool/crawlers/__init__.py +++ b/proxypool/crawlers/__init__.py @@ -1,10 +1,14 @@ -from .daili66 import Daili66Crawler -from .ip3366 import IP3366Crawler -from .iphai import IPHaiCrawler +import pkgutil +from .base import BaseCrawler +import inspect -__all__ = [ - Daili66Crawler, - IP3366Crawler, - IPHaiCrawler -] +# load classes subclass of BaseCrawler +classes = [] +for loader, name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(name).load_module(name) + for name, value in inspect.getmembers(module): + globals()[name] = value + if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler: + classes.append(value) +__ALL__ = classes diff --git a/proxypool/setting.py b/proxypool/setting.py index 0f2e64cd..6c2db74f 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -2,7 +2,6 @@ from os.path import dirname, abspath, join from environs import Env from loguru import logger - from proxypool.utils.parse import parse_redis_connection_string From 78ab931813ed0f73134f03f31b3056a6c55ae8a6 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:09:59 +0800 Subject: [PATCH 032/139] to crawlers --- Dockerfile | 2 +- deployment.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f45f202..0f627c53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,5 +2,5 @@ FROM python:3.6 WORKDIR /app COPY . . RUN pip install -r requirements.txt -VOLUME ["/app/proxypool"] +VOLUME ["/app/proxypool/crawlers"] CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/deployment.yml b/deployment.yml index 0db479fb..6af04f44 100644 --- a/deployment.yml +++ b/deployment.yml @@ -95,7 +95,7 @@ items: - containerPort: 5555 resources: {} volumeMounts: - - mountPath: "/app/proxypool" + - mountPath: "/app/proxypool/crawlers" name: proxypool restartPolicy: Always volumes: diff --git a/docker-compose.yml b/docker-compose.yml index ad49da85..97b0339b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,6 @@ services: - "5555:5555" restart: always volumes: - - /tmp/proxypool:/app/proxypool + - /tmp/proxypool/crawlers:/app/proxypool/crawlers environment: REDIS_HOST: redis \ No newline at end of file From 1d9046979dbb6fea040ec11e2b328a4fa36ba500 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:16:16 +0800 Subject: [PATCH 033/139] deployments --- deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.yml b/deployment.yml index 6af04f44..9c27eb21 100644 --- a/deployment.yml +++ b/deployment.yml @@ -72,7 +72,7 @@ items: namespace: proxypool spec: replicas: 1 - revisionHistoryLimit: 0 + revisionHistoryLimit: 1 strategy: {} template: metadata: From de99a2eb87158883f4df687968c0ad6faf8b7a74 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:19:11 +0800 Subject: [PATCH 034/139] split private and public --- proxypool/crawlers/private/__init__.py | 0 proxypool/crawlers/public/__init__.py | 0 proxypool/crawlers/{ => public}/daili66.py | 0 proxypool/crawlers/{ => public}/ip3366.py | 0 proxypool/crawlers/{ => public}/iphai.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 proxypool/crawlers/private/__init__.py create mode 100644 proxypool/crawlers/public/__init__.py rename proxypool/crawlers/{ => public}/daili66.py (100%) rename proxypool/crawlers/{ => public}/ip3366.py (100%) rename proxypool/crawlers/{ => public}/iphai.py (100%) diff --git a/proxypool/crawlers/private/__init__.py b/proxypool/crawlers/private/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proxypool/crawlers/public/__init__.py b/proxypool/crawlers/public/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proxypool/crawlers/daili66.py b/proxypool/crawlers/public/daili66.py similarity index 100% rename from proxypool/crawlers/daili66.py rename to proxypool/crawlers/public/daili66.py diff --git a/proxypool/crawlers/ip3366.py b/proxypool/crawlers/public/ip3366.py similarity index 100% rename from proxypool/crawlers/ip3366.py rename to proxypool/crawlers/public/ip3366.py diff --git a/proxypool/crawlers/iphai.py b/proxypool/crawlers/public/iphai.py similarity index 100% rename from proxypool/crawlers/iphai.py rename to proxypool/crawlers/public/iphai.py From 8114cab6fc06f7d288621eda31afca269bf97be5 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:19:55 +0800 Subject: [PATCH 035/139] to private --- Dockerfile | 2 +- deployment.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f627c53..dab1227e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,5 +2,5 @@ FROM python:3.6 WORKDIR /app COPY . . RUN pip install -r requirements.txt -VOLUME ["/app/proxypool/crawlers"] +VOLUME ["/app/proxypool/crawlers/private"] CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/deployment.yml b/deployment.yml index 9c27eb21..d493449d 100644 --- a/deployment.yml +++ b/deployment.yml @@ -95,7 +95,7 @@ items: - containerPort: 5555 resources: {} volumeMounts: - - mountPath: "/app/proxypool/crawlers" + - mountPath: "/app/proxypool/crawlers/private" name: proxypool restartPolicy: Always volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 97b0339b..f0181db2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,6 @@ services: - "5555:5555" restart: always volumes: - - /tmp/proxypool/crawlers:/app/proxypool/crawlers + - /tmp/proxypool/crawlers/private:/app/proxypool/crawlers/private environment: REDIS_HOST: redis \ No newline at end of file From 00e6f661946b932d218f2b7f13a5accce9879bf4 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:38:11 +0800 Subject: [PATCH 036/139] update kudaidaili --- deployment.yml | 63 ++------------------------ proxypool/crawlers/private/.gitignore | 2 + proxypool/crawlers/public/kuaidaili.py | 32 +++++++++++++ 3 files changed, 39 insertions(+), 58 deletions(-) create mode 100644 proxypool/crawlers/private/.gitignore create mode 100644 proxypool/crawlers/public/kuaidaili.py diff --git a/deployment.yml b/deployment.yml index d493449d..0aab7613 100644 --- a/deployment.yml +++ b/deployment.yml @@ -39,26 +39,6 @@ items: io.kompose.service: proxypool status: loadBalancer: {} - - apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: redis - name: redis - namespace: proxypool - spec: - ports: - - name: "6379" - port: 6379 - targetPort: 6379 - selector: - io.kompose.service: redis - status: - loadBalancer: {} - apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -85,10 +65,11 @@ items: spec: containers: - env: - - name: REDIS_HOST - value: redis - - name: REDIS_PORT - value: '6379' + - name: REDIS_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: redis + key: connection_string image: germey/proxypool:${TAG} name: proxypool ports: @@ -103,39 +84,5 @@ items: persistentVolumeClaim: claimName: proxypool status: {} - - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: redis - name: redis - namespace: proxypool - spec: - replicas: 1 - revisionHistoryLimit: 0 - strategy: {} - template: - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: redis - spec: - containers: - - args: - - redis-server - image: redis:alpine - name: redis - ports: - - containerPort: 6379 - resources: {} - restartPolicy: Always - status: {} kind: List metadata: {} \ No newline at end of file diff --git a/proxypool/crawlers/private/.gitignore b/proxypool/crawlers/private/.gitignore new file mode 100644 index 00000000..44202946 --- /dev/null +++ b/proxypool/crawlers/private/.gitignore @@ -0,0 +1,2 @@ +* +!__init__.py \ No newline at end of file diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py new file mode 100644 index 00000000..b6061579 --- /dev/null +++ b/proxypool/crawlers/public/kuaidaili.py @@ -0,0 +1,32 @@ +from proxypool.crawlers.base import BaseCrawler +from proxypool.schemas.proxy import Proxy +import re +from pyquery import PyQuery as pq + + +BASE_URL = 'https://www.kuaidaili.com/free/inha/{page}/' + + +class KuaidailiCrawler(BaseCrawler): + """ + kuaidaili crawler, https://www.kuaidaili.com/ + """ + urls = [BASE_URL.format(page=page) for page in range(1, 100)] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + for item in doc('table tr').items(): + td_ip = item.find('td[data-title="IP"]').text() + td_port = item.find('td[data-title="PORT"]').text() + if td_ip and td_port: + yield Proxy(host=td_ip, port=td_port) + + +if __name__ == '__main__': + crawler = KuaidailiCrawler() + for proxy in crawler.crawl(): + print(proxy) From 1834b3e6a13a55ba192d92830c37d15c52d69446 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:43:13 +0800 Subject: [PATCH 037/139] update deploy --- deployment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment.yml b/deployment.yml index 0aab7613..6559813f 100644 --- a/deployment.yml +++ b/deployment.yml @@ -70,6 +70,8 @@ items: secretKeyRef: name: redis key: connection_string + - name: REDIS_PORT + value: '6379' image: germey/proxypool:${TAG} name: proxypool ports: From 58762c2d5e36cfb80e766de81bfa74e692f1176d Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:48:11 +0800 Subject: [PATCH 038/139] fix --- proxypool/crawlers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py index a626b146..4f04715e 100644 --- a/proxypool/crawlers/__init__.py +++ b/proxypool/crawlers/__init__.py @@ -11,4 +11,4 @@ globals()[name] = value if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler: classes.append(value) -__ALL__ = classes +__all__ = __ALL__ = classes From eb5bbc9ced09f0feee2534f818bd9a272bd50923 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 05:55:30 +0800 Subject: [PATCH 039/139] change page --- deployment.yml | 2 +- proxypool/crawlers/public/kuaidaili.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment.yml b/deployment.yml index 6559813f..56a913e8 100644 --- a/deployment.yml +++ b/deployment.yml @@ -51,7 +51,7 @@ items: name: proxypool namespace: proxypool spec: - replicas: 1 + replicas: 2 revisionHistoryLimit: 1 strategy: {} template: diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py index b6061579..9a3c56c3 100644 --- a/proxypool/crawlers/public/kuaidaili.py +++ b/proxypool/crawlers/public/kuaidaili.py @@ -11,7 +11,7 @@ class KuaidailiCrawler(BaseCrawler): """ kuaidaili crawler, https://www.kuaidaili.com/ """ - urls = [BASE_URL.format(page=page) for page in range(1, 100)] + urls = [BASE_URL.format(page=page) for page in range(1, 200)] def parse(self, html): """ From 77dc39c6e0f5cac0f4c2b6f4483c1fcf0e23e325 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 06:04:26 +0800 Subject: [PATCH 040/139] update deploy --- proxypool/crawlers/public/ip3366.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxypool/crawlers/public/ip3366.py b/proxypool/crawlers/public/ip3366.py index e85863aa..78d29447 100644 --- a/proxypool/crawlers/public/ip3366.py +++ b/proxypool/crawlers/public/ip3366.py @@ -6,13 +6,13 @@ MAX_PAGE = 5 BASE_URL = 'http://www.ip3366.net/free/?stype=1&page={page}' + class IP3366Crawler(BaseCrawler): """ ip3366 crawler, http://www.ip3366.net/ """ urls = [BASE_URL.format(page=i) for i in range(1, 8)] - def parse(self, html): """ parse html file to get proxies @@ -25,6 +25,7 @@ def parse(self, html): proxy = Proxy(host=address.strip(), port=int(port.strip())) yield proxy + if __name__ == '__main__': crawler = IP3366Crawler() for proxy in crawler.crawl(): From fab05831f4264240253450935005df9b4918ff57 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 06:12:40 +0800 Subject: [PATCH 041/139] update tester --- proxypool/crawlers/base.py | 2 +- proxypool/processors/tester.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 1c204bb1..843bd4e7 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -24,5 +24,5 @@ def crawl(self): logger.info(f'fetching {url}') html = self.fetch(url) for proxy in self.parse(html): - logger.info(f'fetched proxy {proxy.string()}') + logger.info(f'fetched proxy {proxy.string()} from {url}') yield proxy diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 6df2f5bf..ae4da20b 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS -from aiohttp import ClientProxyConnectionError, ServerDisconnectedError +from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError from asyncio import TimeoutError @@ -12,7 +12,8 @@ ClientProxyConnectionError, ConnectionRefusedError, TimeoutError, - ServerDisconnectedError + ServerDisconnectedError, + ClientOSError ) From db7a5c2dc6d44a334c022115ac840afedd713e42 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 24 Feb 2020 14:54:35 +0800 Subject: [PATCH 042/139] update tester --- proxypool/processors/tester.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index ae4da20b..54ccdd73 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS -from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError +from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError @@ -13,7 +13,8 @@ ConnectionRefusedError, TimeoutError, ServerDisconnectedError, - ClientOSError + ClientOSError, + ClientHttpProxyError ) From 6b59fedaaffe00d556d7752aa0b4b234f326e471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 28 Feb 2020 14:45:06 +0800 Subject: [PATCH 043/139] Update setting.py --- proxypool/setting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxypool/setting.py b/proxypool/setting.py index 6c2db74f..559c2454 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -20,8 +20,8 @@ APP_ENV = env.str('APP_ENV', DEV_MODE).lower() APP_DEBUG = env.bool('APP_DEBUG', True if APP_ENV == DEV_MODE else False) APP_DEV = IS_DEV = APP_ENV == DEV_MODE -APP_PROD = IS_PROD = APP_DEV == PROD_MODE -APP_TEST = IS_TEST = APP_ENV = TEST_MODE +APP_PROD = IS_PROD = APP_ENV == PROD_MODE +APP_TEST = IS_TEST = APP_ENV == TEST_MODE # redis host REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1') From c62e275ffa2fd713a9af4edeff6f422d1d8306ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 28 Feb 2020 15:01:18 +0800 Subject: [PATCH 044/139] Update deployment.yml --- deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment.yml b/deployment.yml index 56a913e8..3e918696 100644 --- a/deployment.yml +++ b/deployment.yml @@ -15,7 +15,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 2Gi --- apiVersion: v1 items: @@ -87,4 +87,4 @@ items: claimName: proxypool status: {} kind: List -metadata: {} \ No newline at end of file +metadata: {} From 8efda1f0f034c040625486190c1e364df045a583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sat, 29 Feb 2020 03:05:17 +0800 Subject: [PATCH 045/139] Update deployment.yml --- deployment.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/deployment.yml b/deployment.yml index 3e918696..9c458d28 100644 --- a/deployment.yml +++ b/deployment.yml @@ -10,12 +10,12 @@ metadata: name: proxypool namespace: proxypool spec: - storageClassName: azure-file + storageClassName: default accessModes: - ReadWriteOnce resources: requests: - storage: 2Gi + storage: 1Gi --- apiVersion: v1 items: @@ -74,6 +74,13 @@ items: value: '6379' image: germey/proxypool:${TAG} name: proxypool + resources: + limits: + memory: "500Mi" + cpu: "300m" + requests: + memory: "500Mi" + cpu: "300m" ports: - containerPort: 5555 resources: {} From 3a7063cf26eee6ea23be0967a87fbe04a18abef6 Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 29 Feb 2020 03:17:22 +0800 Subject: [PATCH 046/139] update ingress --- ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingress.yml b/ingress.yml index ae41e441..7733ded0 100644 --- a/ingress.yml +++ b/ingress.yml @@ -11,7 +11,7 @@ spec: tls: - hosts: - universal.proxypool.cuiqingcai.com - secretName: tls-proxypool-cuiqingcai-com + secretName: tls-wildcard-proxypool-cuiqingcai-com rules: - host: universal.proxypool.cuiqingcai.com http: From 60e9af5c638a716ec1251458e00e92ae082d5c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sat, 29 Feb 2020 17:51:21 +0800 Subject: [PATCH 047/139] Update deployment.yml --- deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.yml b/deployment.yml index 9c458d28..7dea1699 100644 --- a/deployment.yml +++ b/deployment.yml @@ -12,7 +12,7 @@ metadata: spec: storageClassName: default accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 1Gi From 3ed5a8ab3896ef7905176d028cb050124529000c Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 6 Mar 2020 13:29:05 +0800 Subject: [PATCH 048/139] remove attr requirement --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 52e44e61..cb9bdcb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,5 @@ aiohttp==3.6.2 requests==2.22.0 loguru==0.3.2 pyquery==1.4.0 -attr==0.3.1 supervisor==4.1.0 redis==2.10.6 \ No newline at end of file From 0ab79b966c6fb427619940a2a5fc377fea213e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 6 Mar 2020 13:47:56 +0800 Subject: [PATCH 049/139] Update deployment.yml --- deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment.yml b/deployment.yml index 7dea1699..9c458d28 100644 --- a/deployment.yml +++ b/deployment.yml @@ -12,7 +12,7 @@ metadata: spec: storageClassName: default accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 1Gi From cd5c37a26c4a321e46909261829f4a94d8d048d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 6 Mar 2020 13:54:05 +0800 Subject: [PATCH 050/139] Update build.yml --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84773352..a6c43f03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,3 +19,8 @@ jobs: run: docker-compose build - name: Push the Docker Image run: docker-compose push + - name: Tag and Push Latest + run: | + docker tag germey/proxypool germey/proxypool:master + docker push germey/proxypool:master + From d639721537baeaff7e37d54a9e0dc6bc625e66ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 6 Mar 2020 13:54:34 +0800 Subject: [PATCH 051/139] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6c43f03..183e5a79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: run: docker-compose build - name: Push the Docker Image run: docker-compose push - - name: Tag and Push Latest + - name: Tag and Push Master Version run: | docker tag germey/proxypool germey/proxypool:master docker push germey/proxypool:master From 376ecbee0375f59e9566334bb922de2c20341362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Fri, 6 Mar 2020 13:55:34 +0800 Subject: [PATCH 052/139] Update deploy.yml --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3b233849..3e028e21 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: deploy on: push: branches: - - master + - release paths-ignore: - .gitignore - README.md From cb33d90f13c489101066c42030f1366e1ec586ba Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 7 Mar 2020 17:32:09 +0800 Subject: [PATCH 053/139] update deployment --- .github/workflows/deploy.yml | 2 +- deployment.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3b233849..3e028e21 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: deploy on: push: branches: - - master + - release paths-ignore: - .gitignore - README.md diff --git a/deployment.yml b/deployment.yml index 7dea1699..3748e693 100644 --- a/deployment.yml +++ b/deployment.yml @@ -10,12 +10,12 @@ metadata: name: proxypool namespace: proxypool spec: - storageClassName: default + storageClassName: azure-file accessModes: - ReadWriteMany resources: requests: - storage: 1Gi + storage: 2Gi --- apiVersion: v1 items: @@ -91,7 +91,7 @@ items: volumes: - name: proxypool persistentVolumeClaim: - claimName: proxypool + claimName: pvc-proxypool status: {} kind: List metadata: {} From 457c08d20320bd866bba70a494c55734cacda09e Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 7 Mar 2020 17:33:30 +0800 Subject: [PATCH 054/139] update --- deployment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deployment.yml b/deployment.yml index 5319e105..9427bad2 100644 --- a/deployment.yml +++ b/deployment.yml @@ -12,7 +12,7 @@ metadata: spec: storageClassName: azure-file accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 2Gi @@ -78,12 +78,11 @@ items: limits: memory: "500Mi" cpu: "300m" - requests: + requests: memory: "500Mi" cpu: "300m" ports: - containerPort: 5555 - resources: {} volumeMounts: - mountPath: "/app/proxypool/crawlers/private" name: proxypool From 4bca2d6de062a10253f998eaf7f150bf4293c270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sun, 8 Mar 2020 00:59:15 +0800 Subject: [PATCH 055/139] Update redis.py --- proxypool/storages/redis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 06971cea..794f6e40 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -39,8 +39,8 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: return if not self.exists(proxy): if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, score, proxy) - return self.db.zadd(REDIS_KEY, {proxy: score}) + return self.db.zadd(REDIS_KEY, score, proxy.string()) + return self.db.zadd(REDIS_KEY, {proxy.string(): score}) def random(self) -> Proxy: """ @@ -77,7 +77,7 @@ def decrease(self, proxy: Proxy) -> int: # otherwise delete proxy else: logger.info(f'{proxy.string()} current score {score}, remove') - return self.db.zrem(REDIS_KEY, proxy) + return self.db.zrem(REDIS_KEY, proxy.string()) def exists(self, proxy: Proxy) -> bool: """ From 9f14edfc61953124c310f5392f5b6eeb6d4beabc Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 22 May 2020 20:33:00 +0800 Subject: [PATCH 056/139] add xici and data5u --- proxypool/crawlers/base.py | 1 + proxypool/crawlers/public/data5u.py | 47 ++++++++++++++++++++++++ proxypool/crawlers/public/xicidaili.py | 50 ++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 proxypool/crawlers/public/data5u.py create mode 100644 proxypool/crawlers/public/xicidaili.py diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 843bd4e7..fcb28a3f 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -23,6 +23,7 @@ def crawl(self): for url in self.urls: logger.info(f'fetching {url}') html = self.fetch(url) + print('html', html) for proxy in self.parse(html): logger.info(f'fetched proxy {proxy.string()} from {url}') yield proxy diff --git a/proxypool/crawlers/public/data5u.py b/proxypool/crawlers/public/data5u.py new file mode 100644 index 00000000..e36bf664 --- /dev/null +++ b/proxypool/crawlers/public/data5u.py @@ -0,0 +1,47 @@ +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from loguru import logger + +BASE_URL = 'http://www.data5u.com' + + +class Data5UCrawler(BaseCrawler): + """ + data5u crawler, http://www.data5u.com + """ + urls = [BASE_URL] + + headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' + } + + @logger.catch + def crawl(self): + """ + crawl main method + """ + for url in self.urls: + logger.info(f'fetching {url}') + html = self.fetch(url, headers=self.headers) + for proxy in self.parse(html): + logger.info(f'fetched proxy {proxy.string()} from {url}') + yield proxy + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + items = doc('.wlist ul.l2').items() + for item in items: + host = item.find('span:first-child').text() + port = int(item.find('span:nth-child(2)').text()) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = Data5UCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/xicidaili.py b/proxypool/crawlers/public/xicidaili.py new file mode 100644 index 00000000..e8fa1788 --- /dev/null +++ b/proxypool/crawlers/public/xicidaili.py @@ -0,0 +1,50 @@ +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from loguru import logger + +BASE_URL = 'https://www.xicidaili.com/' + + +class XicidailiCrawler(BaseCrawler): + """ + xididaili crawler, https://www.xicidaili.com/ + """ + urls = [BASE_URL] + + headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' + } + + @logger.catch + def crawl(self): + """ + crawl main method + """ + for url in self.urls: + logger.info(f'fetching {url}') + html = self.fetch(url, headers=self.headers) + for proxy in self.parse(html): + logger.info(f'fetched proxy {proxy.string()} from {url}') + yield proxy + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + items = doc('#ip_list tr:contains(高匿)').items() + for item in items: + country = item.find('td.country').text() + if not country or country.strip() != '高匿': + continue + host = item.find('td:nth-child(2)').text() + port = int(item.find('td:nth-child(3)').text()) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = XicidailiCrawler() + for proxy in crawler.crawl(): + print(proxy) From cc58ece963e557fbd923e4ba667b86ca70bc1f74 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 22 May 2020 20:39:47 +0800 Subject: [PATCH 057/139] update workflow --- .github/workflows/deploy.yml | 1 + .github/workflows/package.yml | 25 ------------------------- deployment.yml | 5 ++++- 3 files changed, 5 insertions(+), 26 deletions(-) delete mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3e028e21..8257ba0a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,6 +37,7 @@ jobs: run: | docker-compose build docker tag germey/proxypool germey/proxypool:$BUILD_NUMBER + docker push germey/proxypool docker push germey/proxypool:$BUILD_NUMBER cat deployment.yml | sed 's/\${TAG}/'"$BUILD_NUMBER"'/g' | kubectl apply -f - diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index 7a86907a..00000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: package -on: - push: - branches: - - master - paths-ignore: - - .gitignore - - README.md - - '.github/ISSUE_TEMPLATE/**' -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout Source - uses: actions/checkout@v1 - - name: Setup Docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: Docker Login - run: docker login -u Germey -p ${{ secrets.github_token }} docker.pkg.github.com - - name: Build the Docker Image - run: docker-compose build - - name: Tag the Docker Image - run: docker tag germey/proxypool docker.pkg.github.com/python3webspider/proxypool/proxypool:master - - name: Push the Docker Image - run: docker push docker.pkg.github.com/python3webspider/proxypool/proxypool:master diff --git a/deployment.yml b/deployment.yml index 9427bad2..c7aaea55 100644 --- a/deployment.yml +++ b/deployment.yml @@ -39,7 +39,7 @@ items: io.kompose.service: proxypool status: loadBalancer: {} - - apiVersion: extensions/v1beta1 + - apiVersion: apps/v1 kind: Deployment metadata: annotations: @@ -54,6 +54,9 @@ items: replicas: 2 revisionHistoryLimit: 1 strategy: {} + selector: + matchLabels: + io.kompose.service: proxypool template: metadata: annotations: From a9ef0f407e4b86d1e0a0cde7a231fea3a860827b Mon Sep 17 00:00:00 2001 From: linupy Date: Sun, 24 May 2020 21:43:17 +0800 Subject: [PATCH 058/139] add xiladaili proxy source and add lxml in requirements.txt --- proxypool/crawlers/public/xiladaili.py | 32 ++++++++++++++++++++++++++ requirements.txt | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 proxypool/crawlers/public/xiladaili.py diff --git a/proxypool/crawlers/public/xiladaili.py b/proxypool/crawlers/public/xiladaili.py new file mode 100644 index 00000000..70a75ff1 --- /dev/null +++ b/proxypool/crawlers/public/xiladaili.py @@ -0,0 +1,32 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from lxml import etree + +BASE_URL = "http://www.xiladaili.com/" +MAX_PAGE = 5 + + +class XiladailiCrawler(BaseCrawler): + """ + xiladaili crawler, http://www.xiladaili.com/ + """ + urls = ["http://www.xiladaili.com/"] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + etree_html = etree.HTML(html) + ip_ports = etree_html.xpath("//tbody/tr/td[1]/text()") + + for ip_port in ip_ports: + host = ip_port.partition(":")[0] + port = ip_port.partition(":")[2] + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = XiladailiCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/requirements.txt b/requirements.txt index cb9bdcb5..cc0b6111 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ requests==2.22.0 loguru==0.3.2 pyquery==1.4.0 supervisor==4.1.0 -redis==2.10.6 \ No newline at end of file +redis==2.10.6 +lxml==4.3.3 \ No newline at end of file From 2cd11fa9b949416bd221c77ce81fecceb53518f7 Mon Sep 17 00:00:00 2001 From: Detolv <617248224@qq.com> Date: Sat, 13 Jun 2020 10:22:51 +0800 Subject: [PATCH 059/139] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dis=5Fvalid=5Fproxy?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E7=9A=84bug=EF=BC=8Cadd:=20=E7=AB=99?= =?UTF-8?q?=E5=A4=A7=E7=88=B7crawler=20(#68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复is_valid_proxy函数的bug，add: 站大爷crawler * fix: 修改__init__中 站大爷 detailcrawler的判断逻辑，使其更为通用 --- proxypool/crawlers/__init__.py | 5 ++- proxypool/crawlers/public/zhandaye.py | 59 +++++++++++++++++++++++++++ proxypool/utils/proxy.py | 30 ++++++++++---- 3 files changed, 85 insertions(+), 9 deletions(-) create mode 100755 proxypool/crawlers/public/zhandaye.py diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py index 4f04715e..6b775873 100644 --- a/proxypool/crawlers/__init__.py +++ b/proxypool/crawlers/__init__.py @@ -1,14 +1,15 @@ import pkgutil from .base import BaseCrawler +from .public.zhandaye import ZhandayeDetailCrawler import inspect - # load classes subclass of BaseCrawler classes = [] for loader, name, is_pkg in pkgutil.walk_packages(__path__): module = loader.find_module(name).load_module(name) for name, value in inspect.getmembers(module): globals()[name] = value - if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler: + if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler \ + and not getattr(value, 'ignore', False): classes.append(value) __all__ = __ALL__ = classes diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py new file mode 100755 index 00000000..6a80ee3b --- /dev/null +++ b/proxypool/crawlers/public/zhandaye.py @@ -0,0 +1,59 @@ +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from loguru import logger + + +BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html' +MAX_PAGE = 5 + +class ZhandayeCrawler(BaseCrawler): + """ + zhandaye crawler, https://www.zdaye.com/dayProxy/ + """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)] + + def crawl(self): + for url in self.urls: + logger.info(f'fetching {url}') + if not self.headers: + html = self.fetch(url) + else: + html = self.fetch(url, headers=self.headers) + self.parse(html) + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + for item in doc('#J_posts_list .thread_item div div p a').items(): + post = 'https://www.zdaye.com' + item.attr('href') + logger.info(f'get detail url: {post}') + ZhandayeDetailCrawler(post).crawl() + + +class ZhandayeDetailCrawler(BaseCrawler): + urls = [] + ignore = True + + def __init__(self, url): + self.urls.append(url) + super().__init__() + + def parse(self, html): + doc = pq(html) + trs = doc('.cont br').items() + for tr in trs: + line = tr[0].tail + host = line.split(':')[0] + port = line.split(':')[1][:4] + yield Proxy(host=host, port=port) + + + +if __name__ == '__main__': + crawler = ZhandayeCrawler() + for proxy in crawler.crawl(): + print(proxy) \ No newline at end of file diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index 00302b35..b22e05a4 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -1,14 +1,30 @@ from proxypool.schemas import Proxy -import re def is_valid_proxy(data): - """ - is data is valid proxy format - :param data: - :return: - """ - return re.match('\d+\.\d+\.\d+\.\d+\:\d+', data) + if data.__contains__(':'): + ip = data.split(':')[0] + port = data.split(':')[1] + return is_ip_valid(ip) and is_port_valid(port) + else: + return is_ip_valid(data) + + +def is_ip_valid(ip): + a = ip.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + return True + + +def is_port_valid(port): + return port.isdigit() def convert_proxy_or_proxies(data): From 50a8f7e711950dafde0973dbc3910cd7d520048e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B7=AF=E8=B7=AF?= Date: Sun, 5 Jul 2020 23:51:43 +0800 Subject: [PATCH 060/139] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=9A=E9=87=87=E9=9B=86=E5=88=B0=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=90=86IP=E6=98=AF=E5=90=A6=E4=B8=BA=E5=8C=BF?= =?UTF-8?q?=E5=90=8D=E4=BB=A3=E7=90=86=20(#74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * anonymous proxy ip Add verification of whether proxy IP is anonymous * catch AssertionError fix: AssertionError will now be caught --- proxypool/processors/tester.py | 17 +++++++++++++++-- proxypool/setting.py | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 54ccdd73..1ab37116 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -3,7 +3,7 @@ from loguru import logger from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient -from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError @@ -14,7 +14,8 @@ TimeoutError, ServerDisconnectedError, ClientOSError, - ClientHttpProxyError + ClientHttpProxyError, + AssertionError ) @@ -39,6 +40,18 @@ async def test(self, proxy: Proxy): async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: try: logger.debug(f'testing {proxy.string()}') + # if TEST_ANONYMOUS is True, make sure that + # the proxy has the effect of hiding the real IP + if TEST_ANONYMOUS: + url = 'https://httpbin.org/ip' + async with session.get(url, timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + origin_ip = resp_json['origin'] + async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + anonymous_ip = resp_json['origin'] + assert origin_ip != anonymous_ip + assert proxy.host == anonymous_ip async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, allow_redirects=False) as response: if response.status in TEST_VALID_STATUS: diff --git a/proxypool/setting.py b/proxypool/setting.py index 559c2454..fc53fec5 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -56,6 +56,8 @@ TEST_URL = env.str('TEST_URL', 'http://www.baidu.com') TEST_TIMEOUT = env.int('TEST_TIMEOUT', 10) TEST_BATCH = env.int('TEST_BATCH', 20) +# only save anonymous proxy +TEST_ANONYMOUS = True # TEST_HEADERS = env.json('TEST_HEADERS', { # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) From b42c77f4e00836334658f5b19a8cb14129e32908 Mon Sep 17 00:00:00 2001 From: winturn Date: Sat, 11 Jul 2020 18:29:59 +0800 Subject: [PATCH 061/139] =?UTF-8?q?=E8=A7=A3=E5=86=B3bug=EF=BC=9AZhandayeC?= =?UTF-8?q?rawler=E6=B2=A1=E6=9C=89headers=E5=B1=9E=E6=80=A7=20(#76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 解决bug：ZhandayeCrawler没有headers属性 解决bug: AttributeError: 'ZhandayeCrawler' object has no attribute 'headers' * Add: 在getter中进行重试时，增加时间间隔 --- proxypool/crawlers/base.py | 2 +- proxypool/crawlers/public/zhandaye.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index fcb28a3f..031a962d 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -6,7 +6,7 @@ class BaseCrawler(object): urls = [] - @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None) + @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000) def fetch(self, url, **kwargs): try: response = requests.get(url, **kwargs) diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py index 6a80ee3b..74266f46 100755 --- a/proxypool/crawlers/public/zhandaye.py +++ b/proxypool/crawlers/public/zhandaye.py @@ -13,13 +13,14 @@ class ZhandayeCrawler(BaseCrawler): """ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)] + headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' + } + def crawl(self): for url in self.urls: logger.info(f'fetching {url}') - if not self.headers: - html = self.fetch(url) - else: - html = self.fetch(url, headers=self.headers) + html = self.fetch(url, headers=self.headers) self.parse(html) def parse(self, html): @@ -56,4 +57,4 @@ def parse(self, html): if __name__ == '__main__': crawler = ZhandayeCrawler() for proxy in crawler.crawl(): - print(proxy) \ No newline at end of file + print(proxy) From 51908cb9fec07472b86438a1e5b45a3a689cc009 Mon Sep 17 00:00:00 2001 From: bnighting Date: Sat, 11 Jul 2020 18:31:02 +0800 Subject: [PATCH 062/139] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Redis=E5=8F=AF?= =?UTF-8?q?=E9=80=89=E6=8B=A9db=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/setting.py | 6 ++++-- proxypool/storages/redis.py | 24 ++++++++++++------------ proxypool/utils/parse.py | 6 +++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/proxypool/setting.py b/proxypool/setting.py index fc53fec5..b73ab85a 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -29,11 +29,13 @@ REDIS_PORT = env.int('REDIS_PORT', 6379) # redis password, if no password, set it to None REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) -# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port +# redis db, if no choice, set it to 0 +REDIS_DB = env.int('REDIS_DB', 1) +# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0 REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) if REDIS_CONNECTION_STRING: - REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING) + REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING) # redis hash table key name REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 794f6e40..d2fb2975 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -1,7 +1,7 @@ import redis from proxypool.exceptions import PoolEmptyException from proxypool.schemas.proxy import Proxy -from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ +from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ PROXY_SCORE_INIT from random import choice from typing import List @@ -17,16 +17,16 @@ class RedisClient(object): """ redis connection client of proxypool """ - - def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, **kwargs): + + def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs): """ init redis client :param host: redis host :param port: redis port :param password: redis password """ - self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=True, **kwargs) - + self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) + def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: """ add proxy and set it to init score @@ -41,7 +41,7 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: if IS_REDIS_VERSION_2: return self.db.zadd(REDIS_KEY, score, proxy.string()) return self.db.zadd(REDIS_KEY, {proxy.string(): score}) - + def random(self) -> Proxy: """ get random proxy @@ -60,7 +60,7 @@ def random(self) -> Proxy: return convert_proxy_or_proxies(choice(proxies)) # else raise error raise PoolEmptyException - + def decrease(self, proxy: Proxy) -> int: """ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it @@ -78,7 +78,7 @@ def decrease(self, proxy: Proxy) -> int: else: logger.info(f'{proxy.string()} current score {score}, remove') return self.db.zrem(REDIS_KEY, proxy.string()) - + def exists(self, proxy: Proxy) -> bool: """ if proxy exists @@ -86,7 +86,7 @@ def exists(self, proxy: Proxy) -> bool: :return: if exists, bool """ return not self.db.zscore(REDIS_KEY, proxy.string()) is None - + def max(self, proxy: Proxy) -> int: """ set proxy to max score @@ -97,21 +97,21 @@ def max(self, proxy: Proxy) -> int: if IS_REDIS_VERSION_2: return self.db.zadd(REDIS_KEY, PROXY_SCORE_MAX, proxy.string()) return self.db.zadd(REDIS_KEY, {proxy.string(): PROXY_SCORE_MAX}) - + def count(self) -> int: """ get count of proxies :return: count, int """ return self.db.zcard(REDIS_KEY) - + def all(self) -> List[Proxy]: """ get all proxies :return: list of proxies """ return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) - + def batch(self, start, end) -> List[Proxy]: """ get batch of proxies diff --git a/proxypool/utils/parse.py b/proxypool/utils/parse.py index ceee566d..b3f42f5f 100644 --- a/proxypool/utils/parse.py +++ b/proxypool/utils/parse.py @@ -8,6 +8,6 @@ def parse_redis_connection_string(connection_string): :param connection_string: :return: """ - result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string) - return result.group(2), int(result.group(3)), (result.group(1) or None) if result \ - else ('localhost', 6379, None) \ No newline at end of file + result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)\/(\d+)', connection_string) + return result.group(2), int(result.group(3)), (result.group(1) or None), (result.group(4) or 0) if result \ + else ('localhost', 6379, None) From 2bec99ff3ecc84cedbf2ed6c92f5d917630c58bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sat, 11 Jul 2020 18:33:09 +0800 Subject: [PATCH 063/139] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 52bc1106..47bdc6a8 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ get random proxy 116.196.115.209:8080 * REDIS_HOST：Redis 的 Host * REDIS_PORT：Redis 的端口 * REDIS_PASSWORD：Redis 的密码 +* REDIS_DB：Redis 的数据库索引，如 0、1 * REDIS_CONNECTION_STRING：Redis 连接字符串 * REDIS_KEY：Redis 储存代理使用字典的名称 From 603c08ddcc35713cc0a2aa8ae91784d74b9a15d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sat, 11 Jul 2020 18:35:23 +0800 Subject: [PATCH 064/139] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 47bdc6a8..201ab5cc 100644 --- a/README.md +++ b/README.md @@ -88,10 +88,10 @@ export REDIS_PASSWORD='' 或者只设置连接字符串： ```shell script -export REDIS_CONNECTION_STRING='redis://[password]@host:port' +export REDIS_CONNECTION_STRING='redis://[password]@host:port/db' ``` -这里连接字符串的格式需要符合 `redis://[password]@host:port` 的格式。 +这里连接字符串的格式需要符合 `redis://[password]@host:port/db` 的格式。 以上两种设置任选其一即可。 From def06b19b95c95c42eaebd45443bb2fdd22ca1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sat, 11 Jul 2020 18:36:27 +0800 Subject: [PATCH 065/139] Update setting.py --- proxypool/setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/setting.py b/proxypool/setting.py index b73ab85a..8bc23fd5 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -30,7 +30,7 @@ # redis password, if no password, set it to None REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) # redis db, if no choice, set it to 0 -REDIS_DB = env.int('REDIS_DB', 1) +REDIS_DB = env.int('REDIS_DB', 0) # redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0 REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) From 737998f2f05a3d1025cdd074da00dedf0ca4fbab Mon Sep 17 00:00:00 2001 From: winturn Date: Tue, 14 Jul 2020 03:18:56 +0800 Subject: [PATCH 066/139] =?UTF-8?q?=E8=A7=A3=E5=86=B3bug:=20redis=E6=9C=89?= =?UTF-8?q?=E5=BA=8F=E9=9B=86=E5=90=88=E5=8F=98=E5=8C=96=E5=90=8E=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E9=A1=BA=E5=BA=8F=E9=87=8D=E6=8E=92=EF=BC=8C=E4=BB=A5?= =?UTF-8?q?=E8=87=B3=E4=BA=8E=E9=87=8D=E5=A4=8Dtest=20(#78)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * pip使用镜像 * 获取代理时设置timeout 因为在获取某些网站公布的代理ip时，由于该网站被墙或者其他原因，导致需要几分钟才能反馈。 * 第5页之后的代理ip都是两三天之后的信息，质量很差 * 解决bug: redis有序集合变化后导致顺序重排，以至于重复test 解决bug: #73 因为redis的有序集合是按照分数进行变化的。 当修改了分数之后，再继续遍历时，会导致一部分重复遍历，一部分没有取到。 因此改为通过游标进行遍历 * 保证redis中不存在分数为0的数据 因为之前的逻辑是要么减分，要么删除。 也就是说，如果减完分之后，分数是0，那么还存在于redis中。 现在对这个逻辑进行了优化。减完分之后再和PROXY_SCORE_MIN的值进行判断。 * 解决bug：设置LOG_DIR后没有效果 #62 * 删除没用的import * 修改pip源 * 修改BUG：获取分数和判断分数的逻辑 * 删除不能使用的代理iphai和xicidaili, 解决bug：zhandaye * 解决bug：只获取了一次zhandaye的目录 * 解决bug：因为zhandaye的crawl没有返回值导致报错 * 恢复iphai和xicidaili。同时对这两个类增加了ignore属性 * 增加MAX_PAGE变量 * 设置pip源 * 恢复Dockerfile --- proxypool/crawlers/__init__.py | 3 +- proxypool/crawlers/base.py | 5 +++- proxypool/crawlers/public/iphai.py | 3 +- proxypool/crawlers/public/kuaidaili.py | 3 +- proxypool/crawlers/public/xicidaili.py | 2 ++ proxypool/crawlers/public/zhandaye.py | 40 ++++++++++++-------------- proxypool/processors/getter.py | 3 +- proxypool/processors/tester.py | 17 +++++------ proxypool/setting.py | 6 ++-- proxypool/storages/redis.py | 26 ++++++++--------- 10 files changed, 59 insertions(+), 49 deletions(-) diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py index 6b775873..d2ad0b4c 100644 --- a/proxypool/crawlers/__init__.py +++ b/proxypool/crawlers/__init__.py @@ -1,8 +1,8 @@ import pkgutil from .base import BaseCrawler -from .public.zhandaye import ZhandayeDetailCrawler import inspect + # load classes subclass of BaseCrawler classes = [] for loader, name, is_pkg in pkgutil.walk_packages(__path__): @@ -13,3 +13,4 @@ and not getattr(value, 'ignore', False): classes.append(value) __all__ = __ALL__ = classes + diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 031a962d..aa35430e 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -1,6 +1,7 @@ from retrying import retry import requests from loguru import logger +from proxypool.setting import GET_TIMEOUT class BaseCrawler(object): @@ -9,8 +10,11 @@ class BaseCrawler(object): @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000) def fetch(self, url, **kwargs): try: + kwargs.setdefault('timeout', GET_TIMEOUT) + kwargs.setdefault('verify', False) response = requests.get(url, **kwargs) if response.status_code == 200: + response.encoding = 'utf-8' return response.text except requests.ConnectionError: return @@ -23,7 +27,6 @@ def crawl(self): for url in self.urls: logger.info(f'fetching {url}') html = self.fetch(url) - print('html', html) for proxy in self.parse(html): logger.info(f'fetched proxy {proxy.string()} from {url}') yield proxy diff --git a/proxypool/crawlers/public/iphai.py b/proxypool/crawlers/public/iphai.py index 57638a2b..baa79834 100644 --- a/proxypool/crawlers/public/iphai.py +++ b/proxypool/crawlers/public/iphai.py @@ -10,7 +10,7 @@ class IPHaiCrawler(BaseCrawler): iphai crawler, http://www.iphai.com/ """ urls = [BASE_URL] - + ignore = True def parse(self, html): """ @@ -32,3 +32,4 @@ def parse(self, html): crawler = IPHaiCrawler() for proxy in crawler.crawl(): print(proxy) + diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py index 9a3c56c3..f3fa6437 100644 --- a/proxypool/crawlers/public/kuaidaili.py +++ b/proxypool/crawlers/public/kuaidaili.py @@ -5,13 +5,14 @@ BASE_URL = 'https://www.kuaidaili.com/free/inha/{page}/' +MAX_PAGE = 5 class KuaidailiCrawler(BaseCrawler): """ kuaidaili crawler, https://www.kuaidaili.com/ """ - urls = [BASE_URL.format(page=page) for page in range(1, 200)] + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] def parse(self, html): """ diff --git a/proxypool/crawlers/public/xicidaili.py b/proxypool/crawlers/public/xicidaili.py index e8fa1788..fdd2a317 100644 --- a/proxypool/crawlers/public/xicidaili.py +++ b/proxypool/crawlers/public/xicidaili.py @@ -11,6 +11,7 @@ class XicidailiCrawler(BaseCrawler): xididaili crawler, https://www.xicidaili.com/ """ urls = [BASE_URL] + ignore = True headers = { 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' @@ -48,3 +49,4 @@ def parse(self, html): crawler = XicidailiCrawler() for proxy in crawler.crawl(): print(proxy) + diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py index 74266f46..fe369d41 100755 --- a/proxypool/crawlers/public/zhandaye.py +++ b/proxypool/crawlers/public/zhandaye.py @@ -2,6 +2,7 @@ from proxypool.schemas.proxy import Proxy from proxypool.crawlers.base import BaseCrawler from loguru import logger +import re BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html' @@ -11,50 +12,47 @@ class ZhandayeCrawler(BaseCrawler): """ zhandaye crawler, https://www.zdaye.com/dayProxy/ """ - urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)] - + urls_catalog = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)] headers = { 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' } + urls = [] def crawl(self): - for url in self.urls: + self.crawl_catalog() + yield from super().crawl() + + def crawl_catalog(self): + for url in self.urls_catalog: logger.info(f'fetching {url}') html = self.fetch(url, headers=self.headers) - self.parse(html) + self.parse_catalog(html) - def parse(self, html): + def parse_catalog(self, html): """ parse html file to get proxies :return: """ doc = pq(html) for item in doc('#J_posts_list .thread_item div div p a').items(): - post = 'https://www.zdaye.com' + item.attr('href') - logger.info(f'get detail url: {post}') - ZhandayeDetailCrawler(post).crawl() - - -class ZhandayeDetailCrawler(BaseCrawler): - urls = [] - ignore = True - - def __init__(self, url): - self.urls.append(url) - super().__init__() + url = 'https://www.zdaye.com' + item.attr('href') + logger.info(f'get detail url: {url}') + self.urls.append(url) def parse(self, html): doc = pq(html) trs = doc('.cont br').items() for tr in trs: line = tr[0].tail - host = line.split(':')[0] - port = line.split(':')[1][:4] - yield Proxy(host=host, port=port) - + match = re.search(r'(\d+\.\d+\.\d+\.\d+):(\d+)', line) + if match: + host = match.group(1) + port = match.group(2) + yield Proxy(host=host, port=port) if __name__ == '__main__': crawler = ZhandayeCrawler() for proxy in crawler.crawl(): print(proxy) + diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py index 4bd59a4f..1a1d5261 100644 --- a/proxypool/processors/getter.py +++ b/proxypool/processors/getter.py @@ -33,10 +33,11 @@ def run(self): if self.is_full(): return for crawler in self.crawlers: + logger.info(f'crawler {crawler} to get proxy') for proxy in crawler.crawl(): self.redis.add(proxy) if __name__ == '__main__': getter = Getter() - getter.run() \ No newline at end of file + getter.run() diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 1ab37116..e0812110 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -74,14 +74,15 @@ def run(self): logger.info('stating tester...') count = self.redis.count() logger.debug(f'{count} proxies to test') - for i in range(0, count, TEST_BATCH): - # start end end offset - start, end = i, min(i + TEST_BATCH, count) - logger.debug(f'testing proxies from {start} to {end} indices') - proxies = self.redis.batch(start, end) - tasks = [self.test(proxy) for proxy in proxies] - # run tasks using event loop - self.loop.run_until_complete(asyncio.wait(tasks)) + cursor = 0 + while True: + logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') + cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) + if proxies: + tasks = [self.test(proxy) for proxy in proxies] + self.loop.run_until_complete(asyncio.wait(tasks)) + if not cursor: + break if __name__ == '__main__': diff --git a/proxypool/setting.py b/proxypool/setting.py index 8bc23fd5..7c3008b3 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -53,6 +53,7 @@ CYCLE_TESTER = env.int('CYCLE_TESTER', 20) # definition of getter cycle, it will get proxy every CYCLE_GETTER second CYCLE_GETTER = env.int('CYCLE_GETTER', 100) +GET_TIMEOUT = env.int('GET_TIMEOUT', 10) # definition of tester TEST_URL = env.str('TEST_URL', 'http://www.baidu.com') @@ -75,5 +76,6 @@ ENABLE_GETTER = env.bool('ENABLE_GETTER', True) ENABLE_SERVER = env.bool('ENABLE_SERVER', True) -logger.add(env.str('LOG_RUNTIME_FILE', 'runtime.log'), level='DEBUG', rotation='1 week', retention='20 days') -logger.add(env.str('LOG_ERROR_FILE', 'error.log'), level='ERROR', rotation='1 week') +logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') +logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') + diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index d2fb2975..60a03e9f 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -67,17 +67,15 @@ def decrease(self, proxy: Proxy) -> int: :param proxy: proxy :return: new score """ - score = self.db.zscore(REDIS_KEY, proxy.string()) - # current score is larger than PROXY_SCORE_MIN - if score and score > PROXY_SCORE_MIN: - logger.info(f'{proxy.string()} current score {score}, decrease 1') - if IS_REDIS_VERSION_2: - return self.db.zincrby(REDIS_KEY, proxy.string(), -1) - return self.db.zincrby(REDIS_KEY, -1, proxy.string()) - # otherwise delete proxy + if IS_REDIS_VERSION_2: + self.db.zincrby(REDIS_KEY, proxy.string(), -1) else: + self.db.zincrby(REDIS_KEY, -1, proxy.string()) + score = self.db.zscore(REDIS_KEY, proxy.string()) + logger.info(f'{proxy.string()} score decrease 1, current {score}') + if score <= PROXY_SCORE_MIN: logger.info(f'{proxy.string()} current score {score}, remove') - return self.db.zrem(REDIS_KEY, proxy.string()) + self.db.zrem(REDIS_KEY, proxy.string()) def exists(self, proxy: Proxy) -> bool: """ @@ -112,17 +110,19 @@ def all(self) -> List[Proxy]: """ return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) - def batch(self, start, end) -> List[Proxy]: + def batch(self, cursor, count) -> List[Proxy]: """ get batch of proxies - :param start: start index - :param end: end index + :param cursor: scan cursor + :param count: scan count :return: list of proxies """ - return convert_proxy_or_proxies(self.db.zrevrange(REDIS_KEY, start, end - 1)) + cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) + return cursor, convert_proxy_or_proxies([i[0] for i in proxies]) if __name__ == '__main__': conn = RedisClient() result = conn.random() print(result) + From 2fd55da92233a1b702b77ef7368bb326ab0c068a Mon Sep 17 00:00:00 2001 From: Germey Date: Tue, 14 Jul 2020 03:45:16 +0800 Subject: [PATCH 067/139] ignore zhandaye --- README.md | 9 ++++++++- proxypool/crawlers/public/zhandaye.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 201ab5cc..ee7c108b 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat export REDIS_HOST='localhost' export REDIS_PORT=6379 export REDIS_PASSWORD='' +export REDIS_DB=0 ``` 或者只设置连接字符串： @@ -91,7 +92,13 @@ export REDIS_PASSWORD='' export REDIS_CONNECTION_STRING='redis://[password]@host:port/db' ``` -这里连接字符串的格式需要符合 `redis://[password]@host:port/db` 的格式。 +如果没有密码也要设置为： + +```shell script +export REDIS_CONNECTION_STRING='redis://@host:port/db' +``` + +这里连接字符串的格式需要符合 `redis://[password]@host:port/db` 的格式，注意不要遗漏 `@`。 以上两种设置任选其一即可。 diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py index fe369d41..b6278a28 100755 --- a/proxypool/crawlers/public/zhandaye.py +++ b/proxypool/crawlers/public/zhandaye.py @@ -17,6 +17,7 @@ class ZhandayeCrawler(BaseCrawler): 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' } urls = [] + ignore = True def crawl(self): self.crawl_catalog() From 8c98fd0be72e2f8a9ff4e2719475845b807147d0 Mon Sep 17 00:00:00 2001 From: Germey Date: Tue, 14 Jul 2020 04:24:42 +0800 Subject: [PATCH 068/139] update ingress --- ingress.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ingress.yml b/ingress.yml index 7733ded0..166eb729 100644 --- a/ingress.yml +++ b/ingress.yml @@ -12,8 +12,18 @@ spec: - hosts: - universal.proxypool.cuiqingcai.com secretName: tls-wildcard-proxypool-cuiqingcai-com + - hosts: + - proxypool.scrape.center + secretName: tls-wildcard-scrape-center rules: - host: universal.proxypool.cuiqingcai.com + http: + paths: + - backend: + serviceName: proxypool + servicePort: 5555 + path: / + - host: proxypool.scrape.center http: paths: - backend: From dbba5376a141470fe0e15f12b5026f0d5599f34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Tue, 14 Jul 2020 04:28:42 +0800 Subject: [PATCH 069/139] Update build.yml --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 183e5a79..fe8aa55e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,8 +17,6 @@ jobs: run: docker login -u germey -p ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - name: Build the Docker Image run: docker-compose build - - name: Push the Docker Image - run: docker-compose push - name: Tag and Push Master Version run: | docker tag germey/proxypool germey/proxypool:master From d2320ce04e47c2be05c83747d4ac84e2bd14c4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Tue, 14 Jul 2020 11:09:20 +0800 Subject: [PATCH 070/139] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee7c108b..266fff58 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ## 运行示例 -API Server 可以见[部署样例](https://universal.proxypool.cuiqingcai.com/)，随机代理[取用地址](https://universal.proxypool.cuiqingcai.com/random)，代理源比较少，仅供演示。 +API Server 可以见[部署样例](https://proxypool.scrape.center/)，随机代理[取用地址](https://proxypool.scrape.center/random)，代理源比较少，仅供演示。 本样例为 GitHub Actions + Kubernetes 自动部署 master 分支代码结果。 From 16c817a380d7c317ef56b89cafa03480a833ab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sun, 1 Nov 2020 00:20:24 +0800 Subject: [PATCH 071/139] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 266fff58..74ad0307 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,13 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 +如果下载速度特别慢，可以自行修改 Dockerfile，修改改行： + +``` +- RUN pip install -r requirements.txt ++ RUN pip install -r requirements.txt -i https://pypi.douban.com/simple +``` + ## 常规方式运行 如果不使用 Docker 运行，配置好 Python、Redis 环境之后也可运行，步骤如下。 From 891a19dd083c6bec177d8a6c040df7b4f2a3cb7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Sun, 1 Nov 2020 00:22:26 +0800 Subject: [PATCH 072/139] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74ad0307..5be50019 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,9 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 -如果下载速度特别慢，可以自行修改 Dockerfile，修改改行： +如果下载速度特别慢，可以自行修改 Dockerfile，修改： -``` +```diff - RUN pip install -r requirements.txt + RUN pip install -r requirements.txt -i https://pypi.douban.com/simple ``` From ede3fbbdbfdc86fec5c53396c21f533af4555832 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 02:16:48 +0800 Subject: [PATCH 073/139] update docker-compose --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f0181db2..98ba86df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,12 @@ version: '3' services: - redis: + redis4proxypool: image: redis:alpine - container_name: redis + container_name: redis4proxypool command: redis-server ports: - "6379:6379" - restart: always +# restart: always proxypool: build: . image: 'germey/proxypool' @@ -17,4 +17,4 @@ services: volumes: - /tmp/proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis \ No newline at end of file + REDIS_HOST: redis4proxypool \ No newline at end of file From 08054f7c1e90c3de5a02f34f5a9e77f7b4b10687 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 02:18:44 +0800 Subject: [PATCH 074/139] rm volumes --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 98ba86df..c39ded37 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: command: redis-server ports: - "6379:6379" -# restart: always + # restart: always proxypool: build: . image: 'germey/proxypool' @@ -14,7 +14,7 @@ services: ports: - "5555:5555" restart: always - volumes: - - /tmp/proxypool/crawlers/private:/app/proxypool/crawlers/private + # volumes: + # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: REDIS_HOST: redis4proxypool \ No newline at end of file From b730e9f88b022064aacf3233b4cfa3255ffb0c16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Feb 2021 01:53:07 +0800 Subject: [PATCH 075/139] Bump lxml from 4.3.3 to 4.6.2 (#105) Bumps [lxml](https://github.com/lxml/lxml) from 4.3.3 to 4.6.2. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.3.3...lxml-4.6.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cc0b6111..0e86c5a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ loguru==0.3.2 pyquery==1.4.0 supervisor==4.1.0 redis==2.10.6 -lxml==4.3.3 \ No newline at end of file +lxml==4.6.2 \ No newline at end of file From f91725475cdf9f6533bd19bc88ba79234d82a73a Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 5 Feb 2021 00:40:50 +0800 Subject: [PATCH 076/139] rm log --- examples/usage.py | 2 +- proxypool/setting.py | 4 ++-- supervisord.conf | 14 +++++++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/examples/usage.py b/examples/usage.py index 87c73b35..bc699ba9 100644 --- a/examples/usage.py +++ b/examples/usage.py @@ -2,7 +2,7 @@ proxypool_url = 'http://127.0.0.1:5555/random' -target_url = 'http://httpbin.org/get' +target_url = 'https://antispider5.scrape.center/' def get_random_proxy(): diff --git a/proxypool/setting.py b/proxypool/setting.py index 7c3008b3..5765714b 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -76,6 +76,6 @@ ENABLE_GETTER = env.bool('ENABLE_GETTER', True) ENABLE_SERVER = env.bool('ENABLE_SERVER', True) -logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') -logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') +# logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') +# logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') diff --git a/supervisord.conf b/supervisord.conf index c5828e0c..97f2f86f 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -5,13 +5,25 @@ nodaemon=true process_name=tester command=python3 run.py --processor tester directory=/app +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 [program:getter] process_name=getter command=python3 run.py --processor getter directory=/app +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 [program:server] process_name=server command=python3 run.py --processor server -directory=/app \ No newline at end of file +directory=/app +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 \ No newline at end of file From 9912b980bbe7b1354cac99664fa3fb95db218461 Mon Sep 17 00:00:00 2001 From: xionghaizicuncunzhang <57612793+xionghaizicuncunzhang@users.noreply.github.com> Date: Sun, 7 Feb 2021 02:54:05 +0800 Subject: [PATCH 077/139] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E6=BA=90=E7=88=AC=E5=8F=96=E2=80=94=E2=80=94?= =?UTF-8?q?=E5=A4=AA=E9=98=B3=E4=BB=A3=E7=90=86=20(#107)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add files via upload 新的代理爬虫 * Delete taiyangdaili.py * add taiyang 爬取太阳代理免费ip --- proxypool/crawlers/public/taiyangdaili.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 proxypool/crawlers/public/taiyangdaili.py diff --git a/proxypool/crawlers/public/taiyangdaili.py b/proxypool/crawlers/public/taiyangdaili.py new file mode 100644 index 00000000..7a48cb43 --- /dev/null +++ b/proxypool/crawlers/public/taiyangdaili.py @@ -0,0 +1,31 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from pyquery import PyQuery as pq + +BaseUrl = 'http://www.taiyanghttp.com/free/page{num}' +MAX_PAGE = 5 + + +class TaiyangdailiCrawler(BaseCrawler): + """ + taiyangdaili crawler, http://www.taiyanghttp.com/free/ + """ + urls = [BaseUrl.format(num=i) for i in range(1, 6)] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('#ip_list .tr.ip_tr').items() + for tr in trs: + host = tr.find('div:nth-child(1)').text() + port = tr.find('div:nth-child(2)').text() + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = TaiyangdailiCrawler() + for proxy in crawler.crawl(): + print(proxy) From e3bbd55ab73814c2e2ae4fda859ef147eab6277e Mon Sep 17 00:00:00 2001 From: "j.yao.SUSE" Date: Sun, 7 Feb 2021 02:54:29 +0800 Subject: [PATCH 078/139] =?UTF-8?q?=E6=B7=BB=E5=8A=A0IP=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E7=88=AC=E5=8F=96=20(#106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create ip89.py www.89ip.cn 免费代理 * Update ip89.py update Class name * Create fatezero_proxylist.py 增加 http://proxylist.fatezero.org/ 代理 * Create ihuan.py i幻 代理 --- .../crawlers/public/fatezero_proxylist.py | 32 +++++++++++++++++ proxypool/crawlers/public/ihuan.py | 34 +++++++++++++++++++ proxypool/crawlers/public/ip89.py | 33 ++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 proxypool/crawlers/public/fatezero_proxylist.py create mode 100644 proxypool/crawlers/public/ihuan.py create mode 100644 proxypool/crawlers/public/ip89.py diff --git a/proxypool/crawlers/public/fatezero_proxylist.py b/proxypool/crawlers/public/fatezero_proxylist.py new file mode 100644 index 00000000..8a7d6e27 --- /dev/null +++ b/proxypool/crawlers/public/fatezero_proxylist.py @@ -0,0 +1,32 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import re +import json +BASE_URL = 'http://proxylist.fatezero.org/proxy.list' + + +class FatezeroCrawler(BaseCrawler): + """ + Fatezero crawler,http://proxylist.fatezero.org + """ + urls = [BASE_URL] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + + hosts_ports = html.split('\n') + for addr in hosts_ports: + ip_address = json.loads(addr) + if(True): + host = ip_address['host'] + port = ip_address['port'] + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = FatezeroCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/ihuan.py b/proxypool/crawlers/public/ihuan.py new file mode 100644 index 00000000..7386b705 --- /dev/null +++ b/proxypool/crawlers/public/ihuan.py @@ -0,0 +1,34 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import re +from pyquery import PyQuery as pq +import time +BASE_URL = 'https://ip.ihuan.me/today/{path}.html' + + +class IhuanCrawler(BaseCrawler): + """ + ip ihuan crawler, https://ip.ihuan.me + """ + urls = [BASE_URL.format(path=time.strftime("%Y/%m/%d/%H", time.localtime()))] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + # doc = pq(html)('.text-left') + ip_address = re.compile('([\d:\.]*).*?
    ') + hosts_ports = ip_address.findall(html) + for addr in hosts_ports: + addr_split = addr.split(':') + if(len(addr_split) == 2): + host = addr_split[0] + port = addr_split[1] + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = IhuanCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/ip89.py b/proxypool/crawlers/public/ip89.py new file mode 100644 index 00000000..f67c3870 --- /dev/null +++ b/proxypool/crawlers/public/ip89.py @@ -0,0 +1,33 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import re + +MAX_NUM = 9999 +BASE_URL = 'http://api.89ip.cn/tqdl.html?api=1&num={MAX_NUM}&port=&address=&isp='.format(MAX_NUM=MAX_NUM) + + +class Ip89Crawler(BaseCrawler): + """ + 89ip crawler, http://api.89ip.cn + """ + urls = [BASE_URL] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + ip_address = re.compile('([\d:\.]*)
    ') + hosts_ports = ip_address.findall(html) + for addr in hosts_ports: + addr_split = addr.split(':') + if(len(addr_split) == 2): + host = addr_split[0] + port = addr_split[1] + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = Ip89Crawler() + for proxy in crawler.crawl(): + print(proxy) From 63ca6da268073f6254cbb3dfeb32ec88226ec538 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 14 Feb 2021 01:29:00 +0800 Subject: [PATCH 079/139] update ignore --- .dockerignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 803baf5e..f9306766 100644 --- a/.dockerignore +++ b/.dockerignore @@ -128,4 +128,6 @@ venv.bak/ dmypy.json # Pyre type checker -.pyre/ \ No newline at end of file +.pyre/ + +proxypool/.env \ No newline at end of file From cf03d87bfaec0a932d135125182f98b317af757f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 02:30:39 +0800 Subject: [PATCH 080/139] Bump aiohttp from 3.6.2 to 3.7.4 (#112) Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.6.2 to 3.7.4. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.6.2...v3.7.4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0e86c5a7..74c8426c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ environs==7.2.0 Flask==1.0.3 attrs==19.1.0 retrying==1.3.3 -aiohttp==3.6.2 +aiohttp==3.7.4 requests==2.22.0 loguru==0.3.2 pyquery==1.4.0 From 4878bf5ae35fd767f2bd3c83f1333850cf64e22f Mon Sep 17 00:00:00 2001 From: "j.yao.SUSE" Date: Mon, 8 Mar 2021 02:32:41 +0800 Subject: [PATCH 081/139] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8B=A5=E5=B9=B2?= =?UTF-8?q?=E4=BB=A3=E7=90=86=EF=BC=8C=20=E4=BC=98=E5=8C=96=E9=83=A8?= =?UTF-8?q?=E5=88=86=E4=BB=A3=E7=A0=81=20(#108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create ip89.py www.89ip.cn 免费代理 * Update ip89.py update Class name * Create fatezero_proxylist.py 增加 http://proxylist.fatezero.org/ 代理 * Create ihuan.py i幻 代理 * update example usage2 * update requirements.txt * 优化 public crawlers * add proxy jiangxianli * tester 增加单个proxy测试方法 * reset setting Dockerfile docker-compose to default Co-authored-by: jy Co-authored-by: 崔庆才丨静觅 <1016903103@qq.com> --- Dockerfile | 3 +- docker-compose.yml | 2 +- examples/usage2.py | 95 +++++++++++++++++++ proxypool/crawlers/base.py | 9 +- proxypool/crawlers/public/daili66.py | 2 +- .../crawlers/public/fatezero_proxylist.py | 5 +- proxypool/crawlers/public/goubanjia.py | 44 +++++++++ proxypool/crawlers/public/ihuan.py | 5 +- proxypool/crawlers/public/ip3366.py | 6 +- proxypool/crawlers/public/jiangxianli.py | 35 +++++++ proxypool/crawlers/public/kuaidaili.py | 6 +- proxypool/crawlers/public/zhandaye.py | 2 +- proxypool/processors/server.py | 15 +++ proxypool/processors/tester.py | 7 ++ proxypool/storages/redis.py | 4 +- requirements.txt | 20 ++-- 16 files changed, 231 insertions(+), 29 deletions(-) create mode 100644 examples/usage2.py create mode 100644 proxypool/crawlers/public/goubanjia.py create mode 100644 proxypool/crawlers/public/jiangxianli.py diff --git a/Dockerfile b/Dockerfile index dab1227e..4b8f9c0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.6 WORKDIR /app COPY . . -RUN pip install -r requirements.txt +# RUN pip install -r requirements.txt -i https://pypi.douban.com/simple +RUN pip install -r requirements.txt -i VOLUME ["/app/proxypool/crawlers/private"] CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/docker-compose.yml b/docker-compose.yml index c39ded37..03d85de7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: command: redis-server ports: - "6379:6379" - # restart: always + # restart: always proxypool: build: . image: 'germey/proxypool' diff --git a/examples/usage2.py b/examples/usage2.py new file mode 100644 index 00000000..918c5eb2 --- /dev/null +++ b/examples/usage2.py @@ -0,0 +1,95 @@ +# -*- coding: UTF-8 -*- + +''' +''' +import requests +import time +import threading +import urllib3 +from fake_headers import Headers +import uuid +from geolite2 import geolite2 +ips = [] + +# 爬数据的线程类 + +def getChinaIP(ip='127.0.0.1'): + reader = geolite2.reader() + ip_info = reader.get(ip) + geolite2.close() + print(ip_info) + return True if ip_info['country']['iso_code'] == 'CN' else False + + + +class CrawlThread(threading.Thread): + def __init__(self, proxyip): + super(CrawlThread, self).__init__() + self.proxyip = proxyip + + def run(self): + # 开始计时 + pure_ip_address = self.proxyip.split(':')[0] + # 验证IP归属 + if not getChinaIP(pure_ip_address): + # pass + raise ValueError('不是有效IP') + # + start = time.time() + # 消除关闭证书验证的警告 + urllib3.disable_warnings() + headers = Headers(headers=True).generate() + headers['Referer'] = 'http://bb.cf08tp.cn/Home/index.php?m=Index&a=index&id=2676' + headers['Pragma'] = 'no-cache' + headers['Host'] = 'bb.cf08tp.cn' + headers['x-forward-for'] = pure_ip_address + headers['Cookie'] = 'PHPSESSID={}'.format( + ''.join(str(uuid.uuid1()).split('-'))) + print(headers) + html = requests.get(headers=headers, url=targetUrl, proxies={ + "http": 'http://' + self.proxyip, "https": 'https://' + self.proxyip}, verify=False, timeout=2).content.decode() + # 结束计时 + end = time.time() + # 输出内容 + print(threading.current_thread().getName() + "使用代理IP, 耗时 " + str(end - start) + + "毫秒 " + self.proxyip + " 获取到如下HTML内容：\n" + html + "\n*************") + +# 获取代理IP的线程类 + + +class GetIpThread(threading.Thread): + def __init__(self, fetchSecond): + super(GetIpThread, self).__init__() + self.fetchSecond = fetchSecond + + def run(self): + global ips + while True: + # 获取IP列表 + res = requests.get(apiUrl).content.decode() + # 按照\n分割获取到的IP + ips = res.split('\n') + # 利用每一个IP + for proxyip in ips: + if proxyip.strip(): + # 开启一个线程 + # CrawlThread(proxyip).start() + try: + CrawlThread(proxyip).run() + time.sleep(1.5) + except Exception as e: + print(e) + # 休眠 + time.sleep(len(ips) /self.fetchSecond ) + + +if __name__ == '__main__': + # 获取IP的API接口 + # apiUrl = "http://127.0.0.1:5555/all" + apiUrl = "http://127.0.0.1:5555/random" + # 要抓取的目标网站地址 + targetUrl = "http://bb.cf08tp.cn/Home/index.php?m=Index&a=vote&vid=335688&id=2676&tp=" + # targetUrl = 'http://bb.cf08tp.cn/Home/index.php?m=Index&a=vote&vid=335608&id=2676&tp=' + fetchSecond = 5 + # 开始自动获取IP + GetIpThread(fetchSecond).start() diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index aa35430e..563d49bb 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -2,17 +2,19 @@ import requests from loguru import logger from proxypool.setting import GET_TIMEOUT - - +from fake_headers import Headers +import time class BaseCrawler(object): urls = [] @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000) def fetch(self, url, **kwargs): try: + headers = Headers(headers=True).generate() kwargs.setdefault('timeout', GET_TIMEOUT) kwargs.setdefault('verify', False) - response = requests.get(url, **kwargs) + kwargs.setdefault('headers', headers) + response = requests.get(url ,**kwargs) if response.status_code == 200: response.encoding = 'utf-8' return response.text @@ -27,6 +29,7 @@ def crawl(self): for url in self.urls: logger.info(f'fetching {url}') html = self.fetch(url) + time.sleep(.5) for proxy in self.parse(html): logger.info(f'fetched proxy {proxy.string()} from {url}') yield proxy diff --git a/proxypool/crawlers/public/daili66.py b/proxypool/crawlers/public/daili66.py index 09a3ee45..7b3bf7c2 100644 --- a/proxypool/crawlers/public/daili66.py +++ b/proxypool/crawlers/public/daili66.py @@ -4,7 +4,7 @@ BASE_URL = 'http://www.66ip.cn/{page}.html' -MAX_PAGE = 5 +MAX_PAGE = 50 class Daili66Crawler(BaseCrawler): diff --git a/proxypool/crawlers/public/fatezero_proxylist.py b/proxypool/crawlers/public/fatezero_proxylist.py index 8a7d6e27..681cf9e4 100644 --- a/proxypool/crawlers/public/fatezero_proxylist.py +++ b/proxypool/crawlers/public/fatezero_proxylist.py @@ -19,13 +19,12 @@ def parse(self, html): hosts_ports = html.split('\n') for addr in hosts_ports: - ip_address = json.loads(addr) - if(True): + if(addr): + ip_address = json.loads(addr) host = ip_address['host'] port = ip_address['port'] yield Proxy(host=host, port=port) - if __name__ == '__main__': crawler = FatezeroCrawler() for proxy in crawler.crawl(): diff --git a/proxypool/crawlers/public/goubanjia.py b/proxypool/crawlers/public/goubanjia.py new file mode 100644 index 00000000..57157858 --- /dev/null +++ b/proxypool/crawlers/public/goubanjia.py @@ -0,0 +1,44 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import re +from pyquery import PyQuery as pq +import time +BASE_URL = 'http://www.goubanjia.com/' + + +class GoubanjiaCrawler(BaseCrawler): + """ + ip Goubanjia crawler, http://www.goubanjia.com/ + """ + urls = [BASE_URL] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html)('.ip').items() + # ''.join([*filter(lambda x: x != '',re.compile('\>([\d:\.]*)\<').findall(td.html()))]) + for td in doc: + trs = td.children() + ip_str = '' + for tr in trs: + attrib = tr.attrib + if 'style' in attrib and 'none' in tr.attrib['style']: + continue + ip_str+= '' if not tr.text else tr.text + addr_split = ip_str.split(':') + if(len(addr_split) == 2): + host = addr_split[0] + port = addr_split[1] + yield Proxy(host=host, port=port) + else: + port = trs[-1].text + host = ip_str.replace(port,'') + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = GoubanjiaCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/ihuan.py b/proxypool/crawlers/public/ihuan.py index 7386b705..ccf90a13 100644 --- a/proxypool/crawlers/public/ihuan.py +++ b/proxypool/crawlers/public/ihuan.py @@ -10,8 +10,9 @@ class IhuanCrawler(BaseCrawler): """ ip ihuan crawler, https://ip.ihuan.me """ - urls = [BASE_URL.format(path=time.strftime("%Y/%m/%d/%H", time.localtime()))] - + path = time.strftime("%Y/%m/%d/%H", time.localtime()) + urls = [BASE_URL.format(path=path)] + ignore = False def parse(self, html): """ parse html file to get proxies diff --git a/proxypool/crawlers/public/ip3366.py b/proxypool/crawlers/public/ip3366.py index 78d29447..474a4f77 100644 --- a/proxypool/crawlers/public/ip3366.py +++ b/proxypool/crawlers/public/ip3366.py @@ -3,15 +3,15 @@ import re -MAX_PAGE = 5 -BASE_URL = 'http://www.ip3366.net/free/?stype=1&page={page}' +MAX_PAGE = 8 +BASE_URL = 'http://www.ip3366.net/free/?stype={stype}&page={page}' class IP3366Crawler(BaseCrawler): """ ip3366 crawler, http://www.ip3366.net/ """ - urls = [BASE_URL.format(page=i) for i in range(1, 8)] + urls = [BASE_URL.format(stype=stype,page=i) for stype in range(1,3) for i in range(1, 8)] def parse(self, html): """ diff --git a/proxypool/crawlers/public/jiangxianli.py b/proxypool/crawlers/public/jiangxianli.py new file mode 100644 index 00000000..14fc46cc --- /dev/null +++ b/proxypool/crawlers/public/jiangxianli.py @@ -0,0 +1,35 @@ +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import re +import json +BASE_URL = 'https://ip.jiangxianli.com/api/proxy_ips?page={page}' + +MAX_PAGE = 10 +class JiangxianliCrawler(BaseCrawler): + """ + jiangxianli crawler,https://ip.jiangxianli.com/ + """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + + result =json.loads(html) + if result['code'] != 0: + return + MAX_PAGE = int(result['data']['last_page']) + hosts_ports = result['data']['data'] + for ip_address in hosts_ports: + if(ip_address): + host = ip_address['ip'] + port = ip_address['port'] + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = JiangxianliCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py index f3fa6437..71ab1717 100644 --- a/proxypool/crawlers/public/kuaidaili.py +++ b/proxypool/crawlers/public/kuaidaili.py @@ -4,15 +4,15 @@ from pyquery import PyQuery as pq -BASE_URL = 'https://www.kuaidaili.com/free/inha/{page}/' -MAX_PAGE = 5 +BASE_URL = 'https://www.kuaidaili.com/free/{type}/{page}/' +MAX_PAGE = 300 class KuaidailiCrawler(BaseCrawler): """ kuaidaili crawler, https://www.kuaidaili.com/ """ - urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] + urls = [BASE_URL.format(type=type,page=page) for type in ('intr','inha') for page in range(1, MAX_PAGE + 1)] def parse(self, html): """ diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py index b6278a28..83af04b6 100755 --- a/proxypool/crawlers/public/zhandaye.py +++ b/proxypool/crawlers/public/zhandaye.py @@ -6,7 +6,7 @@ BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html' -MAX_PAGE = 5 +MAX_PAGE = 5 * 2 class ZhandayeCrawler(BaseCrawler): """ diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index e87f82f5..d3edd70d 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -37,6 +37,21 @@ def get_proxy(): return conn.random().string() +@app.route('/all') +def get_proxy_all(): + """ + get a random proxy + :return: get a random proxy + """ + conn = get_conn() + proxies = conn.all() + proxies_string = '' + for proxy in proxies: + proxies_string += str(proxy) + '\n' + + return proxies_string + + @app.route('/count') def get_count(): """ diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index e0812110..f002056a 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -84,7 +84,14 @@ def run(self): if not cursor: break +def run_tester(): + host = '96.113.165.182' + port = '3128' + tasks = [tester.test(Proxy(host=host, port=port))] + tester.loop.run_until_complete(asyncio.wait(tasks)) if __name__ == '__main__': tester = Tester() tester.run() + # run_tester() + diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 60a03e9f..0ebbccc2 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -51,11 +51,11 @@ def random(self) -> Proxy: :return: proxy, like 8.8.8.8:8 """ # try to get proxy with max score - proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank - proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error diff --git a/requirements.txt b/requirements.txt index 74c8426c..3bccfa78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ -environs==7.2.0 -Flask==1.0.3 -attrs==19.1.0 +environs==9.3.0 +Flask==1.1.2 +attrs==20.3.0 retrying==1.3.3 aiohttp==3.7.4 -requests==2.22.0 -loguru==0.3.2 -pyquery==1.4.0 -supervisor==4.1.0 -redis==2.10.6 -lxml==4.6.2 \ No newline at end of file +requests==2.25.1 +loguru==0.5.3 +pyquery==1.4.3 +supervisor==4.2.1 +redis==3.5.3 +lxml==4.6.2 +fake_headers==1.0.2 +maxminddb_geolite2==2018.703 From 84d738b8e0134bb7772ef05750b18147a0a56a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= <1016903103@qq.com> Date: Mon, 8 Mar 2021 02:34:16 +0800 Subject: [PATCH 082/139] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b8f9c0e..280f8b07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,6 @@ FROM python:3.6 WORKDIR /app COPY . . # RUN pip install -r requirements.txt -i https://pypi.douban.com/simple -RUN pip install -r requirements.txt -i +RUN pip install -r requirements.txt VOLUME ["/app/proxypool/crawlers/private"] CMD ["supervisord", "-c", "supervisord.conf"] From 216c58f85b210d83edd8237d3c74fd7f0d7d353c Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 28 Mar 2021 19:46:26 +0800 Subject: [PATCH 083/139] change port --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 03d85de7..5716aca6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: redis4proxypool command: redis-server ports: - - "6379:6379" + - "6374:6379" # restart: always proxypool: build: . From 3884b660d21ed003863f757c9a95c51323c5dd9d Mon Sep 17 00:00:00 2001 From: K8sCat Date: Thu, 1 Apr 2021 00:57:00 +0800 Subject: [PATCH 084/139] :fire: optimize: remove redundant command (#114) --- docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5716aca6..c069ec56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,14 @@ -version: '3' +version: "3" services: redis4proxypool: image: redis:alpine container_name: redis4proxypool - command: redis-server ports: - "6374:6379" # restart: always proxypool: build: . - image: 'germey/proxypool' + image: "germey/proxypool" container_name: proxypool ports: - "5555:5555" @@ -17,4 +16,4 @@ services: # volumes: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis4proxypool \ No newline at end of file + REDIS_HOST: redis4proxypool From 0e9935d9b3c17bd0b902b88642ba7c8c89afea3a Mon Sep 17 00:00:00 2001 From: K8sCat Date: Tue, 13 Apr 2021 23:45:34 +0800 Subject: [PATCH 085/139] optimize: use python:3.6-alpine (#116) * :zap: optimize: use python:3.6-alpine * :rocket: update: del build pkgs --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 280f8b07..c38bf89b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ -FROM python:3.6 +FROM python:3.6-alpine WORKDIR /app COPY . . # RUN pip install -r requirements.txt -i https://pypi.douban.com/simple -RUN pip install -r requirements.txt +RUN apk add --no-cache libxml2-dev libxslt-dev gcc musl-dev && \ +pip install -r requirements.txt && \ +apk del gcc musl-dev libxml2-dev VOLUME ["/app/proxypool/crawlers/private"] CMD ["supervisord", "-c", "supervisord.conf"] From ab917a8f0a6d65ce771501539047c776851a0e67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Apr 2021 23:46:45 +0800 Subject: [PATCH 086/139] Bump lxml from 4.6.2 to 4.6.3 (#115) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.2...lxml-4.6.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3bccfa78..afa85606 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ loguru==0.5.3 pyquery==1.4.3 supervisor==4.2.1 redis==3.5.3 -lxml==4.6.2 +lxml==4.6.3 fake_headers==1.0.2 maxminddb_geolite2==2018.703 From ad5d25406790548dee96459945ebdcb97654e4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Tue, 1 Jun 2021 02:32:20 +0800 Subject: [PATCH 087/139] Update README.md --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5be50019..fafb8d7a 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,20 @@ API Server 可以见[部署样例](https://proxypool.scrape.center/)，随机代 本样例为 GitHub Actions + Kubernetes 自动部署 master 分支代码结果。 +## 使用准备 + +首先当然是克隆代码并进入 ProxyPool 文件夹： + +``` +git clone https://github.com/Python3WebSpider/ProxyPool.git +cd ProxyPool +``` + +然后选用下面 Docker 和常规方式任意一个执行即可。 + ## 使用要求 -可以通过两种方式来运行代理池，一种方式是使用 Docker（推荐），另一种方式是常规方式运行。 +可以通过两种方式来运行代理池，一种方式是使用 Docker（推荐），另一种方式是常规方式运行，要求如下： ### Docker @@ -31,6 +42,8 @@ API Server 可以见[部署样例](https://proxypool.scrape.center/)，随机代 * Docker * Docker-Compose +安装方法自行搜索即可。 + ### 常规方式 常规方式要求有 Python 环境、Redis 环境，具体要求如下： From 1de4f95d3ac8b7fc1657f8e4fd127b263709be0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Tue, 1 Jun 2021 02:32:56 +0800 Subject: [PATCH 088/139] Update README.md --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index fafb8d7a..441f7c4e 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,6 @@ 代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」，建议使用之前阅读。 -## 运行示例 - -API Server 可以见[部署样例](https://proxypool.scrape.center/)，随机代理[取用地址](https://proxypool.scrape.center/random)，代理源比较少，仅供演示。 - -本样例为 GitHub Actions + Kubernetes 自动部署 master 分支代码结果。 - ## 使用准备 首先当然是克隆代码并进入 ProxyPool 文件夹： From dac05be5db7f6c81db5461259ac4fb37e3743616 Mon Sep 17 00:00:00 2001 From: wc571498244 <51046547+wc571498244@users.noreply.github.com> Date: Tue, 12 Oct 2021 21:46:20 +0800 Subject: [PATCH 089/139] =?UTF-8?q?=E6=96=B0=E5=A2=9E=EF=BC=9A=E5=B0=8F?= =?UTF-8?q?=E8=88=92=E4=BB=A3=E7=90=86=E3=80=81seo=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E3=80=81yqie=E4=BB=A3=E7=90=86=20(#124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/crawlers/public/seofangfa.py | 34 ++++++++++++++++ proxypool/crawlers/public/xiaoshudaili.py | 49 +++++++++++++++++++++++ proxypool/crawlers/public/yqie.py | 32 +++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 proxypool/crawlers/public/seofangfa.py create mode 100644 proxypool/crawlers/public/xiaoshudaili.py create mode 100644 proxypool/crawlers/public/yqie.py diff --git a/proxypool/crawlers/public/seofangfa.py b/proxypool/crawlers/public/seofangfa.py new file mode 100644 index 00000000..1f5a20a2 --- /dev/null +++ b/proxypool/crawlers/public/seofangfa.py @@ -0,0 +1,34 @@ +import requests +from pyquery import PyQuery as pq + +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler + +requests.packages.urllib3.disable_warnings() +BASE_URL = "https://proxy.seofangfa.com/" +MAX_PAGE = 1 + + +class SeoFangFaCrawler(BaseCrawler): + """ + seo方法 crawler, https://proxy.seofangfa.com/ + """ + urls = ["https://proxy.seofangfa.com/"] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('.table tr:gt(0)').items() + for tr in trs: + host = tr.find('td:nth-child(1)').text() + port = int(tr.find('td:nth-child(2)').text()) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = SeoFangFaCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/xiaoshudaili.py b/proxypool/crawlers/public/xiaoshudaili.py new file mode 100644 index 00000000..bb4d7d42 --- /dev/null +++ b/proxypool/crawlers/public/xiaoshudaili.py @@ -0,0 +1,49 @@ +import re + +from pyquery import PyQuery as pq + +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler + +BASE_URL = "http://www.xsdaili.cn/" +PAGE_BASE_URL = "http://www.xsdaili.cn/dayProxy/ip/{page}.html" +MAX_PAGE = 50 + + +class XiaoShuCrawler(BaseCrawler): + """ + 小舒代理 crawler, http://www.xsdaili.cn/ + """ + + def __init__(self): + html = self.fetch(url=BASE_URL) + doc = pq(html) + title = doc(".title:eq(0) a").items() + + latest_page = 0 + for t in title: + res = re.search(r"/(\d+)\.html", t.attr("href")) + latest_page = int(res.group(1)) if res else 0 + if latest_page: + self.urls = [PAGE_BASE_URL.format(page=page) for page in range(latest_page - MAX_PAGE, latest_page)] + else: + self.urls = [] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + contents = doc('.cont').text() + contents = contents.split("\n") + for content in contents: + c = content[:content.find("@")] + host, port = c.split(":") + yield Proxy(host=host, port=int(port)) + + +if __name__ == '__main__': + crawler = XiaoShuCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/yqie.py b/proxypool/crawlers/public/yqie.py new file mode 100644 index 00000000..fb3feaf8 --- /dev/null +++ b/proxypool/crawlers/public/yqie.py @@ -0,0 +1,32 @@ +from pyquery import PyQuery as pq + +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler + +BASE_URL = "http://ip.yqie.com/ipproxy.htm" +MAX_PAGE = 1 + + +class YqIeCrawler(BaseCrawler): + """ + ip yqie crawler, http://ip.yqie.com/ipproxy.htm + """ + urls = [BASE_URL] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('#GridViewOrder tr:gt(0)').items() + for tr in trs: + host = tr.find('td:nth-child(1)').text() + port = int(tr.find('td:nth-child(2)').text()) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = YqIeCrawler() + for proxy in crawler.crawl(): + print(proxy) From 47cb3bcbac80afafc4fd7d15e94216830c4e3599 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Dec 2021 01:36:35 +0800 Subject: [PATCH 090/139] Bump lxml from 4.6.3 to 4.6.5 (#131) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.3 to 4.6.5. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.3...lxml-4.6.5) --- updated-dependencies: - dependency-name: lxml dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index afa85606..aa5eddcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ loguru==0.5.3 pyquery==1.4.3 supervisor==4.2.1 redis==3.5.3 -lxml==4.6.3 +lxml==4.6.5 fake_headers==1.0.2 maxminddb_geolite2==2018.703 From e552819a4ac168ad3055fcb70fdcb2dd57dbcbca Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 02:45:47 +0800 Subject: [PATCH 091/139] update docker-compose file --- .github/workflows/build.yml | 2 +- .github/workflows/deploy.yml | 2 +- README.md | 94 +++++++++++++++++++----------------- build.yml | 19 ++++++++ docker-compose.yml | 6 +-- 5 files changed, 73 insertions(+), 50 deletions(-) create mode 100644 build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe8aa55e..d1724a5a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: - name: Docker Login run: docker login -u germey -p ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - name: Build the Docker Image - run: docker-compose build + run: docker-compose -f build.yml build - name: Tag and Push Master Version run: | docker tag germey/proxypool germey/proxypool:master diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8257ba0a..ac5a5c69 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,7 @@ jobs: echo $BUILD_NUMBER - name: Build Push Deploy run: | - docker-compose build + docker-compose -f build.yml build docker tag germey/proxypool germey/proxypool:$BUILD_NUMBER docker push germey/proxypool docker push germey/proxypool:$BUILD_NUMBER diff --git a/README.md b/README.md index 441f7c4e..3db7a2ea 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ 简易高效的代理池，提供如下功能： -* 定时抓取免费代理网站，简易可扩展。 -* 使用 Redis 对代理进行存储并对代理可用性进行排序。 -* 定时测试和筛选，剔除不可用代理，留下可用代理。 -* 提供代理 API，随机取用测试通过的可用代理。 +- 定时抓取免费代理网站，简易可扩展。 +- 使用 Redis 对代理进行存储并对代理可用性进行排序。 +- 定时测试和筛选，剔除不可用代理，留下可用代理。 +- 提供代理 API，随机取用测试通过的可用代理。 代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」，建议使用之前阅读。 @@ -33,8 +33,8 @@ cd ProxyPool 如果使用 Docker，则需要安装如下环境： -* Docker -* Docker-Compose +- Docker +- Docker-Compose 安装方法自行搜索即可。 @@ -42,8 +42,8 @@ cd ProxyPool 常规方式要求有 Python 环境、Redis 环境，具体要求如下： -* Python>=3.6 -* Redis +- Python>=3.6 +- Redis ## Docker 运行 @@ -72,6 +72,12 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 +当然你也可以选择自己 Build，直接运行如下命令即可： + +``` +docker-compose -f build.yml up +``` + 如果下载速度特别慢，可以自行修改 Dockerfile，修改： ```diff @@ -118,7 +124,7 @@ export REDIS_CONNECTION_STRING='redis://@host:port/db' ### 安装依赖包 -这里强烈推荐使用 [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands) +这里强烈推荐使用 [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands) 或 [virtualenv](https://virtualenv.pypa.io/en/latest/user_guide.html) 创建虚拟环境，Python 版本不低于 3.6。 然后 pip 安装依赖即可： @@ -198,15 +204,15 @@ if __name__ == '__main__': ``` get random proxy 116.196.115.209:8080 { - "args": {}, + "args": {}, "headers": { - "Accept": "*/*", - "Accept-Encoding": "gzip, deflate", - "Host": "httpbin.org", - "User-Agent": "python-requests/2.22.0", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Host": "httpbin.org", + "User-Agent": "python-requests/2.22.0", "X-Amzn-Trace-Id": "Root=1-5e4d7140-662d9053c0a2e513c7278364" - }, - "origin": "116.196.115.209", + }, + "origin": "116.196.115.209", "url": "https://httpbin.org/get" } ``` @@ -219,41 +225,41 @@ get random proxy 116.196.115.209:8080 ### 开关 -* ENABLE_TESTER：允许 Tester 启动，默认 true -* ENABLE_GETTER：允许 Getter 启动，默认 true -* ENABLE_SERVER：运行 Server 启动，默认 true +- ENABLE_TESTER：允许 Tester 启动，默认 true +- ENABLE_GETTER：允许 Getter 启动，默认 true +- ENABLE_SERVER：运行 Server 启动，默认 true ### 环境 -* APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev -* APP_DEBUG：调试模式，可以设置 true 或 false，默认 true +- APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev +- APP_DEBUG：调试模式，可以设置 true 或 false，默认 true ### Redis 连接 -* REDIS_HOST：Redis 的 Host -* REDIS_PORT：Redis 的端口 -* REDIS_PASSWORD：Redis 的密码 -* REDIS_DB：Redis 的数据库索引，如 0、1 -* REDIS_CONNECTION_STRING：Redis 连接字符串 -* REDIS_KEY：Redis 储存代理使用字典的名称 +- REDIS_HOST：Redis 的 Host +- REDIS_PORT：Redis 的端口 +- REDIS_PASSWORD：Redis 的密码 +- REDIS_DB：Redis 的数据库索引，如 0、1 +- REDIS_CONNECTION_STRING：Redis 连接字符串 +- REDIS_KEY：Redis 储存代理使用字典的名称 ### 处理器 -* CYCLE_TESTER：Tester 运行周期，即间隔多久运行一次测试，默认 20 秒 -* CYCLE_GETTER：Getter 运行周期，即间隔多久运行一次代理获取，默认 100 秒 -* TEST_URL：测试 URL，默认百度 -* TEST_TIMEOUT：测试超时时间，默认 10 秒 -* TEST_BATCH：批量测试数量，默认 20 个代理 -* TEST_VALID_STATUS：测试有效的状态吗 -* API_HOST：代理 Server 运行 Host，默认 0.0.0.0 -* API_PORT：代理 Server 运行端口，默认 5555 -* API_THREADED：代理 Server 是否使用多线程，默认 true +- CYCLE_TESTER：Tester 运行周期，即间隔多久运行一次测试，默认 20 秒 +- CYCLE_GETTER：Getter 运行周期，即间隔多久运行一次代理获取，默认 100 秒 +- TEST_URL：测试 URL，默认百度 +- TEST_TIMEOUT：测试超时时间，默认 10 秒 +- TEST_BATCH：批量测试数量，默认 20 个代理 +- TEST_VALID_STATUS：测试有效的状态吗 +- API_HOST：代理 Server 运行 Host，默认 0.0.0.0 +- API_PORT：代理 Server 运行端口，默认 5555 +- API_THREADED：代理 Server 是否使用多线程，默认 true ### 日志 -* LOG_DIR：日志相对路径 -* LOG_RUNTIME_FILE：运行日志文件名称 -* LOG_ERROR_FILE：错误日志文件名称 +- LOG_DIR：日志相对路径 +- LOG_RUNTIME_FILE：运行日志文件名称 +- LOG_ERROR_FILE：错误日志文件名称 以上内容均可使用环境变量配置，即在运行前设置对应环境变量值即可，如更改测试地址和 Redis 键名： @@ -267,7 +273,7 @@ export REDIS_KEY=proxies:weibo 如果使用 Docker-Compose 启动代理池，则需要在 docker-compose.yml 文件里面指定环境变量，如： ```yaml -version: '3' +version: "3" services: redis: image: redis:alpine @@ -278,7 +284,7 @@ services: restart: always proxypool: build: . - image: 'germey/proxypool' + image: "germey/proxypool" container_name: proxypool ports: - "5555:5555" @@ -310,7 +316,7 @@ class Daili66Crawler(BaseCrawler): daili66 crawler, http://www.66ip.cn/1.html """ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] - + def parse(self, html): """ parse html file to get proxies @@ -326,8 +332,8 @@ class Daili66Crawler(BaseCrawler): 在这里只需要定义一个 Crawler 继承 BaseCrawler 即可，然后定义好 urls 变量和 parse 方法即可。 -* urls 变量即为爬取的代理网站网址列表，可以用程序定义也可写成固定内容。 -* parse 方法接收一个参数即 html，代理网址的 html，在 parse 方法里只需要写好 html 的解析，解析出 host 和 port，并构建 Proxy 对象 yield 返回即可。 +- urls 变量即为爬取的代理网站网址列表，可以用程序定义也可写成固定内容。 +- parse 方法接收一个参数即 html，代理网址的 html，在 parse 方法里只需要写好 html 的解析，解析出 host 和 port，并构建 Proxy 对象 yield 返回即可。 网页的爬取不需要实现，BaseCrawler 已经有了默认实现，如需更改爬取方式，重写 crawl 方法即可。 diff --git a/build.yml b/build.yml new file mode 100644 index 00000000..c069ec56 --- /dev/null +++ b/build.yml @@ -0,0 +1,19 @@ +version: "3" +services: + redis4proxypool: + image: redis:alpine + container_name: redis4proxypool + ports: + - "6374:6379" + # restart: always + proxypool: + build: . + image: "germey/proxypool" + container_name: proxypool + ports: + - "5555:5555" + restart: always + # volumes: + # - proxypool/crawlers/private:/app/proxypool/crawlers/private + environment: + REDIS_HOST: redis4proxypool diff --git a/docker-compose.yml b/docker-compose.yml index c069ec56..8ab8228f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,15 +5,13 @@ services: container_name: redis4proxypool ports: - "6374:6379" - # restart: always proxypool: - build: . image: "germey/proxypool" container_name: proxypool ports: - "5555:5555" restart: always - # volumes: - # - proxypool/crawlers/private:/app/proxypool/crawlers/private + # volumes: + # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: REDIS_HOST: redis4proxypool From 30d5b0c4be4d4ed12b0c03e28203944d1c5bc607 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 02:47:34 +0800 Subject: [PATCH 092/139] update to master --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8ab8228f..3c57f280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: ports: - "6374:6379" proxypool: - image: "germey/proxypool" + image: "germey/proxypool:master" container_name: proxypool ports: - "5555:5555" From dcf93a3cc1d171e4bed759407468ece2de90f115 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 14:49:50 +0800 Subject: [PATCH 093/139] add kubernetes --- Dockerfile | 12 ++- README.md | 6 +- build.yml => build.yaml | 2 +- deployment.yml | 99 ------------------- ingress.yml | 32 ------ kubernetes/.helmignore | 24 +++++ kubernetes/Chart.yaml | 27 +++++ kubernetes/README.md | 37 +++++++ kubernetes/templates/_helpers.tpl | 53 ++++++++++ .../templates/proxypool-deployment.yaml | 29 ++++++ kubernetes/templates/proxypool-ingress.yaml | 41 ++++++++ kubernetes/templates/proxypool-service.yaml | 15 +++ kubernetes/templates/redis-deployment.yaml | 30 ++++++ kubernetes/templates/redis-service.yaml | 13 +++ kubernetes/values.yaml | 37 +++++++ 15 files changed, 316 insertions(+), 141 deletions(-) rename build.yml => build.yaml (91%) delete mode 100644 deployment.yml delete mode 100644 ingress.yml create mode 100644 kubernetes/.helmignore create mode 100644 kubernetes/Chart.yaml create mode 100644 kubernetes/README.md create mode 100644 kubernetes/templates/_helpers.tpl create mode 100644 kubernetes/templates/proxypool-deployment.yaml create mode 100644 kubernetes/templates/proxypool-ingress.yaml create mode 100644 kubernetes/templates/proxypool-service.yaml create mode 100644 kubernetes/templates/redis-deployment.yaml create mode 100644 kubernetes/templates/redis-service.yaml create mode 100644 kubernetes/values.yaml diff --git a/Dockerfile b/Dockerfile index c38bf89b..0db474f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ -FROM python:3.6-alpine +FROM nginx:alpine WORKDIR /app +RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev \ + openssl-dev libxml2-dev libxslt-dev gcc musl-dev py3-pip && \ + apk add --no-cache --update python3 && \ + pip3 install --upgrade pip setuptools +COPY requirements.txt . +RUN pip3 install -r requirements.txt && \ +apk del g++ gcc musl-dev libxml2-dev COPY . . # RUN pip install -r requirements.txt -i https://pypi.douban.com/simple -RUN apk add --no-cache libxml2-dev libxslt-dev gcc musl-dev && \ -pip install -r requirements.txt && \ -apk del gcc musl-dev libxml2-dev VOLUME ["/app/proxypool/crawlers/private"] CMD ["supervisord", "-c", "supervisord.conf"] diff --git a/README.md b/README.md index 3db7a2ea..17a8fab3 100644 --- a/README.md +++ b/README.md @@ -341,11 +341,7 @@ class Daili66Crawler(BaseCrawler): ## 部署 -本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，执行如下命令即可： - -```shell script -cat deployment.yml | sed 's/\${TAG}/latest/g' | kubectl apply -f - -``` +本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，请参考 [kubernetes](./kubernetes)。 ## 待开发 diff --git a/build.yml b/build.yaml similarity index 91% rename from build.yml rename to build.yaml index c069ec56..82fd9559 100644 --- a/build.yml +++ b/build.yaml @@ -8,7 +8,7 @@ services: # restart: always proxypool: build: . - image: "germey/proxypool" + image: "germey/proxypool:master" container_name: proxypool ports: - "5555:5555" diff --git a/deployment.yml b/deployment.yml deleted file mode 100644 index c7aaea55..00000000 --- a/deployment.yml +++ /dev/null @@ -1,99 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - creationTimestamp: null - name: proxypool ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: proxypool - namespace: proxypool -spec: - storageClassName: azure-file - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 -items: - - apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: proxypool - name: proxypool - namespace: proxypool - spec: - ports: - - name: "5555" - port: 5555 - targetPort: 5555 - selector: - io.kompose.service: proxypool - status: - loadBalancer: {} - - apiVersion: apps/v1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: proxypool - name: proxypool - namespace: proxypool - spec: - replicas: 2 - revisionHistoryLimit: 1 - strategy: {} - selector: - matchLabels: - io.kompose.service: proxypool - template: - metadata: - annotations: - kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml - kompose.version: 1.20.0 () - creationTimestamp: null - labels: - io.kompose.service: proxypool - spec: - containers: - - env: - - name: REDIS_CONNECTION_STRING - valueFrom: - secretKeyRef: - name: redis - key: connection_string - - name: REDIS_PORT - value: '6379' - image: germey/proxypool:${TAG} - name: proxypool - resources: - limits: - memory: "500Mi" - cpu: "300m" - requests: - memory: "500Mi" - cpu: "300m" - ports: - - containerPort: 5555 - volumeMounts: - - mountPath: "/app/proxypool/crawlers/private" - name: proxypool - restartPolicy: Always - volumes: - - name: proxypool - persistentVolumeClaim: - claimName: pvc-proxypool - status: {} -kind: List -metadata: {} diff --git a/ingress.yml b/ingress.yml deleted file mode 100644 index 166eb729..00000000 --- a/ingress.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1beta1 -kind: Ingress -metadata: - name: ingress-universal-proxypool - namespace: proxypool - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - tls: - - hosts: - - universal.proxypool.cuiqingcai.com - secretName: tls-wildcard-proxypool-cuiqingcai-com - - hosts: - - proxypool.scrape.center - secretName: tls-wildcard-scrape-center - rules: - - host: universal.proxypool.cuiqingcai.com - http: - paths: - - backend: - serviceName: proxypool - servicePort: 5555 - path: / - - host: proxypool.scrape.center - http: - paths: - - backend: - serviceName: proxypool - servicePort: 5555 - path: / \ No newline at end of file diff --git a/kubernetes/.helmignore b/kubernetes/.helmignore new file mode 100644 index 00000000..9716c30e --- /dev/null +++ b/kubernetes/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +image/ \ No newline at end of file diff --git a/kubernetes/Chart.yaml b/kubernetes/Chart.yaml new file mode 100644 index 00000000..58db2bc2 --- /dev/null +++ b/kubernetes/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +name: proxypool +description: A Efficient Proxy Pool + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# Keywords about this application. +keywords: + - proxypool + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 00000000..70860d24 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,37 @@ +# Kubernetes 部署 + +这是用来快速部署本代理池的 Helm Charts。 + +首先需要有一个 Kubernetes 集群，其次需要安装 Helm，确保 helm 命令可以正常运行。 + +## 安装 + +安装直接使用 helm 命令在本文件夹运行即可，使用 `-n` 可以制定 NameSpace。 + +```shell +helm install proxypool-app . -n scrape +``` + +其中 proxypool-app 就是应用的名字，可以任意取名，它会用作代理池 Deplyment 的名称。 + +如果需要覆盖变量，可以修改 values.yaml 文件，执行如下命令安装： + +```shell +helm install proxypool-app . -f values.yaml -n scrape +``` + +## 更新 + +如果需要更新配置，可以修改 values.yaml 文件，执行如下命令更新版本： + +```shell +helm upgrade proxypool-app . -f values.yaml -n scrape +``` + +## 卸载 + +如果不想使用了，可以只用 uninstall 命令卸载： + +```shell +helm uninstall proxypool-app -n scrape +``` diff --git a/kubernetes/templates/_helpers.tpl b/kubernetes/templates/_helpers.tpl new file mode 100644 index 00000000..31911df1 --- /dev/null +++ b/kubernetes/templates/_helpers.tpl @@ -0,0 +1,53 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "proxypool.name" -}} +{{- default .Chart.Name .Values.name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxypool.fullname" -}} +{{- if .Values.fullname }} +{{- .Values.fullname | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.name }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxypool.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxypool.labels" -}} +helm.sh/chart: {{ include "proxypool.chart" . }} +{{ include "proxypool.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxypool.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxypool.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/kubernetes/templates/proxypool-deployment.yaml b/kubernetes/templates/proxypool-deployment.yaml new file mode 100644 index 00000000..29ed068e --- /dev/null +++ b/kubernetes/templates/proxypool-deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxypool.fullname" . }} + labels: + {{- include "proxypool.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.deployment.replicas }} + revisionHistoryLimit: {{ .Values.deployment.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "proxypool.labels" . | nindent 8 }} + template: + metadata: + labels: + {{- include "proxypool.labels" . | nindent 8 }} + spec: + restartPolicy: {{ .Values.deployment.restartPolicy }} + containers: + - name: {{ include "proxypool.fullname" . }} + image: {{ .Values.deployment.image }} + ports: + - containerPort: 5555 + protocol: TCP + imagePullPolicy: {{ .Values.deployment.imagePullPolicy }} + resources: + {{- toYaml .Values.deployment.resources | nindent 12 }} + env: + {{- toYaml .Values.deployment.env | nindent 12 }} diff --git a/kubernetes/templates/proxypool-ingress.yaml b/kubernetes/templates/proxypool-ingress.yaml new file mode 100644 index 00000000..0706f5d2 --- /dev/null +++ b/kubernetes/templates/proxypool-ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "proxypool.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "proxypool.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} diff --git a/kubernetes/templates/proxypool-service.yaml b/kubernetes/templates/proxypool-service.yaml new file mode 100644 index 00000000..3d4285b4 --- /dev/null +++ b/kubernetes/templates/proxypool-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "proxypool.fullname" . }} + labels: + {{- include "proxypool.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: 5555 + protocol: TCP + name: http + selector: + {{- include "proxypool.selectorLabels" . | nindent 4 }} diff --git a/kubernetes/templates/redis-deployment.yaml b/kubernetes/templates/redis-deployment.yaml new file mode 100644 index 00000000..4acf4351 --- /dev/null +++ b/kubernetes/templates/redis-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: proxypool-redis + name: proxypool-redis +spec: + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: + app: proxypool-redis + template: + metadata: + labels: + app: proxypool-redis + spec: + containers: + - image: redis:alpine + name: proxypool-redis + ports: + - containerPort: 6379 + resources: + limits: + memory: "100Mi" + cpu: "100m" + requests: + memory: "100Mi" + cpu: "100m" + restartPolicy: Always diff --git a/kubernetes/templates/redis-service.yaml b/kubernetes/templates/redis-service.yaml new file mode 100644 index 00000000..5dbda554 --- /dev/null +++ b/kubernetes/templates/redis-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: proxypool-redis + name: proxypool-redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + app: proxypool-redis \ No newline at end of file diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml new file mode 100644 index 00000000..1a7421d1 --- /dev/null +++ b/kubernetes/values.yaml @@ -0,0 +1,37 @@ +name: proxypool +fullname: proxypool-app + +deployment: + image: germey/proxypool:master + imagePullPolicy: Always + restartPolicy: Always + revisionHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + replicas: 1 + resources: + limits: + memory: "200Mi" + cpu: "80m" + requests: + memory: "200Mi" + cpu: "80m" + env: + - name: REDIS_HOST + value: "proxypool-redis" + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + hosts: + - host: proxypool.scrape.center + paths: + - "/" + tls: + - secretName: tls-wildcard-scrape-center + hosts: + - proxypool.scrape.center From b40f033f6aeb6dc3743581d7313f87595f9b5317 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 14:51:01 +0800 Subject: [PATCH 094/139] rename --- .github/workflows/build.yml | 2 +- .github/workflows/deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1724a5a..966d6d23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: - name: Docker Login run: docker login -u germey -p ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - name: Build the Docker Image - run: docker-compose -f build.yml build + run: docker-compose -f build.yaml build - name: Tag and Push Master Version run: | docker tag germey/proxypool germey/proxypool:master diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ac5a5c69..871642d7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,7 +35,7 @@ jobs: echo $BUILD_NUMBER - name: Build Push Deploy run: | - docker-compose -f build.yml build + docker-compose -f build.yaml build docker tag germey/proxypool germey/proxypool:$BUILD_NUMBER docker push germey/proxypool docker push germey/proxypool:$BUILD_NUMBER From 3abbf218a7c45c9be2b1f44d6bd980fd398f6867 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 14:54:25 +0800 Subject: [PATCH 095/139] update readme --- kubernetes/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kubernetes/README.md b/kubernetes/README.md index 70860d24..327880df 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -4,6 +4,11 @@ 首先需要有一个 Kubernetes 集群，其次需要安装 Helm，确保 helm 命令可以正常运行。 +安装参考： + +- Kubernetes：[https://setup.scrape.center/kubernetes](https://setup.scrape.center/kubernetes)。 +- Helm: [https://setup.scrape.center/helm](https://setup.scrape.center/helm)。 + ## 安装 安装直接使用 helm 命令在本文件夹运行即可，使用 `-n` 可以制定 NameSpace。 From 598d02dcf8321f9a706ce0ee4ce5ef12ed8e4927 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 24 Dec 2021 15:22:28 +0800 Subject: [PATCH 096/139] update --- Dockerfile | 2 +- kubernetes/templates/proxypool-deployment.yaml | 8 ++++++++ proxypool/crawlers/base.py | 8 +++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0db474f1..1eb790fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:alpine +FROM alpine:3.7 WORKDIR /app RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev \ openssl-dev libxml2-dev libxslt-dev gcc musl-dev py3-pip && \ diff --git a/kubernetes/templates/proxypool-deployment.yaml b/kubernetes/templates/proxypool-deployment.yaml index 29ed068e..a12854d9 100644 --- a/kubernetes/templates/proxypool-deployment.yaml +++ b/kubernetes/templates/proxypool-deployment.yaml @@ -23,6 +23,14 @@ spec: - containerPort: 5555 protocol: TCP imagePullPolicy: {{ .Values.deployment.imagePullPolicy }} + livenessProbe: + httpGet: + path: /random + port: 5555 + initialDelaySeconds: 60 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 10 resources: {{- toYaml .Values.deployment.resources | nindent 12 }} env: diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 563d49bb..69d13fcd 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -4,9 +4,11 @@ from proxypool.setting import GET_TIMEOUT from fake_headers import Headers import time + + class BaseCrawler(object): urls = [] - + @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000) def fetch(self, url, **kwargs): try: @@ -14,13 +16,13 @@ def fetch(self, url, **kwargs): kwargs.setdefault('timeout', GET_TIMEOUT) kwargs.setdefault('verify', False) kwargs.setdefault('headers', headers) - response = requests.get(url ,**kwargs) + response = requests.get(url, **kwargs) if response.status_code == 200: response.encoding = 'utf-8' return response.text except requests.ConnectionError: return - + @logger.catch def crawl(self): """ From f7cf6007994fce9da9050d887346b30d577802c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Mon, 27 Dec 2021 02:46:05 +0800 Subject: [PATCH 097/139] Update build.yml --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 966d6d23..641861f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,5 @@ jobs: run: docker-compose -f build.yaml build - name: Tag and Push Master Version run: | - docker tag germey/proxypool germey/proxypool:master docker push germey/proxypool:master From 08385f6463b3b1cc4be8654bdb0a3dffae131854 Mon Sep 17 00:00:00 2001 From: Germey Date: Tue, 28 Dec 2021 16:18:46 +0800 Subject: [PATCH 098/139] catch retry error --- proxypool/crawlers/base.py | 28 +++++++++++++------ proxypool/crawlers/public/data5u.py | 16 ----------- .../{fatezero_proxylist.py => fatezero.py} | 0 proxypool/crawlers/public/ihuan.py | 1 + proxypool/crawlers/public/jiangxianli.py | 12 +++++--- proxypool/crawlers/public/xiaoshudaili.py | 15 ++++++---- proxypool/crawlers/public/xicidaili.py | 17 ----------- proxypool/crawlers/public/zhandaye.py | 2 +- proxypool/processors/getter.py | 6 ++-- 9 files changed, 42 insertions(+), 55 deletions(-) rename proxypool/crawlers/public/{fatezero_proxylist.py => fatezero.py} (100%) diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 69d13fcd..4a4bf5cd 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -1,4 +1,4 @@ -from retrying import retry +from retrying import RetryError, retry import requests from loguru import logger from proxypool.setting import GET_TIMEOUT @@ -23,15 +23,25 @@ def fetch(self, url, **kwargs): except requests.ConnectionError: return - @logger.catch + def process(self, html, url): + """ + used for parse html + """ + for proxy in self.parse(html): + logger.info(f'fetched proxy {proxy.string()} from {url}') + yield proxy + def crawl(self): """ crawl main method """ - for url in self.urls: - logger.info(f'fetching {url}') - html = self.fetch(url) - time.sleep(.5) - for proxy in self.parse(html): - logger.info(f'fetched proxy {proxy.string()} from {url}') - yield proxy + try: + for url in self.urls: + logger.info(f'fetching {url}') + html = self.fetch(url) + time.sleep(.5) + yield from self.process(html, url) + except RetryError: + logger.error( + f'crawler {self} crawled proxy unsuccessfully, ' + 'please check if target url is valid or network issue') diff --git a/proxypool/crawlers/public/data5u.py b/proxypool/crawlers/public/data5u.py index e36bf664..62158c20 100644 --- a/proxypool/crawlers/public/data5u.py +++ b/proxypool/crawlers/public/data5u.py @@ -11,23 +11,7 @@ class Data5UCrawler(BaseCrawler): data5u crawler, http://www.data5u.com """ urls = [BASE_URL] - - headers = { - 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' - } - @logger.catch - def crawl(self): - """ - crawl main method - """ - for url in self.urls: - logger.info(f'fetching {url}') - html = self.fetch(url, headers=self.headers) - for proxy in self.parse(html): - logger.info(f'fetched proxy {proxy.string()} from {url}') - yield proxy - def parse(self, html): """ parse html file to get proxies diff --git a/proxypool/crawlers/public/fatezero_proxylist.py b/proxypool/crawlers/public/fatezero.py similarity index 100% rename from proxypool/crawlers/public/fatezero_proxylist.py rename to proxypool/crawlers/public/fatezero.py diff --git a/proxypool/crawlers/public/ihuan.py b/proxypool/crawlers/public/ihuan.py index ccf90a13..4ca5e529 100644 --- a/proxypool/crawlers/public/ihuan.py +++ b/proxypool/crawlers/public/ihuan.py @@ -13,6 +13,7 @@ class IhuanCrawler(BaseCrawler): path = time.strftime("%Y/%m/%d/%H", time.localtime()) urls = [BASE_URL.format(path=path)] ignore = False + def parse(self, html): """ parse html file to get proxies diff --git a/proxypool/crawlers/public/jiangxianli.py b/proxypool/crawlers/public/jiangxianli.py index 14fc46cc..04533e3b 100644 --- a/proxypool/crawlers/public/jiangxianli.py +++ b/proxypool/crawlers/public/jiangxianli.py @@ -1,23 +1,27 @@ from proxypool.schemas.proxy import Proxy from proxypool.crawlers.base import BaseCrawler -import re import json + + BASE_URL = 'https://ip.jiangxianli.com/api/proxy_ips?page={page}' MAX_PAGE = 10 + + class JiangxianliCrawler(BaseCrawler): """ jiangxianli crawler,https://ip.jiangxianli.com/ """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] - + def parse(self, html): """ parse html file to get proxies :return: """ - - result =json.loads(html) + + result = json.loads(html) if result['code'] != 0: return MAX_PAGE = int(result['data']['last_page']) diff --git a/proxypool/crawlers/public/xiaoshudaili.py b/proxypool/crawlers/public/xiaoshudaili.py index bb4d7d42..44977605 100644 --- a/proxypool/crawlers/public/xiaoshudaili.py +++ b/proxypool/crawlers/public/xiaoshudaili.py @@ -1,7 +1,5 @@ import re - from pyquery import PyQuery as pq - from proxypool.schemas.proxy import Proxy from proxypool.crawlers.base import BaseCrawler @@ -16,16 +14,23 @@ class XiaoShuCrawler(BaseCrawler): """ def __init__(self): - html = self.fetch(url=BASE_URL) + """ + init urls + """ + try: + html = self.fetch(url=BASE_URL) + except: + self.urls = [] + return doc = pq(html) title = doc(".title:eq(0) a").items() - latest_page = 0 for t in title: res = re.search(r"/(\d+)\.html", t.attr("href")) latest_page = int(res.group(1)) if res else 0 if latest_page: - self.urls = [PAGE_BASE_URL.format(page=page) for page in range(latest_page - MAX_PAGE, latest_page)] + self.urls = [PAGE_BASE_URL.format(page=page) for page in range( + latest_page - MAX_PAGE, latest_page)] else: self.urls = [] diff --git a/proxypool/crawlers/public/xicidaili.py b/proxypool/crawlers/public/xicidaili.py index fdd2a317..53a4872e 100644 --- a/proxypool/crawlers/public/xicidaili.py +++ b/proxypool/crawlers/public/xicidaili.py @@ -12,23 +12,7 @@ class XicidailiCrawler(BaseCrawler): """ urls = [BASE_URL] ignore = True - - headers = { - 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' - } - @logger.catch - def crawl(self): - """ - crawl main method - """ - for url in self.urls: - logger.info(f'fetching {url}') - html = self.fetch(url, headers=self.headers) - for proxy in self.parse(html): - logger.info(f'fetched proxy {proxy.string()} from {url}') - yield proxy - def parse(self, html): """ parse html file to get proxies @@ -49,4 +33,3 @@ def parse(self, html): crawler = XicidailiCrawler() for proxy in crawler.crawl(): print(proxy) - diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py index 83af04b6..1522cdf0 100755 --- a/proxypool/crawlers/public/zhandaye.py +++ b/proxypool/crawlers/public/zhandaye.py @@ -8,6 +8,7 @@ BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html' MAX_PAGE = 5 * 2 + class ZhandayeCrawler(BaseCrawler): """ zhandaye crawler, https://www.zdaye.com/dayProxy/ @@ -56,4 +57,3 @@ def parse(self, html): crawler = ZhandayeCrawler() for proxy in crawler.crawl(): print(proxy) - diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py index 1a1d5261..877e198a 100644 --- a/proxypool/processors/getter.py +++ b/proxypool/processors/getter.py @@ -8,7 +8,7 @@ class Getter(object): """ getter of proxypool """ - + def __init__(self): """ init db and crawlers @@ -16,14 +16,14 @@ def __init__(self): self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] - + def is_full(self): """ if proxypool if full return: bool """ return self.redis.count() >= PROXY_NUMBER_MAX - + @logger.catch def run(self): """ From 642a59571621788d4a7fef7218b05a8a2694fd3a Mon Sep 17 00:00:00 2001 From: "Hiroshi.tao" Date: Thu, 30 Dec 2021 13:43:05 +0800 Subject: [PATCH 099/139] update dockerfile and build.yaml (#135) --- .dockerignore | 4 ++- .github/workflows/build.yml | 49 ++++++++++++++++++++++++++----------- .gitignore | 3 ++- Dockerfile | 24 ++++++++++-------- supervisord.conf | 11 +++++++++ 5 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.dockerignore b/.dockerignore index f9306766..3eab792f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -130,4 +130,6 @@ dmypy.json # Pyre type checker .pyre/ -proxypool/.env \ No newline at end of file +proxypool/.env +.DS_Store +.vscode \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 641861f1..1e5caef8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,22 +2,43 @@ name: build on: push: branches: - - master + - master paths-ignore: - - .gitignore - - README.md - - '.github/ISSUE_TEMPLATE/**' + - .gitignore + - README.md + - '.github/ISSUE_TEMPLATE/**' + release: + types: [published] + jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout Source - uses: actions/checkout@v1 - - name: Docker Login - run: docker login -u germey -p ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - - name: Build the Docker Image - run: docker-compose -f build.yaml build - - name: Tag and Push Master Version - run: | - docker push germey/proxypool:master - + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: germey + password: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} + + - name: Extract branch name + id: branch + shell: bash + run: echo "##[set-output name=tag;]$(echo ${GITHUB_REF##*/} | sed 's/master/latest/')" + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64 + tags: | + germey/proxypool:${{ steps.branch.outputs.tag }} diff --git a/.gitignore b/.gitignore index 7f21799f..16a7490c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ *.db venv /.idea -*.log \ No newline at end of file +*.log +.DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1eb790fe..68ce7b40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,17 @@ -FROM alpine:3.7 -WORKDIR /app -RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev \ - openssl-dev libxml2-dev libxslt-dev gcc musl-dev py3-pip && \ - apk add --no-cache --update python3 && \ - pip3 install --upgrade pip setuptools +FROM python:3.7-alpine AS build COPY requirements.txt . -RUN pip3 install -r requirements.txt && \ -apk del g++ gcc musl-dev libxml2-dev +RUN apk update &&\ + apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev &&\ + pip install --timeout 30 --user --no-cache-dir --no-warn-script-location -r requirements.txt + +FROM python:3.7-alpine +ENV APP_ENV=prod +ENV LOCAL_PKG="/root/.local" +COPY --from=build ${LOCAL_PKG} ${LOCAL_PKG} +RUN apk update && apk add --no-cache libffi-dev openssl-dev libxslt-dev &&\ + ln -sf ${LOCAL_PKG}/bin/* /usr/local/bin/ +WORKDIR /app COPY . . -# RUN pip install -r requirements.txt -i https://pypi.douban.com/simple +EXPOSE 5555 VOLUME ["/app/proxypool/crawlers/private"] -CMD ["supervisord", "-c", "supervisord.conf"] +ENTRYPOINT ["supervisord", "-c", "supervisord.conf"] \ No newline at end of file diff --git a/supervisord.conf b/supervisord.conf index 97f2f86f..aff2cd64 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -1,6 +1,17 @@ +[unix_http_server] +file=/run/supervisor.sock +chmod=0700 + [supervisord] +pidfile=/run/supervisord.pid nodaemon=true +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface + [program:tester] process_name=tester command=python3 run.py --processor tester From 8a21396633c6d2455e411dbd9cc3509e6daad7b4 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 30 Dec 2021 14:18:28 +0800 Subject: [PATCH 100/139] add env prefix --- proxypool/scheduler.py | 50 +++++++++++++++++++++---------------- proxypool/setting.py | 20 +++++++-------- proxypool/storages/redis.py | 20 ++++++++++----- proxypool/utils/proxy.py | 6 +++++ 4 files changed, 58 insertions(+), 38 deletions(-) diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index bec0c595..46a6b6d0 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -18,7 +18,7 @@ class Scheduler(): """ scheduler """ - + def run_tester(self, cycle=CYCLE_TESTER): """ run tester @@ -33,7 +33,7 @@ def run_tester(self, cycle=CYCLE_TESTER): tester.run() loop += 1 time.sleep(cycle) - + def run_getter(self, cycle=CYCLE_GETTER): """ run getter @@ -48,7 +48,7 @@ def run_getter(self, cycle=CYCLE_GETTER): getter.run() loop += 1 time.sleep(cycle) - + def run_server(self): """ run server for api @@ -57,42 +57,48 @@ def run_server(self): logger.info('server not enabled, exit') return app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) - + def run(self): global tester_process, getter_process, server_process try: logger.info('starting proxypool...') if ENABLE_TESTER: - tester_process = multiprocessing.Process(target=self.run_tester) + tester_process = multiprocessing.Process( + target=self.run_tester) logger.info(f'starting tester, pid {tester_process.pid}...') tester_process.start() - + if ENABLE_GETTER: - getter_process = multiprocessing.Process(target=self.run_getter) + getter_process = multiprocessing.Process( + target=self.run_getter) logger.info(f'starting getter, pid{getter_process.pid}...') getter_process.start() - + if ENABLE_SERVER: - server_process = multiprocessing.Process(target=self.run_server) + server_process = multiprocessing.Process( + target=self.run_server) logger.info(f'starting server, pid{server_process.pid}...') server_process.start() - - tester_process.join() - getter_process.join() - server_process.join() + + tester_process and tester_process.join() + getter_process and getter_process.join() + server_process and server_process.join() except KeyboardInterrupt: logger.info('received keyboard interrupt signal') - tester_process.terminate() - getter_process.terminate() - server_process.terminate() + tester_process and tester_process.terminate() + getter_process and getter_process.terminate() + server_process and server_process.terminate() finally: # must call join method before calling is_alive - tester_process.join() - getter_process.join() - server_process.join() - logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}') - logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}') - logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}') + tester_process and tester_process.join() + getter_process and getter_process.join() + server_process and server_process.join() + logger.info( + f'tester is {"alive" if tester_process.is_alive() else "dead"}') + logger.info( + f'getter is {"alive" if getter_process.is_alive() else "dead"}') + logger.info( + f'server is {"alive" if server_process.is_alive() else "dead"}') logger.info('proxy terminated') diff --git a/proxypool/setting.py b/proxypool/setting.py index 5765714b..d6d2d8ca 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -24,18 +24,19 @@ APP_TEST = IS_TEST = APP_ENV == TEST_MODE # redis host -REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1') +REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST', + env.str('REDIS_HOST', '127.0.0.1')) # redis port -REDIS_PORT = env.int('REDIS_PORT', 6379) +REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379)) # redis password, if no password, set it to None -REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) +REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD', + env.str('REDIS_PASSWORD', None)) # redis db, if no choice, set it to 0 -REDIS_DB = env.int('REDIS_DB', 0) -# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0 -REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) - -if REDIS_CONNECTION_STRING: - REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING) +REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0)) +# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0, +# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url +REDIS_CONNECTION_STRING = env.str( + 'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None)) # redis hash table key name REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') @@ -78,4 +79,3 @@ # logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') # logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') - diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 0ebbccc2..5ae145bc 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -1,7 +1,7 @@ import redis from proxypool.exceptions import PoolEmptyException from proxypool.schemas.proxy import Proxy -from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ +from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ PROXY_SCORE_INIT from random import choice from typing import List @@ -18,14 +18,21 @@ class RedisClient(object): redis connection client of proxypool """ - def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs): + def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, + connection_string=REDIS_CONNECTION_STRING, **kwargs): """ init redis client :param host: redis host :param port: redis port :param password: redis password + :param connection_string: redis connection_string """ - self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) + # if set connection_string, just use it + if connection_string: + self.db = redis.StrictRedis.from_url(connection_string) + else: + self.db = redis.StrictRedis( + host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: """ @@ -51,11 +58,13 @@ def random(self) -> Proxy: :return: proxy, like 8.8.8.8:8 """ # try to get proxy with max score - proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX) + proxies = self.db.zrangebyscore( + REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank - proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX) + proxies = self.db.zrevrange( + REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error @@ -125,4 +134,3 @@ def batch(self, cursor, count) -> List[Proxy]: conn = RedisClient() result = conn.random() print(result) - diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index b22e05a4..294033fc 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -2,6 +2,9 @@ def is_valid_proxy(data): + """ + check this string is within proxy format + """ if data.__contains__(':'): ip = data.split(':')[0] port = data.split(':')[1] @@ -11,6 +14,9 @@ def is_valid_proxy(data): def is_ip_valid(ip): + """ + check this string is within ip format + """ a = ip.split('.') if len(a) != 4: return False From 00d438809cbb454228422da02da89e61039951c7 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 30 Dec 2021 14:44:31 +0800 Subject: [PATCH 101/139] add prefix --- build.yaml | 3 +-- docker-compose.yml | 2 +- proxypool/setting.py | 4 ++-- proxypool/utils/parse.py | 13 ------------- 4 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 proxypool/utils/parse.py diff --git a/build.yaml b/build.yaml index 82fd9559..74b2fd0b 100644 --- a/build.yaml +++ b/build.yaml @@ -5,7 +5,6 @@ services: container_name: redis4proxypool ports: - "6374:6379" - # restart: always proxypool: build: . image: "germey/proxypool:master" @@ -16,4 +15,4 @@ services: # volumes: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis4proxypool + PROXYPOOL_REDIS_CONNECTION_STRING: redis://@redis4proxypool:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index 3c57f280..0b369788 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,4 +14,4 @@ services: # volumes: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis4proxypool + PROXYPOOL_REDIS_HOST: redis4proxypool diff --git a/proxypool/setting.py b/proxypool/setting.py index d6d2d8ca..d11771f4 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -2,7 +2,6 @@ from os.path import dirname, abspath, join from environs import Env from loguru import logger -from proxypool.utils.parse import parse_redis_connection_string env = Env() @@ -39,7 +38,8 @@ 'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None)) # redis hash table key name -REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') +REDIS_KEY = env.str('PROXYPOOL_REDIS_KEY', env.str( + 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores PROXY_SCORE_MAX = 100 diff --git a/proxypool/utils/parse.py b/proxypool/utils/parse.py deleted file mode 100644 index b3f42f5f..00000000 --- a/proxypool/utils/parse.py +++ /dev/null @@ -1,13 +0,0 @@ -import re - -def parse_redis_connection_string(connection_string): - """ - parse a redis connection string, for example: - redis://[password]@host:port - rediss://[password]@host:port - :param connection_string: - :return: - """ - result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)\/(\d+)', connection_string) - return result.group(2), int(result.group(3)), (result.group(1) or None), (result.group(4) or 0) if result \ - else ('localhost', 6379, None) From 04949758adf81f83ca3b7ab3c8396283c2ff57d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Thu, 30 Dec 2021 14:47:56 +0800 Subject: [PATCH 102/139] Add prefix for redis env (#137) * add env prefix * add prefix Co-authored-by: Germey --- build.yaml | 3 +-- docker-compose.yml | 2 +- proxypool/scheduler.py | 50 +++++++++++++++++++++---------------- proxypool/setting.py | 24 +++++++++--------- proxypool/storages/redis.py | 20 ++++++++++----- proxypool/utils/parse.py | 13 ---------- proxypool/utils/proxy.py | 6 +++++ 7 files changed, 62 insertions(+), 56 deletions(-) delete mode 100644 proxypool/utils/parse.py diff --git a/build.yaml b/build.yaml index 82fd9559..74b2fd0b 100644 --- a/build.yaml +++ b/build.yaml @@ -5,7 +5,6 @@ services: container_name: redis4proxypool ports: - "6374:6379" - # restart: always proxypool: build: . image: "germey/proxypool:master" @@ -16,4 +15,4 @@ services: # volumes: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis4proxypool + PROXYPOOL_REDIS_CONNECTION_STRING: redis://@redis4proxypool:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index 3c57f280..0b369788 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,4 +14,4 @@ services: # volumes: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: - REDIS_HOST: redis4proxypool + PROXYPOOL_REDIS_HOST: redis4proxypool diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index bec0c595..46a6b6d0 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -18,7 +18,7 @@ class Scheduler(): """ scheduler """ - + def run_tester(self, cycle=CYCLE_TESTER): """ run tester @@ -33,7 +33,7 @@ def run_tester(self, cycle=CYCLE_TESTER): tester.run() loop += 1 time.sleep(cycle) - + def run_getter(self, cycle=CYCLE_GETTER): """ run getter @@ -48,7 +48,7 @@ def run_getter(self, cycle=CYCLE_GETTER): getter.run() loop += 1 time.sleep(cycle) - + def run_server(self): """ run server for api @@ -57,42 +57,48 @@ def run_server(self): logger.info('server not enabled, exit') return app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) - + def run(self): global tester_process, getter_process, server_process try: logger.info('starting proxypool...') if ENABLE_TESTER: - tester_process = multiprocessing.Process(target=self.run_tester) + tester_process = multiprocessing.Process( + target=self.run_tester) logger.info(f'starting tester, pid {tester_process.pid}...') tester_process.start() - + if ENABLE_GETTER: - getter_process = multiprocessing.Process(target=self.run_getter) + getter_process = multiprocessing.Process( + target=self.run_getter) logger.info(f'starting getter, pid{getter_process.pid}...') getter_process.start() - + if ENABLE_SERVER: - server_process = multiprocessing.Process(target=self.run_server) + server_process = multiprocessing.Process( + target=self.run_server) logger.info(f'starting server, pid{server_process.pid}...') server_process.start() - - tester_process.join() - getter_process.join() - server_process.join() + + tester_process and tester_process.join() + getter_process and getter_process.join() + server_process and server_process.join() except KeyboardInterrupt: logger.info('received keyboard interrupt signal') - tester_process.terminate() - getter_process.terminate() - server_process.terminate() + tester_process and tester_process.terminate() + getter_process and getter_process.terminate() + server_process and server_process.terminate() finally: # must call join method before calling is_alive - tester_process.join() - getter_process.join() - server_process.join() - logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}') - logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}') - logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}') + tester_process and tester_process.join() + getter_process and getter_process.join() + server_process and server_process.join() + logger.info( + f'tester is {"alive" if tester_process.is_alive() else "dead"}') + logger.info( + f'getter is {"alive" if getter_process.is_alive() else "dead"}') + logger.info( + f'server is {"alive" if server_process.is_alive() else "dead"}') logger.info('proxy terminated') diff --git a/proxypool/setting.py b/proxypool/setting.py index 5765714b..d11771f4 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -2,7 +2,6 @@ from os.path import dirname, abspath, join from environs import Env from loguru import logger -from proxypool.utils.parse import parse_redis_connection_string env = Env() @@ -24,21 +23,23 @@ APP_TEST = IS_TEST = APP_ENV == TEST_MODE # redis host -REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1') +REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST', + env.str('REDIS_HOST', '127.0.0.1')) # redis port -REDIS_PORT = env.int('REDIS_PORT', 6379) +REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379)) # redis password, if no password, set it to None -REDIS_PASSWORD = env.str('REDIS_PASSWORD', None) +REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD', + env.str('REDIS_PASSWORD', None)) # redis db, if no choice, set it to 0 -REDIS_DB = env.int('REDIS_DB', 0) -# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0 -REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None) - -if REDIS_CONNECTION_STRING: - REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING) +REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0)) +# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0, +# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url +REDIS_CONNECTION_STRING = env.str( + 'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None)) # redis hash table key name -REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal') +REDIS_KEY = env.str('PROXYPOOL_REDIS_KEY', env.str( + 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores PROXY_SCORE_MAX = 100 @@ -78,4 +79,3 @@ # logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') # logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') - diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 0ebbccc2..5ae145bc 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -1,7 +1,7 @@ import redis from proxypool.exceptions import PoolEmptyException from proxypool.schemas.proxy import Proxy -from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ +from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ PROXY_SCORE_INIT from random import choice from typing import List @@ -18,14 +18,21 @@ class RedisClient(object): redis connection client of proxypool """ - def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs): + def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, + connection_string=REDIS_CONNECTION_STRING, **kwargs): """ init redis client :param host: redis host :param port: redis port :param password: redis password + :param connection_string: redis connection_string """ - self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) + # if set connection_string, just use it + if connection_string: + self.db = redis.StrictRedis.from_url(connection_string) + else: + self.db = redis.StrictRedis( + host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: """ @@ -51,11 +58,13 @@ def random(self) -> Proxy: :return: proxy, like 8.8.8.8:8 """ # try to get proxy with max score - proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX) + proxies = self.db.zrangebyscore( + REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank - proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX) + proxies = self.db.zrevrange( + REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error @@ -125,4 +134,3 @@ def batch(self, cursor, count) -> List[Proxy]: conn = RedisClient() result = conn.random() print(result) - diff --git a/proxypool/utils/parse.py b/proxypool/utils/parse.py deleted file mode 100644 index b3f42f5f..00000000 --- a/proxypool/utils/parse.py +++ /dev/null @@ -1,13 +0,0 @@ -import re - -def parse_redis_connection_string(connection_string): - """ - parse a redis connection string, for example: - redis://[password]@host:port - rediss://[password]@host:port - :param connection_string: - :return: - """ - result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)\/(\d+)', connection_string) - return result.group(2), int(result.group(3)), (result.group(1) or None), (result.group(4) or 0) if result \ - else ('localhost', 6379, None) diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index b22e05a4..294033fc 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -2,6 +2,9 @@ def is_valid_proxy(data): + """ + check this string is within proxy format + """ if data.__contains__(':'): ip = data.split(':')[0] port = data.split(':')[1] @@ -11,6 +14,9 @@ def is_valid_proxy(data): def is_ip_valid(ip): + """ + check this string is within ip format + """ a = ip.split('.') if len(a) != 4: return False From 1ccb847bbf03d01b06f016ccb8dd303f88a272cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Thu, 30 Dec 2021 14:53:34 +0800 Subject: [PATCH 103/139] Update build.yml --- .github/workflows/build.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e5caef8..f04456d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,11 +29,6 @@ jobs: username: germey password: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - - name: Extract branch name - id: branch - shell: bash - run: echo "##[set-output name=tag;]$(echo ${GITHUB_REF##*/} | sed 's/master/latest/')" - - name: Build and push uses: docker/build-push-action@v2 with: @@ -41,4 +36,5 @@ jobs: push: true platforms: linux/amd64 tags: | - germey/proxypool:${{ steps.branch.outputs.tag }} + germey/proxypool:latest + germey/proxypool:master From 9c78753f2e6943244e74c0ba143c7db5755c8ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Thu, 30 Dec 2021 15:01:50 +0800 Subject: [PATCH 104/139] Update build.yml --- .github/workflows/build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f04456d6..fe439bc2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,11 @@ jobs: with: username: germey password: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} - + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y%m%d')" + - name: Build and push uses: docker/build-push-action@v2 with: @@ -38,3 +42,4 @@ jobs: tags: | germey/proxypool:latest germey/proxypool:master + germey/proxypool:${{ steps.date.outputs.date }} From a5ff1833599bbf8bd915b83336bfafba02d4d11f Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 30 Dec 2021 15:09:41 +0800 Subject: [PATCH 105/139] update README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 17a8fab3..7897c7b2 100644 --- a/README.md +++ b/README.md @@ -236,12 +236,12 @@ get random proxy 116.196.115.209:8080 ### Redis 连接 -- REDIS_HOST：Redis 的 Host -- REDIS_PORT：Redis 的端口 -- REDIS_PASSWORD：Redis 的密码 -- REDIS_DB：Redis 的数据库索引，如 0、1 -- REDIS_CONNECTION_STRING：Redis 连接字符串 -- REDIS_KEY：Redis 储存代理使用字典的名称 +- PROXYPOOL_REDIS_HOST / REDIS_HOST：Redis 的 Host，其中 PROXYPOOL_REDIS_HOST 会覆盖 REDIS_HOST 的值。 +- PROXYPOOL_REDIS_PORT / REDIS_PORT：Redis 的端口，其中 PROXYPOOL_REDIS_PORT 会覆盖 REDIS_PORT 的值。 +- PROXYPOOL_REDIS_PASSWORD / REDIS_PASSWORD：Redis 的密码，其中 PROXYPOOL_REDIS_PASSWORD 会覆盖 REDIS_PASSWORD 的值。 +- PROXYPOOL_REDIS_DB / REDIS_DB：Redis 的数据库索引，如 0、1，其中 PROXYPOOL_REDIS_DB 会覆盖 REDIS_DB 的值。 +- PROXYPOOL_REDIS_CONNECTION_STRING / REDIS_CONNECTION_STRING：Redis 连接字符串，其中 PROXYPOOL_REDIS_CONNECTION_STRING 会覆盖 REDIS_CONNECTION_STRING 的值。 +- PROXYPOOL_REDIS_KEY / REDIS_KEY：Redis 储存代理使用字典的名称，其中 PROXYPOOL_REDIS_KEY 会覆盖 REDIS_KEY 的值。 ### 处理器 From 43336d45c33b438d25886f6d8b4b9629fc29cf5e Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 30 Dec 2021 15:19:05 +0800 Subject: [PATCH 106/139] update release --- release.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 release.sh diff --git a/release.sh b/release.sh new file mode 100755 index 00000000..342cd06f --- /dev/null +++ b/release.sh @@ -0,0 +1,2 @@ +git tag -a "`date +'%Y%m%d'`" -m "Release `date +'%Y%m%d'`" +git push origin --tags \ No newline at end of file From f37f3ce46fbef656e061eceaa472ff87afc37c10 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 31 Dec 2021 11:14:11 +0800 Subject: [PATCH 107/139] change max number --- proxypool/crawlers/public/daili66.py | 2 +- proxypool/crawlers/public/ip3366.py | 2 +- proxypool/crawlers/public/jiangxianli.py | 2 +- proxypool/crawlers/public/kuaidaili.py | 2 +- proxypool/crawlers/public/taiyangdaili.py | 2 +- proxypool/crawlers/public/xiaoshudaili.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/proxypool/crawlers/public/daili66.py b/proxypool/crawlers/public/daili66.py index 7b3bf7c2..aec7ea68 100644 --- a/proxypool/crawlers/public/daili66.py +++ b/proxypool/crawlers/public/daili66.py @@ -4,7 +4,7 @@ BASE_URL = 'http://www.66ip.cn/{page}.html' -MAX_PAGE = 50 +MAX_PAGE = 3 class Daili66Crawler(BaseCrawler): diff --git a/proxypool/crawlers/public/ip3366.py b/proxypool/crawlers/public/ip3366.py index 474a4f77..dfbc06f2 100644 --- a/proxypool/crawlers/public/ip3366.py +++ b/proxypool/crawlers/public/ip3366.py @@ -3,7 +3,7 @@ import re -MAX_PAGE = 8 +MAX_PAGE = 3 BASE_URL = 'http://www.ip3366.net/free/?stype={stype}&page={page}' diff --git a/proxypool/crawlers/public/jiangxianli.py b/proxypool/crawlers/public/jiangxianli.py index 04533e3b..861dd1e5 100644 --- a/proxypool/crawlers/public/jiangxianli.py +++ b/proxypool/crawlers/public/jiangxianli.py @@ -5,7 +5,7 @@ BASE_URL = 'https://ip.jiangxianli.com/api/proxy_ips?page={page}' -MAX_PAGE = 10 +MAX_PAGE = 3 class JiangxianliCrawler(BaseCrawler): diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py index 71ab1717..3602833e 100644 --- a/proxypool/crawlers/public/kuaidaili.py +++ b/proxypool/crawlers/public/kuaidaili.py @@ -5,7 +5,7 @@ BASE_URL = 'https://www.kuaidaili.com/free/{type}/{page}/' -MAX_PAGE = 300 +MAX_PAGE = 3 class KuaidailiCrawler(BaseCrawler): diff --git a/proxypool/crawlers/public/taiyangdaili.py b/proxypool/crawlers/public/taiyangdaili.py index 7a48cb43..b42388cc 100644 --- a/proxypool/crawlers/public/taiyangdaili.py +++ b/proxypool/crawlers/public/taiyangdaili.py @@ -3,7 +3,7 @@ from pyquery import PyQuery as pq BaseUrl = 'http://www.taiyanghttp.com/free/page{num}' -MAX_PAGE = 5 +MAX_PAGE = 3 class TaiyangdailiCrawler(BaseCrawler): diff --git a/proxypool/crawlers/public/xiaoshudaili.py b/proxypool/crawlers/public/xiaoshudaili.py index 44977605..f6fd0869 100644 --- a/proxypool/crawlers/public/xiaoshudaili.py +++ b/proxypool/crawlers/public/xiaoshudaili.py @@ -5,7 +5,7 @@ BASE_URL = "http://www.xsdaili.cn/" PAGE_BASE_URL = "http://www.xsdaili.cn/dayProxy/ip/{page}.html" -MAX_PAGE = 50 +MAX_PAGE = 3 class XiaoShuCrawler(BaseCrawler): From ad82d46fcd101fee1a491e7b3b450031652b9509 Mon Sep 17 00:00:00 2001 From: "Hiroshi.tao" Date: Fri, 31 Dec 2021 11:17:47 +0800 Subject: [PATCH 108/139] =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=E7=A7=8D=E6=96=B9=E6=B3=95=E8=BF=90?= =?UTF-8?q?=E8=A1=8Cserver=20(#138)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update dockerfile and build.yaml * update run app mode in prod - update readme * fix alpine python install gevent * fix #138 --- Dockerfile | 3 ++- README.md | 23 +++++++++--------- proxypool/processors/server.py | 9 ++++--- proxypool/scheduler.py | 44 ++++++++++++++++++++++++++++++---- proxypool/setting.py | 6 +++++ requirements.txt | 3 +++ 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 68ce7b40..c5ca5440 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM python:3.7-alpine AS build COPY requirements.txt . RUN apk update &&\ - apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev &&\ + apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev build-base musl-dev &&\ + pip install -U pip &&\ pip install --timeout 30 --user --no-cache-dir --no-warn-script-location -r requirements.txt FROM python:3.7-alpine diff --git a/README.md b/README.md index 7897c7b2..3e7fb011 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ cd ProxyPool 安装方法自行搜索即可。 +官方 Docker Hub 镜像：[germey/proxypool](https://hub.docker.com/r/germey/proxypool) + ### 常规方式 常规方式要求有 Python 环境、Redis 环境，具体要求如下： @@ -100,25 +102,20 @@ docker-compose -f build.yml up 设置 host、port、password，如果 password 为空可以设置为空字符串，示例如下： ```shell script -export REDIS_HOST='localhost' -export REDIS_PORT=6379 -export REDIS_PASSWORD='' -export REDIS_DB=0 +export PROXYPOOL_REDIS_HOST='localhost' +export PROXYPOOL_REDIS_PORT=6379 +export PROXYPOOL_REDIS_PASSWORD='' +export PROXYPOOL_REDIS_DB=0 ``` 或者只设置连接字符串： ```shell script -export REDIS_CONNECTION_STRING='redis://[password]@host:port/db' -``` - -如果没有密码也要设置为： - -```shell script -export REDIS_CONNECTION_STRING='redis://@host:port/db' +export PROXYPOOL_REDIS_CONNECTION_STRING='redis://localhost' ``` -这里连接字符串的格式需要符合 `redis://[password]@host:port/db` 的格式，注意不要遗漏 `@`。 +这里连接字符串的格式需要符合 `redis://[:password@]host[:port][/database]` 的格式， +中括号参数可以省略，port默认是6379，database默认是0，密码默认为空。 以上两种设置任选其一即可。 @@ -233,6 +230,8 @@ get random proxy 116.196.115.209:8080 - APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev - APP_DEBUG：调试模式，可以设置 true 或 false，默认 true +- APP_PROD_METHOD: 正式环境启动应用方式，默认是`gevent`， + 可选：`tornado`，`meinheld`（分别需要安装tornado或meinheld模块） ### Redis 连接 diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index d3edd70d..f7138c64 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,11 +1,13 @@ from flask import Flask, g from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED +from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV __all__ = ['app'] app = Flask(__name__) +if IS_DEV: + app.debug = True def get_conn(): @@ -46,8 +48,9 @@ def get_proxy_all(): conn = get_conn() proxies = conn.all() proxies_string = '' - for proxy in proxies: - proxies_string += str(proxy) + '\n' + if proxies: + for proxy in proxies: + proxies_string += str(proxy) + '\n' return proxies_string diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index 46a6b6d0..16c3c47a 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -3,7 +3,8 @@ from proxypool.processors.server import app from proxypool.processors.getter import Getter from proxypool.processors.tester import Tester -from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, API_THREADED, API_PORT, ENABLE_SERVER, \ +from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, \ + API_THREADED, API_PORT, ENABLE_SERVER, IS_PROD, APP_PROD_METHOD, \ ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS from loguru import logger @@ -56,7 +57,42 @@ def run_server(self): if not ENABLE_SERVER: logger.info('server not enabled, exit') return - app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) + if IS_PROD: + if APP_PROD_METHOD == 'gevent': + try: + from gevent.pywsgi import WSGIServer + except ImportError as e: + logger.exception(e) + else: + http_server = WSGIServer((API_HOST, API_PORT), app) + http_server.serve_forever() + + elif APP_PROD_METHOD == 'tornado': + try: + from tornado.wsgi import WSGIContainer + from tornado.httpserver import HTTPServer + from tornado.ioloop import IOLoop + except ImportError as e: + logger.exception(e) + else: + http_server = HTTPServer(WSGIContainer(app)) + http_server.listen(API_PORT) + IOLoop.instance().start() + + elif APP_PROD_METHOD == "meinheld": + try: + import meinheld + except ImportError as e: + logger.exception(e) + else: + meinheld.listen((API_HOST, API_PORT)) + meinheld.run(app) + + else: + logger.error("unsupported APP_PROD_METHOD") + return + else: + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) def run(self): global tester_process, getter_process, server_process @@ -71,13 +107,13 @@ def run(self): if ENABLE_GETTER: getter_process = multiprocessing.Process( target=self.run_getter) - logger.info(f'starting getter, pid{getter_process.pid}...') + logger.info(f'starting getter, pid {getter_process.pid}...') getter_process.start() if ENABLE_SERVER: server_process = multiprocessing.Process( target=self.run_server) - logger.info(f'starting server, pid{server_process.pid}...') + logger.info(f'starting server, pid {server_process.pid}...') server_process.start() tester_process and tester_process.join() diff --git a/proxypool/setting.py b/proxypool/setting.py index d11771f4..ba8b3895 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -22,6 +22,12 @@ APP_PROD = IS_PROD = APP_ENV == PROD_MODE APP_TEST = IS_TEST = APP_ENV == TEST_MODE +# Which WSGI container is used to run applications +# - gevent: pip install gevent +# - tornado: pip install tornado +# - meinheld: pip install meinheld +APP_PROD_METHOD = env.str('APP_PROD_METHOD', "gevent").lower() + # redis host REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST', env.str('REDIS_HOST', '127.0.0.1')) diff --git a/requirements.txt b/requirements.txt index aa5eddcc..3c797aed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,6 @@ redis==3.5.3 lxml==4.6.5 fake_headers==1.0.2 maxminddb_geolite2==2018.703 +gevent>=21.1.0 +tornado>=6.0 +meinheld>=1.0.0 \ No newline at end of file From 20d16f97b428364649de2e9036d5f235f771f2e5 Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 31 Dec 2021 11:20:32 +0800 Subject: [PATCH 109/139] fix --- proxypool/scheduler.py | 8 ++++---- proxypool/setting.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index 16c3c47a..f29b5655 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -3,7 +3,7 @@ from proxypool.processors.server import app from proxypool.processors.getter import Getter from proxypool.processors.tester import Tester -from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, \ +from proxypool.setting import APP_PROD_METHOD_GEVENT, APP_PROD_METHOD_MEINHELD, APP_PROD_METHOD_TORNADO, CYCLE_GETTER, CYCLE_TESTER, API_HOST, \ API_THREADED, API_PORT, ENABLE_SERVER, IS_PROD, APP_PROD_METHOD, \ ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS from loguru import logger @@ -58,7 +58,7 @@ def run_server(self): logger.info('server not enabled, exit') return if IS_PROD: - if APP_PROD_METHOD == 'gevent': + if APP_PROD_METHOD == APP_PROD_METHOD_GEVENT: try: from gevent.pywsgi import WSGIServer except ImportError as e: @@ -67,7 +67,7 @@ def run_server(self): http_server = WSGIServer((API_HOST, API_PORT), app) http_server.serve_forever() - elif APP_PROD_METHOD == 'tornado': + elif APP_PROD_METHOD == APP_PROD_METHOD_TORNADO: try: from tornado.wsgi import WSGIContainer from tornado.httpserver import HTTPServer @@ -79,7 +79,7 @@ def run_server(self): http_server.listen(API_PORT) IOLoop.instance().start() - elif APP_PROD_METHOD == "meinheld": + elif APP_PROD_METHOD == APP_PROD_METHOD_MEINHELD: try: import meinheld except ImportError as e: diff --git a/proxypool/setting.py b/proxypool/setting.py index ba8b3895..b00bc888 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -26,7 +26,10 @@ # - gevent: pip install gevent # - tornado: pip install tornado # - meinheld: pip install meinheld -APP_PROD_METHOD = env.str('APP_PROD_METHOD', "gevent").lower() +APP_PROD_METHOD_GEVENT = 'gevent' +APP_PROD_METHOD_TORNADO = 'tornado' +APP_PROD_METHOD_MEINHELD = 'meinheld' +APP_PROD_METHOD = env.str('APP_PROD_METHOD', APP_PROD_METHOD_GEVENT).lower() # redis host REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST', From 34d5a4389ac75d50fbc07903d3080ec69b6529db Mon Sep 17 00:00:00 2001 From: Germey Date: Fri, 31 Dec 2021 11:34:55 +0800 Subject: [PATCH 110/139] update decode --- proxypool/storages/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 5ae145bc..8ab0e41d 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -29,7 +29,7 @@ def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db """ # if set connection_string, just use it if connection_string: - self.db = redis.StrictRedis.from_url(connection_string) + self.db = redis.StrictRedis.from_url(connection_string, decode_responses=True, **kwargs) else: self.db = redis.StrictRedis( host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) From b1dd8b2cec27880c0f9b36569adc3eab6ae82185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Fri, 31 Dec 2021 15:28:12 +0800 Subject: [PATCH 111/139] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e7fb011..00015c60 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 当然你也可以选择自己 Build，直接运行如下命令即可： ``` -docker-compose -f build.yml up +docker-compose -f build.yaml up ``` 如果下载速度特别慢，可以自行修改 Dockerfile，修改： From be7e3edf0028211c6e7e49d20ce9a9650d9b1d1d Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 2 Jan 2022 17:16:37 +0800 Subject: [PATCH 112/139] add port --- kubernetes/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml index 1a7421d1..15b25377 100644 --- a/kubernetes/values.yaml +++ b/kubernetes/values.yaml @@ -16,8 +16,10 @@ deployment: memory: "200Mi" cpu: "80m" env: - - name: REDIS_HOST + - name: PROXYPOOL_REDIS_HOST value: "proxypool-redis" + - name: PROXYPOOL_REDIS_PORT + value: "6379" service: type: ClusterIP From feb5d354ff60f121848ed1e1bbfbedb49ef65583 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 27 Feb 2022 14:54:56 +0800 Subject: [PATCH 113/139] upgrade aiohttp --- proxypool/setting.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxypool/setting.py b/proxypool/setting.py index b00bc888..d3f65da9 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -70,7 +70,7 @@ TEST_TIMEOUT = env.int('TEST_TIMEOUT', 10) TEST_BATCH = env.int('TEST_BATCH', 20) # only save anonymous proxy -TEST_ANONYMOUS = True +TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True) # TEST_HEADERS = env.json('TEST_HEADERS', { # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) diff --git a/requirements.txt b/requirements.txt index 3c797aed..f25c61d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ environs==9.3.0 Flask==1.1.2 attrs==20.3.0 retrying==1.3.3 -aiohttp==3.7.4 +aiohttp==3.8.1 requests==2.25.1 loguru==0.5.3 pyquery==1.4.3 From 23a3b2bce2f328182713bf795632cee93b1403be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Fri, 4 Mar 2022 12:02:47 +0800 Subject: [PATCH 114/139] Update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f25c61d7..81757ad9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ fake_headers==1.0.2 maxminddb_geolite2==2018.703 gevent>=21.1.0 tornado>=6.0 -meinheld>=1.0.0 \ No newline at end of file +meinheld>=1.0.0 +itsdangerous==0.24 From c683f8265efd6036337541ab70770c4046455e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Sun, 6 Mar 2022 12:09:17 +0800 Subject: [PATCH 115/139] fix (#146) --- proxypool/setting.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/proxypool/setting.py b/proxypool/setting.py index d3f65da9..4fb80fea 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -22,6 +22,7 @@ APP_PROD = IS_PROD = APP_ENV == PROD_MODE APP_TEST = IS_TEST = APP_ENV == TEST_MODE + # Which WSGI container is used to run applications # - gevent: pip install gevent # - tornado: pip install tornado @@ -86,5 +87,24 @@ ENABLE_GETTER = env.bool('ENABLE_GETTER', True) ENABLE_SERVER = env.bool('ENABLE_SERVER', True) -# logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days') -# logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week') + +ENABLE_LOG_FILE = env.bool('ENABLE_LOG_FILE', True) +ENABLE_LOG_RUNTIME_FILE = env.bool('ENABLE_LOG_RUNTIME_FILE', True) +ENABLE_LOG_ERROR_FILE = env.bool('ENABLE_LOG_ERROR_FILE', True) + + +LOG_LEVEL_MAP = { + DEV_MODE: "DEBUG", + TEST_MODE: "INFO", + PROD_MODE: "ERROR" +} + +LOG_LEVEL = LOG_LEVEL_MAP.get(APP_ENV) + +if ENABLE_LOG_FILE: + if ENABLE_LOG_RUNTIME_FILE: + logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), + level=LOG_LEVEL, rotation='1 week', retention='20 days') + if ENABLE_LOG_ERROR_FILE: + logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), + level='ERROR', rotation='1 week') From bd6c2fdcb3b0dc9aeb7547d50bdadb6898492ea3 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 6 Mar 2022 12:09:55 +0800 Subject: [PATCH 116/139] update version --- requirements.txt | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index 81757ad9..75ffa829 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -environs==9.3.0 -Flask==1.1.2 -attrs==20.3.0 -retrying==1.3.3 -aiohttp==3.8.1 -requests==2.25.1 -loguru==0.5.3 -pyquery==1.4.3 -supervisor==4.2.1 -redis==3.5.3 -lxml==4.6.5 -fake_headers==1.0.2 +environs>=9.3.0,<10.0.0 +Flask>=1.1.2,<2.0.0 +attrs>=20.3.0,<21.0.0 +retrying>=1.3.3,<2.0.0 +aiohttp>=3.8.1,<4.0.0 +requests>=2.25.1,<3.0.0 +loguru>=0.5.3,<1.0.0 +pyquery>=1.4.3,<2.0.0 +supervisor>=4.2.1,<5.0.0 +redis>=3.5.3,<4.0.0 +lxml>=4.6.5,<5.0.0 +fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.1.0 -tornado>=6.0 -meinheld>=1.0.0 -itsdangerous==0.24 +gevent>=21.1.0,<22.0.0 +tornado>=6.0,<7.0 +meinheld>=1.0.0,<2.0.0 +itsdangerous>=0.24,<1.0.0 From 8cdab49e417c0593adfac9fa8567572714a74b3c Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 6 Mar 2022 12:12:40 +0800 Subject: [PATCH 117/139] update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 00015c60..8389e50d 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ get random proxy 116.196.115.209:8080 - ENABLE_GETTER：允许 Getter 启动，默认 true - ENABLE_SERVER：运行 Server 启动，默认 true + ### 环境 - APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev @@ -259,6 +260,9 @@ get random proxy 116.196.115.209:8080 - LOG_DIR：日志相对路径 - LOG_RUNTIME_FILE：运行日志文件名称 - LOG_ERROR_FILE：错误日志文件名称 +- ENABLE_LOG_FILE：是否输出 log 文件，默认 true，如果设置为 false，那么 ENABLE_LOG_RUNTIME_FILE 和 ENABLE_LOG_ERROR_FILE 都不会生效 +- ENABLE_LOG_RUNTIME_FILE：是否输出 runtime log 文件，默认 true +- ENABLE_LOG_ERROR_FILE：是否输出 error log 文件，默认 true 以上内容均可使用环境变量配置，即在运行前设置对应环境变量值即可，如更改测试地址和 Redis 键名： From 1615d5de7ca4c5e263c6b5aa7200cf6191aee994 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 6 Mar 2022 12:25:24 +0800 Subject: [PATCH 118/139] remove port --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0b369788..782f5477 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,8 @@ services: redis4proxypool: image: redis:alpine container_name: redis4proxypool - ports: - - "6374:6379" + # ports: + # - "6374:6379" proxypool: image: "germey/proxypool:master" container_name: proxypool From d7a0bcef7e63ae57a99ad094929a5889bb3168bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Thu, 24 Mar 2022 11:26:20 +0800 Subject: [PATCH 119/139] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 75ffa829..38011966 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ gevent>=21.1.0,<22.0.0 tornado>=6.0,<7.0 meinheld>=1.0.0,<2.0.0 itsdangerous>=0.24,<1.0.0 +MarkupSafe<2.1.0 From ebb0bc6fe1b59f846751bb63bea0fba308178b21 Mon Sep 17 00:00:00 2001 From: Weltolk <40228052+Weltolk@users.noreply.github.com> Date: Sun, 3 Apr 2022 10:23:32 +0800 Subject: [PATCH 120/139] Update README.md (#153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 错了一个字 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8389e50d..0a44eb42 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ get random proxy 116.196.115.209:8080 - TEST_URL：测试 URL，默认百度 - TEST_TIMEOUT：测试超时时间，默认 10 秒 - TEST_BATCH：批量测试数量，默认 20 个代理 -- TEST_VALID_STATUS：测试有效的状态吗 +- TEST_VALID_STATUS：测试有效的状态码 - API_HOST：代理 Server 运行 Host，默认 0.0.0.0 - API_PORT：代理 Server 运行端口，默认 5555 - API_THREADED：代理 Server 是否使用多线程，默认 true From 0ebac358a2644ab93fbf078802e639f5c5c15a25 Mon Sep 17 00:00:00 2001 From: Germey Date: Wed, 6 Apr 2022 22:16:35 +0800 Subject: [PATCH 121/139] fix --- proxypool/crawlers/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py index 4a4bf5cd..611a816f 100644 --- a/proxypool/crawlers/base.py +++ b/proxypool/crawlers/base.py @@ -20,7 +20,7 @@ def fetch(self, url, **kwargs): if response.status_code == 200: response.encoding = 'utf-8' return response.text - except requests.ConnectionError: + except (requests.ConnectionError, requests.ReadTimeout): return def process(self, html, url): @@ -39,6 +39,8 @@ def crawl(self): for url in self.urls: logger.info(f'fetching {url}') html = self.fetch(url) + if not html: + continue time.sleep(.5) yield from self.process(html, url) except RetryError: From 2154cefc9a8ccd4a1a31edab8dbe09cff2350474 Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 11 Jul 2022 00:33:26 +0800 Subject: [PATCH 122/139] add log settings --- README.md | 7 ++++--- docker-compose.yml | 1 + proxypool/setting.py | 9 +++++++-- requirements.txt | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0a44eb42..256f1155 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ export PROXYPOOL_REDIS_CONNECTION_STRING='redis://localhost' ``` 这里连接字符串的格式需要符合 `redis://[:password@]host[:port][/database]` 的格式， -中括号参数可以省略，port默认是6379，database默认是0，密码默认为空。 +中括号参数可以省略，port 默认是 6379，database 默认是 0，密码默认为空。 以上两种设置任选其一即可。 @@ -226,13 +226,12 @@ get random proxy 116.196.115.209:8080 - ENABLE_GETTER：允许 Getter 启动，默认 true - ENABLE_SERVER：运行 Server 启动，默认 true - ### 环境 - APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev - APP_DEBUG：调试模式，可以设置 true 或 false，默认 true - APP_PROD_METHOD: 正式环境启动应用方式，默认是`gevent`， - 可选：`tornado`，`meinheld`（分别需要安装tornado或meinheld模块） + 可选：`tornado`，`meinheld`（分别需要安装 tornado 或 meinheld 模块） ### Redis 连接 @@ -260,6 +259,8 @@ get random proxy 116.196.115.209:8080 - LOG_DIR：日志相对路径 - LOG_RUNTIME_FILE：运行日志文件名称 - LOG_ERROR_FILE：错误日志文件名称 +- LOG_ROTATION: 日志记录周转周期或大小，见 [loguru - rotation](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) +- LOG_RETENTION: 日志保留日期，见 [loguru - retention](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) - ENABLE_LOG_FILE：是否输出 log 文件，默认 true，如果设置为 false，那么 ENABLE_LOG_RUNTIME_FILE 和 ENABLE_LOG_ERROR_FILE 都不会生效 - ENABLE_LOG_RUNTIME_FILE：是否输出 runtime log 文件，默认 true - ENABLE_LOG_ERROR_FILE：是否输出 error log 文件，默认 true diff --git a/docker-compose.yml b/docker-compose.yml index 782f5477..cf367f42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,4 @@ services: # - proxypool/crawlers/private:/app/proxypool/crawlers/private environment: PROXYPOOL_REDIS_HOST: redis4proxypool + diff --git a/proxypool/setting.py b/proxypool/setting.py index 4fb80fea..e68f45ca 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -2,6 +2,7 @@ from os.path import dirname, abspath, join from environs import Env from loguru import logger +import shutil env = Env() @@ -100,11 +101,15 @@ } LOG_LEVEL = LOG_LEVEL_MAP.get(APP_ENV) +LOG_ROTATION = env.str('LOG_ROTATION', '500MB') +LOG_RETENTION = env.str('LOG_RETENTION', '1 week') if ENABLE_LOG_FILE: if ENABLE_LOG_RUNTIME_FILE: logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), - level=LOG_LEVEL, rotation='1 week', retention='20 days') + level=LOG_LEVEL, rotation=LOG_ROTATION, retention=LOG_RETENTION) if ENABLE_LOG_ERROR_FILE: logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), - level='ERROR', rotation='1 week') + level='ERROR', rotation=LOG_ROTATION) +else: + shutil.rmtree(LOG_DIR, ignore_errors=True) diff --git a/requirements.txt b/requirements.txt index 38011966..4e7d5388 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,5 @@ maxminddb_geolite2==2018.703 gevent>=21.1.0,<22.0.0 tornado>=6.0,<7.0 meinheld>=1.0.0,<2.0.0 -itsdangerous>=0.24,<1.0.0 +itsdangerous==0.24 MarkupSafe<2.1.0 From 78325d0ea06b026123a45abe31611ffab18493ab Mon Sep 17 00:00:00 2001 From: Germey Date: Mon, 11 Jul 2022 00:36:03 +0800 Subject: [PATCH 123/139] add readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 256f1155..d01d7a08 100644 --- a/README.md +++ b/README.md @@ -259,8 +259,8 @@ get random proxy 116.196.115.209:8080 - LOG_DIR：日志相对路径 - LOG_RUNTIME_FILE：运行日志文件名称 - LOG_ERROR_FILE：错误日志文件名称 -- LOG_ROTATION: 日志记录周转周期或大小，见 [loguru - rotation](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) -- LOG_RETENTION: 日志保留日期，见 [loguru - retention](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) +- LOG_ROTATION: 日志记录周转周期或大小，默认 500MB，见 [loguru - rotation](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) +- LOG_RETENTION: 日志保留日期，默认 7 天，见 [loguru - retention](https://github.com/Delgan/loguru#easier-file-logging-with-rotation--retention--compression) - ENABLE_LOG_FILE：是否输出 log 文件，默认 true，如果设置为 false，那么 ENABLE_LOG_RUNTIME_FILE 和 ENABLE_LOG_ERROR_FILE 都不会生效 - ENABLE_LOG_RUNTIME_FILE：是否输出 runtime log 文件，默认 true - ENABLE_LOG_ERROR_FILE：是否输出 error log 文件，默认 true From 4c50711dde75e33de813acccab1042d5f95487de Mon Sep 17 00:00:00 2001 From: Takayama <49364055+MGMCN@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:55:52 +0900 Subject: [PATCH 124/139] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=20geonodeda?= =?UTF-8?q?ili.py=20=E7=88=AC=E5=8F=96=E4=BB=A3=E7=90=86=20(#186)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add geonodedaili.py * add headers through crawl function --- proxypool/crawlers/public/geonodedaili.py | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 proxypool/crawlers/public/geonodedaili.py diff --git a/proxypool/crawlers/public/geonodedaili.py b/proxypool/crawlers/public/geonodedaili.py new file mode 100644 index 00000000..f71f16ec --- /dev/null +++ b/proxypool/crawlers/public/geonodedaili.py @@ -0,0 +1,71 @@ +import time +from retrying import RetryError +from loguru import logger +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import json + +BASE_URL = 'https://proxylist.geonode.com/api/proxy-list?limit=500&page={page}&sort_by=lastChecked&sort_type=desc' +MAX_PAGE = 18 + + +class GeonodeCrawler(BaseCrawler): + """ + Geonode crawler, https://proxylist.geonode.com/ + """ + urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + try: + result = json.loads(html) + proxy_list = result['data'] + for proxy_item in proxy_list: + host = proxy_item['ip'] + port = proxy_item['port'] + yield Proxy(host=host, port=port) + except json.JSONDecodeError: + print("json.JSONDecodeError") + return + + def crawl(self): + """ + override crawl main method + add headers + """ + headers = { + 'authority': 'proxylist.geonode.com', + 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"', + 'accept': 'application/json, text/plain, */*', + 'sec-ch-ua-mobile': '?0', + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', + 'sec-ch-ua-platform': '"macOS"', + 'origin': 'https://geonode.com', + 'sec-fetch-site': 'same-site', + 'sec-fetch-mode': 'cors', + 'sec-fetch-dest': 'empty', + 'referer': 'https://geonode.com/', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7', + 'if-none-match': 'W/"c25d-BXjLTmP+/yYXtIz4OEcmdOWSv88"', + } + try: + for url in self.urls: + logger.info(f'fetching {url}') + html = self.fetch(url, headers=headers) + if not html: + continue + time.sleep(.5) + yield from self.process(html, url) + except RetryError: + logger.error( + f'crawler {self} crawled proxy unsuccessfully, ' + 'please check if target url is valid or network issue') + + +if __name__ == '__main__': + crawler = GeonodeCrawler() + for proxy in crawler.crawl(): + print(proxy) From 0fd5e6cf025848d4ef1c0f157b1ba92aa38dea71 Mon Sep 17 00:00:00 2001 From: Takayama <49364055+MGMCN@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:56:58 +0900 Subject: [PATCH 125/139] requirements.txt modified (#187) --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4e7d5388..c9407c7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,7 @@ redis>=3.5.3,<4.0.0 lxml>=4.6.5,<5.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.1.0,<22.0.0 +gevent>=21.8.0,<22.0.0 tornado>=6.0,<7.0 -meinheld>=1.0.0,<2.0.0 itsdangerous==0.24 MarkupSafe<2.1.0 From a0edcb0c4ce274b22ac0a8dcc7ea54225bcfdd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E5=A4=A7=E5=B8=85Aiden?= <38374506+dashuaixu@users.noreply.github.com> Date: Wed, 30 Aug 2023 21:49:30 +0800 Subject: [PATCH 126/139] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=A7=81=E4=BA=BA?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E6=B1=A0=E5=9C=BA=E6=99=AF=E4=B8=AD=EF=BC=8C?= =?UTF-8?q?=E5=B8=A6=E8=BA=AB=E4=BB=BD=E8=AE=A4=E8=AF=81=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E7=9A=84=E6=94=AF=E6=8C=81=20(#198)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 增加私人代理池场景中，带身份认证的代理的支持 * 修复忘记修改部分 --------- Co-authored-by: dashuai xu --- proxypool/utils/proxy.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index 294033fc..79cc27fb 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -5,7 +5,10 @@ def is_valid_proxy(data): """ check this string is within proxy format """ - if data.__contains__(':'): + if is_auth_proxy(data): + host, port = extract_auth_proxy(data) + return is_ip_valid(host) and is_port_valid(port) + elif data.__contains__(':'): ip = data.split(':')[0] port = data.split(':')[1] return is_ip_valid(ip) and is_port_valid(port) @@ -17,6 +20,8 @@ def is_ip_valid(ip): """ check this string is within ip format """ + if is_auth_proxy(ip): + ip = ip.split('@')[1] a = ip.split('.') if len(a) != 4: return False @@ -48,9 +53,36 @@ def convert_proxy_or_proxies(data): # skip invalid item item = item.strip() if not is_valid_proxy(item): continue - host, port = item.split(':') + if is_auth_proxy(item): + host, port = extract_auth_proxy(item) + else: + host, port = item.split(':') result.append(Proxy(host=host, port=int(port))) return result if isinstance(data, str) and is_valid_proxy(data): - host, port = data.split(':') + if is_auth_proxy(data): + host, port = extract_auth_proxy(data) + else: + host, port = data.split(':') return Proxy(host=host, port=int(port)) + + +def is_auth_proxy(data: str) -> bool: + return '@' in data + + +def extract_auth_proxy(data: str) -> (str, str): + """ + extract host and port from a proxy with authentication + """ + auth = data.split('@')[0] + ip_port = data.split('@')[1] + ip = ip_port.split(':')[0] + port = ip_port.split(':')[1] + host = auth + '@' + ip + return host, port + + +if __name__ == '__main__': + proxy = 'test1234:test5678.@117.68.216.212:32425' + print(extract_auth_proxy(proxy)) From 0a6d8610c135ae10748aff6f1164d4b911286bf6 Mon Sep 17 00:00:00 2001 From: inVains <2282229+inVains@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:59:24 +0800 Subject: [PATCH 127/139] =?UTF-8?q?#140=20=E5=8F=8A=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=8F=AF=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=9A=84=E5=8F=82=E6=95=B0=20(#202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 1、设置scheduler.py中dev模式的flask run，不进行启动时的自动重新加载reload * 1、修改PROXY_SCORE_MAX，PROXY_SCORE_MIN，PROXY_SCORE_INIT三项配置，为可环境变量配置 2、添加可环境变量配置项TEST_DONT_SET_MAX_SCORE，允许设置当tester检测到某个proxy可用时，只是保持原score，而不将其score设置成max。 * 增加获取proxy接口的认证header, API-KEY。可配置，默认不需要 --- proxypool/processors/server.py | 29 ++++++++++++++++++++++++++--- proxypool/processors/tester.py | 20 ++++++++++++-------- proxypool/scheduler.py | 2 +- proxypool/setting.py | 12 +++++++++--- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index f7138c64..aa0500ac 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,7 +1,7 @@ -from flask import Flask, g +from flask import Flask, g, request from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, IS_DEV - +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV +import functools __all__ = ['app'] @@ -10,6 +10,25 @@ app.debug = True +def auth_required(func): + @functools.wraps(func) + def decorator(*args, **kwargs): + # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator + if API_KEY == "": + return func(*args, **kwargs) + if request.headers.get('API-KEY', None) is not None: + api_key = request.headers.get('API-KEY') + else: + return {"message": "Please provide an API key in header"}, 400 + # Check if API key is correct and valid + if request.method == "GET" and api_key == API_KEY: + return func(*args, **kwargs) + else: + return {"message": "The provided API key is not valid"}, 403 + + return decorator + + def get_conn(): """ get redis client object @@ -21,6 +40,7 @@ def get_conn(): @app.route('/') +@auth_required def index(): """ get home page, you can define your own templates @@ -30,6 +50,7 @@ def index(): @app.route('/random') +@auth_required def get_proxy(): """ get a random proxy @@ -40,6 +61,7 @@ def get_proxy(): @app.route('/all') +@auth_required def get_proxy_all(): """ get a random proxy @@ -56,6 +78,7 @@ def get_proxy_all(): @app.route('/count') +@auth_required def get_count(): """ get the count of proxies diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index f002056a..353332ac 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -3,11 +3,11 @@ from loguru import logger from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient -from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS +from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ + TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError - EXCEPTIONS = ( ClientProxyConnectionError, ConnectionRefusedError, @@ -23,14 +23,14 @@ class Tester(object): """ tester for testing proxies in queue """ - + def __init__(self): """ init redis """ self.redis = RedisClient() self.loop = asyncio.get_event_loop() - + async def test(self, proxy: Proxy): """ test single proxy @@ -55,15 +55,18 @@ async def test(self, proxy: Proxy): async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, allow_redirects=False) as response: if response.status in TEST_VALID_STATUS: - self.redis.max(proxy) - logger.debug(f'proxy {proxy.string()} is valid, set max score') + if TEST_DONT_SET_MAX_SCORE: + logger.debug(f'proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max(proxy) + logger.debug(f'proxy {proxy.string()} is valid, set max score') else: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') except EXCEPTIONS: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') - + @logger.catch def run(self): """ @@ -84,14 +87,15 @@ def run(self): if not cursor: break + def run_tester(): host = '96.113.165.182' port = '3128' tasks = [tester.test(Proxy(host=host, port=port))] tester.loop.run_until_complete(asyncio.wait(tasks)) + if __name__ == '__main__': tester = Tester() tester.run() # run_tester() - diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index f29b5655..a2d18abe 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -92,7 +92,7 @@ def run_server(self): logger.error("unsupported APP_PROD_METHOD") return else: - app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED) + app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False) def run(self): global tester_process, getter_process, server_process diff --git a/proxypool/setting.py b/proxypool/setting.py index e68f45ca..495ad55f 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -53,9 +53,9 @@ 'REDIS_KEY', 'proxies:universal')) # definition of proxy scores -PROXY_SCORE_MAX = 100 -PROXY_SCORE_MIN = 0 -PROXY_SCORE_INIT = 10 +PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) +PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) +PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) # definition of proxy number PROXY_NUMBER_MAX = 50000 @@ -77,11 +77,17 @@ # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # }) TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302]) +# whether to set max score when one proxy is tested valid +TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False) # definition of api API_HOST = env.str('API_HOST', '0.0.0.0') API_PORT = env.int('API_PORT', 5555) API_THREADED = env.bool('API_THREADED', True) +# add an api key to get proxy +# need a header of `API-KEY` in get request to pass the authenticate +# API_KEY='', do not need `API-KEY` header +API_KEY = env.str('API_KEY', '') # flags of enable ENABLE_TESTER = env.bool('ENABLE_TESTER', True) From 2344ee1db07e9095e00153fa41280cfea436c8a4 Mon Sep 17 00:00:00 2001 From: Shayne Wang <1614565666@qq.com> Date: Fri, 1 Dec 2023 19:05:08 +0800 Subject: [PATCH 128/139] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E4=BB=A3=E7=90=86=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proxypool/crawlers/public/docip.py | 38 +++++++++++++++++++++ proxypool/crawlers/public/uqidata.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 proxypool/crawlers/public/docip.py create mode 100644 proxypool/crawlers/public/uqidata.py diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py new file mode 100644 index 00000000..070c5983 --- /dev/null +++ b/proxypool/crawlers/public/docip.py @@ -0,0 +1,38 @@ +import time +from retrying import RetryError +from loguru import logger +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +import json + +BASE_URL = 'https://www.docip.net/data/free.json?t={date}' + + + +class DocipCrawler(BaseCrawler): + """ + Docip crawler, https://www.docip.net/data/free.json + """ + urls = [BASE_URL.format(date=time.strftime("%Y%m%d", time.localtime()))] + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + try: + result = json.loads(html) + proxy_list = result['data'] + for proxy_item in proxy_list: + host = proxy_item['ip'] + port = proxy_item['port'] + yield Proxy(host=host, port=port) + except json.JSONDecodeError: + print("json.JSONDecodeError") + return + + +if __name__ == '__main__': + crawler = DocipCrawler() + for proxy in crawler.crawl(): + print(proxy) diff --git a/proxypool/crawlers/public/uqidata.py b/proxypool/crawlers/public/uqidata.py new file mode 100644 index 00000000..3e54b2dc --- /dev/null +++ b/proxypool/crawlers/public/uqidata.py @@ -0,0 +1,49 @@ +from pyquery import PyQuery as pq +from proxypool.schemas.proxy import Proxy +from proxypool.crawlers.base import BaseCrawler +from loguru import logger + +BASE_URL = 'https://ip.uqidata.com/free/index.html' + + +class UqidataCrawler(BaseCrawler): + """ + Uqidata crawler, https://ip.uqidata.com/free/index.html + """ + urls = [BASE_URL] + ignore = True + + def encode(input_str): + tmp = [] + for i in range(len(input_str)): + tmp.append("ABCDEFGHIZ".find(input_str[i])) + result = "".join(str(i) for i in tmp) + result = int(result) >> 0x03 + return result + + def parse(self, html): + """ + parse html file to get proxies + :return: + """ + doc = pq(html) + trs = doc('#main_container .inner table tbody tr:nth-child(n+3)').items() + for tr in trs: + ip_html = tr('td.ip').find("*").items() + host = '' + for i in ip_html: + if i.attr('style') is not None and 'none' in i.attr('style'): + continue + if i.text() == '': + continue + host += i.text() + + port_code = tr('td.port').attr('class').split(' ')[1] + port = UqidataCrawler.encode(port_code) + yield Proxy(host=host, port=port) + + +if __name__ == '__main__': + crawler = UqidataCrawler() + for proxy in crawler.crawl(): + print(proxy) From 97229e6f470ee0dd834e0ba4b60091ce85657b4f Mon Sep 17 00:00:00 2001 From: Dawei Feng <397615103@qq.com> Date: Wed, 27 Dec 2023 02:58:22 +0800 Subject: [PATCH 129/139] fix docip bug & add support for python 3.11 (#205) --- proxypool/crawlers/public/docip.py | 2 +- proxypool/processors/tester.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py index 070c5983..154871fb 100644 --- a/proxypool/crawlers/public/docip.py +++ b/proxypool/crawlers/public/docip.py @@ -25,7 +25,7 @@ def parse(self, html): proxy_list = result['data'] for proxy_item in proxy_list: host = proxy_item['ip'] - port = proxy_item['port'] + port = host.split(':')[-1] yield Proxy(host=host, port=port) except json.JSONDecodeError: print("json.JSONDecodeError") diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 353332ac..58795285 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -82,7 +82,7 @@ def run(self): logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) if proxies: - tasks = [self.test(proxy) for proxy in proxies] + tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies] self.loop.run_until_complete(asyncio.wait(tasks)) if not cursor: break diff --git a/requirements.txt b/requirements.txt index c9407c7d..33f35c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ redis>=3.5.3,<4.0.0 lxml>=4.6.5,<5.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.8.0,<22.0.0 +gevent>=21.8.0,<24.0.0 tornado>=6.0,<7.0 itsdangerous==0.24 MarkupSafe<2.1.0 From 003358686720e6450867d35ff821921a819d8e80 Mon Sep 17 00:00:00 2001 From: Cesaryuan <35998162+cesaryuan@users.noreply.github.com> Date: Mon, 1 Jan 2024 22:30:20 +0800 Subject: [PATCH 130/139] fix: avoid error on "ip:port:port" format (#207) --- proxypool/utils/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index 79cc27fb..5330ceb4 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -56,7 +56,7 @@ def convert_proxy_or_proxies(data): if is_auth_proxy(item): host, port = extract_auth_proxy(item) else: - host, port = item.split(':') + host, port, *_ = item.split(':') result.append(Proxy(host=host, port=int(port))) return result if isinstance(data, str) and is_valid_proxy(data): From 78b324498bc3ae9370a78bb08a7b89f1f25a50e3 Mon Sep 17 00:00:00 2001 From: inVains <2282229+inVains@users.noreply.github.com> Date: Tue, 26 Mar 2024 00:36:24 +0800 Subject: [PATCH 131/139] add sub proxy pool mechanics (#213) --- proxypool/processors/getter.py | 5 +++- proxypool/processors/server.py | 21 +++++++++++--- proxypool/processors/tester.py | 28 +++++++++++++++++++ proxypool/setting.py | 2 ++ proxypool/storages/redis.py | 50 +++++++++++++++++----------------- proxypool/testers/__init__.py | 16 +++++++++++ proxypool/testers/base.py | 19 +++++++++++++ 7 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 proxypool/testers/__init__.py create mode 100644 proxypool/testers/base.py diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py index 877e198a..c5c16296 100644 --- a/proxypool/processors/getter.py +++ b/proxypool/processors/getter.py @@ -2,7 +2,7 @@ from proxypool.storages.redis import RedisClient from proxypool.setting import PROXY_NUMBER_MAX from proxypool.crawlers import __all__ as crawlers_cls - +from proxypool.testers import __all__ as testers_cls class Getter(object): """ @@ -16,6 +16,8 @@ def __init__(self): self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] def is_full(self): """ @@ -36,6 +38,7 @@ def run(self): logger.info(f'crawler {crawler} to get proxy') for proxy in crawler.crawl(): self.redis.add(proxy) + [self.redis.add(proxy, redis_key=tester.key) for tester in self.testers] if __name__ == '__main__': diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index aa0500ac..50144590 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,6 +1,7 @@ from flask import Flask, g, request +from proxypool.exceptions import PoolEmptyException from proxypool.storages.redis import RedisClient -from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV +from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED import functools __all__ = ['app'] @@ -53,10 +54,19 @@ def index(): @auth_required def get_proxy(): """ - get a random proxy + get a random proxy, can query the specific sub-pool according the (redis) key + if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool :return: get a random proxy """ + key = request.args.get('key') conn = get_conn() + # return conn.random(key).string() if key else conn.random().string() + if key: + try: + return conn.random(key).string() + except PoolEmptyException: + if not PROXY_RAND_KEY_DEGRADED: + raise return conn.random().string() @@ -67,8 +77,10 @@ def get_proxy_all(): get a random proxy :return: get a random proxy """ + key = request.args.get('key') + conn = get_conn() - proxies = conn.all() + proxies = conn.all(key) if key else conn.all() proxies_string = '' if proxies: for proxy in proxies: @@ -85,7 +97,8 @@ def get_count(): :return: count, int """ conn = get_conn() - return str(conn.count()) + key = request.args.get('key') + return str(conn.count(key)) if key else conn.count() if __name__ == '__main__': diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 58795285..470259a9 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -7,6 +7,7 @@ TEST_DONT_SET_MAX_SCORE from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError from asyncio import TimeoutError +from proxypool.testers import __all__ as testers_cls EXCEPTIONS = ( ClientProxyConnectionError, @@ -30,6 +31,8 @@ def __init__(self): """ self.redis = RedisClient() self.loop = asyncio.get_event_loop() + self.testers_cls = testers_cls + self.testers = [tester_cls() for tester_cls in self.testers_cls] async def test(self, proxy: Proxy): """ @@ -63,8 +66,33 @@ async def test(self, proxy: Proxy): else: self.redis.decrease(proxy) logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + # if independent tester class found, create new set of storage and do the extra test + for tester in self.testers: + key = tester.key + if self.redis.exists(proxy, key): + test_url = tester.test_url + headers = tester.headers() + cookies = tester.cookies() + async with session.get(test_url, proxy=f'http://{proxy.string()}', + timeout=TEST_TIMEOUT, + headers=headers, + cookies=cookies, + allow_redirects=False) as response: + resp_text = await response.text() + is_valid = await tester.parse(resp_text, test_url, proxy.string()) + if is_valid: + if tester.test_dont_set_max_score: + logger.info(f'key[{key}] proxy {proxy.string()} is valid, remain current score') + else: + self.redis.max(proxy, key, tester.proxy_score_max) + logger.info(f'key[{key}] proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease(proxy, tester.key, tester.proxy_score_min) + logger.info(f'key[{key}] proxy {proxy.string()} is invalid, decrease score') + except EXCEPTIONS: self.redis.decrease(proxy) + [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) for tester in self.testers] logger.debug(f'proxy {proxy.string()} is invalid, decrease score') @logger.catch diff --git a/proxypool/setting.py b/proxypool/setting.py index 495ad55f..a445667e 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -56,6 +56,8 @@ PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100) PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0) PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10) +# whether to get a universal random proxy if no proxy exists in the sub-pool identified by a specific key +PROXY_RAND_KEY_DEGRADED = env.bool('TEST_ANONYMOUS', True) # definition of proxy number PROXY_NUMBER_MAX = 50000 diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 8ab0e41d..2d052323 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -34,7 +34,7 @@ def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db self.db = redis.StrictRedis( host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) - def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: + def add(self, proxy: Proxy, score=PROXY_SCORE_INIT, redis_key=REDIS_KEY) -> int: """ add proxy and set it to init score :param proxy: proxy, ip:port, like 8.8.8.8:88 @@ -44,12 +44,12 @@ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int: if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): logger.info(f'invalid proxy {proxy}, throw it') return - if not self.exists(proxy): + if not self.exists(proxy, redis_key): if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, score, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): score}) + return self.db.zadd(redis_key, score, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): score}) - def random(self) -> Proxy: + def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> Proxy: """ get random proxy firstly try to get proxy with max score @@ -59,74 +59,74 @@ def random(self) -> Proxy: """ # try to get proxy with max score proxies = self.db.zrangebyscore( - REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX) + redis_key, proxy_score_max, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else get proxy by rank proxies = self.db.zrevrange( - REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX) + redis_key, proxy_score_min, proxy_score_max) if len(proxies): return convert_proxy_or_proxies(choice(proxies)) # else raise error raise PoolEmptyException - def decrease(self, proxy: Proxy) -> int: + def decrease(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN) -> int: """ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it :param proxy: proxy :return: new score """ if IS_REDIS_VERSION_2: - self.db.zincrby(REDIS_KEY, proxy.string(), -1) + self.db.zincrby(redis_key, proxy.string(), -1) else: - self.db.zincrby(REDIS_KEY, -1, proxy.string()) - score = self.db.zscore(REDIS_KEY, proxy.string()) + self.db.zincrby(redis_key, -1, proxy.string()) + score = self.db.zscore(redis_key, proxy.string()) logger.info(f'{proxy.string()} score decrease 1, current {score}') - if score <= PROXY_SCORE_MIN: + if score <= proxy_score_min: logger.info(f'{proxy.string()} current score {score}, remove') - self.db.zrem(REDIS_KEY, proxy.string()) + self.db.zrem(redis_key, proxy.string()) - def exists(self, proxy: Proxy) -> bool: + def exists(self, proxy: Proxy, redis_key=REDIS_KEY) -> bool: """ if proxy exists :param proxy: proxy :return: if exists, bool """ - return not self.db.zscore(REDIS_KEY, proxy.string()) is None + return not self.db.zscore(redis_key, proxy.string()) is None - def max(self, proxy: Proxy) -> int: + def max(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_max=PROXY_SCORE_MAX) -> int: """ set proxy to max score :param proxy: proxy :return: new score """ - logger.info(f'{proxy.string()} is valid, set to {PROXY_SCORE_MAX}') + logger.info(f'{proxy.string()} is valid, set to {proxy_score_max}') if IS_REDIS_VERSION_2: - return self.db.zadd(REDIS_KEY, PROXY_SCORE_MAX, proxy.string()) - return self.db.zadd(REDIS_KEY, {proxy.string(): PROXY_SCORE_MAX}) + return self.db.zadd(redis_key, proxy_score_max, proxy.string()) + return self.db.zadd(redis_key, {proxy.string(): proxy_score_max}) - def count(self) -> int: + def count(self, redis_key=REDIS_KEY) -> int: """ get count of proxies :return: count, int """ - return self.db.zcard(REDIS_KEY) + return self.db.zcard(redis_key) - def all(self) -> List[Proxy]: + def all(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: """ get all proxies :return: list of proxies """ - return convert_proxy_or_proxies(self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)) + return convert_proxy_or_proxies(self.db.zrangebyscore(redis_key, proxy_score_min, proxy_score_max)) - def batch(self, cursor, count) -> List[Proxy]: + def batch(self, cursor, count, redis_key=REDIS_KEY) -> List[Proxy]: """ get batch of proxies :param cursor: scan cursor :param count: scan count :return: list of proxies """ - cursor, proxies = self.db.zscan(REDIS_KEY, cursor, count=count) + cursor, proxies = self.db.zscan(redis_key, cursor, count=count) return cursor, convert_proxy_or_proxies([i[0] for i in proxies]) diff --git a/proxypool/testers/__init__.py b/proxypool/testers/__init__.py new file mode 100644 index 00000000..4e4df95e --- /dev/null +++ b/proxypool/testers/__init__.py @@ -0,0 +1,16 @@ +import pkgutil +from .base import BaseTester +import inspect + + +# load classes subclass of BaseCrawler +classes = [] +for loader, name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(name).load_module(name) + for name, value in inspect.getmembers(module): + globals()[name] = value + if inspect.isclass(value) and issubclass(value, BaseTester) and value is not BaseTester \ + and not getattr(value, 'ignore', False): + classes.append(value) +__all__ = __ALL__ = classes + diff --git a/proxypool/testers/base.py b/proxypool/testers/base.py new file mode 100644 index 00000000..796b7cfc --- /dev/null +++ b/proxypool/testers/base.py @@ -0,0 +1,19 @@ +from proxypool.setting import TEST_DONT_SET_MAX_SCORE, PROXY_SCORE_INIT, PROXY_SCORE_MAX, PROXY_SCORE_MIN + + +class BaseTester(object): + test_url = "" + key = "" + test_dont_set_max_score = TEST_DONT_SET_MAX_SCORE + proxy_score_init = PROXY_SCORE_INIT + proxy_score_max = PROXY_SCORE_MAX + proxy_score_min = PROXY_SCORE_MIN + + def headers(self): + return None + + def cookies(self): + return None + + async def parse(self, html, url, proxy, expr='{"code":0'): + return True if expr in html else False From 7c77ad0b078702918882c1fd0976419642daad24 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 30 Jun 2024 11:08:48 +0800 Subject: [PATCH 132/139] validate testing ip --- README.md | 11 ---------- build.yaml | 18 ---------------- docker-compose.yml | 5 ++--- proxypool/processors/tester.py | 39 +++++++++++++++++++++++----------- 4 files changed, 29 insertions(+), 44 deletions(-) delete mode 100644 build.yaml diff --git a/README.md b/README.md index d01d7a08..e1435763 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,6 @@ proxypool | 2020-02-19 17:09:46,596 INFO success: tester entered RUNNING stat 这时候访问 [http://localhost:5555/random](http://localhost:5555/random) 即可获取一个随机可用代理。 -当然你也可以选择自己 Build，直接运行如下命令即可： - -``` -docker-compose -f build.yaml up -``` - 如果下载速度特别慢，可以自行修改 Dockerfile，修改： ```diff @@ -347,11 +341,6 @@ class Daili66Crawler(BaseCrawler): 本项目提供了 Kubernetes 部署脚本，如需部署到 Kubernetes，请参考 [kubernetes](./kubernetes)。 -## 待开发 - -- [ ] 前端页面管理 -- [ ] 使用情况统计分析 - 如有一起开发的兴趣可以在 Issue 留言，非常感谢！ ## LICENSE diff --git a/build.yaml b/build.yaml deleted file mode 100644 index 74b2fd0b..00000000 --- a/build.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" -services: - redis4proxypool: - image: redis:alpine - container_name: redis4proxypool - ports: - - "6374:6379" - proxypool: - build: . - image: "germey/proxypool:master" - container_name: proxypool - ports: - - "5555:5555" - restart: always - # volumes: - # - proxypool/crawlers/private:/app/proxypool/crawlers/private - environment: - PROXYPOOL_REDIS_CONNECTION_STRING: redis://@redis4proxypool:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index cf367f42..4e4d5936 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,16 +3,15 @@ services: redis4proxypool: image: redis:alpine container_name: redis4proxypool - # ports: - # - "6374:6379" proxypool: + build: . image: "germey/proxypool:master" container_name: proxypool ports: - "5555:5555" restart: always # volumes: - # - proxypool/crawlers/private:/app/proxypool/crawlers/private + # - proxypool/crawlers/private:~/proxypool/crawlers/private environment: PROXYPOOL_REDIS_HOST: redis4proxypool diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 470259a9..6937af8c 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -45,27 +45,33 @@ async def test(self, proxy: Proxy): logger.debug(f'testing {proxy.string()}') # if TEST_ANONYMOUS is True, make sure that # the proxy has the effect of hiding the real IP + # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') if TEST_ANONYMOUS: url = 'https://httpbin.org/ip' async with session.get(url, timeout=TEST_TIMEOUT) as response: resp_json = await response.json() origin_ip = resp_json['origin'] + # logger.debug(f'origin ip is {origin_ip}') async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: resp_json = await response.json() anonymous_ip = resp_json['origin'] + logger.debug(f'anonymous ip is {anonymous_ip}') assert origin_ip != anonymous_ip assert proxy.host == anonymous_ip async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, allow_redirects=False) as response: if response.status in TEST_VALID_STATUS: if TEST_DONT_SET_MAX_SCORE: - logger.debug(f'proxy {proxy.string()} is valid, remain current score') + logger.debug( + f'proxy {proxy.string()} is valid, remain current score') else: self.redis.max(proxy) - logger.debug(f'proxy {proxy.string()} is valid, set max score') + logger.debug( + f'proxy {proxy.string()} is valid, set max score') else: self.redis.decrease(proxy) - logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') # if independent tester class found, create new set of storage and do the extra test for tester in self.testers: key = tester.key @@ -82,18 +88,25 @@ async def test(self, proxy: Proxy): is_valid = await tester.parse(resp_text, test_url, proxy.string()) if is_valid: if tester.test_dont_set_max_score: - logger.info(f'key[{key}] proxy {proxy.string()} is valid, remain current score') + logger.info( + f'key[{key}] proxy {proxy.string()} is valid, remain current score') else: - self.redis.max(proxy, key, tester.proxy_score_max) - logger.info(f'key[{key}] proxy {proxy.string()} is valid, set max score') + self.redis.max( + proxy, key, tester.proxy_score_max) + logger.info( + f'key[{key}] proxy {proxy.string()} is valid, set max score') else: - self.redis.decrease(proxy, tester.key, tester.proxy_score_min) - logger.info(f'key[{key}] proxy {proxy.string()} is invalid, decrease score') + self.redis.decrease( + proxy, tester.key, tester.proxy_score_min) + logger.info( + f'key[{key}] proxy {proxy.string()} is invalid, decrease score') except EXCEPTIONS: self.redis.decrease(proxy) - [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) for tester in self.testers] - logger.debug(f'proxy {proxy.string()} is invalid, decrease score') + [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) + for tester in self.testers] + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') @logger.catch def run(self): @@ -107,10 +120,12 @@ def run(self): logger.debug(f'{count} proxies to test') cursor = 0 while True: - logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') + logger.debug( + f'testing proxies use cursor {cursor}, count {TEST_BATCH}') cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) if proxies: - tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies] + tasks = [self.loop.create_task( + self.test(proxy)) for proxy in proxies] self.loop.run_until_complete(asyncio.wait(tasks)) if not cursor: break From 3ed558b2255d0790d51b838422163e5cdc0d414f Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 30 Jun 2024 11:16:40 +0800 Subject: [PATCH 133/139] add proxy --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index e1435763..474bf65b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,18 @@ 代理池原理解析可见「[如何搭建一个高效的代理池](https://cuiqingcai.com/7048.html)」，建议使用之前阅读。 +## 使用前注意 + +本代理池是基于市面上各种公开代理源搭建的，所以可用性并不高，很可能上百上千个代理中才能找到一两个可用代理，不适合直接用于爬虫爬取任务。 + +如果您的目的是为了尽快使用代理完成爬取任务，建议您对接一些付费代理或者直接使用已有代理资源；如果您的目的是为了学习如何搭建一个代理池，您可以参考本项目继续完成后续步骤。 + +付费代理推荐： + +- [ADSL 拨号代理](https://platform.acedata.cloud/documents/a82a528a-8e32-4c4c-a9d0-a21be7c9ef8c)：海量拨号（中国境内）高质量代理 +- [海外/全球代理](https://platform.acedata.cloud/documents/50f1437a-1857-43c5-85cf-5800ae1b31e4)：中国境外高质量代理 +- [蜂窝 4G/5G 代理](https://platform.acedata.cloud/documents/1cc59b19-1550-4169-a59d-ad6faf7f7517)：极高质量（中国境内）防风控代理 + ## 使用准备 首先当然是克隆代码并进入 ProxyPool 文件夹： From dc32b04c94724f4d0e37d155eb191df6fee1d466 Mon Sep 17 00:00:00 2001 From: Teehoo AI Date: Fri, 27 Mar 2026 20:47:19 -0700 Subject: [PATCH 134/139] fix: use constant-time comparison for auth token (CWE-208) (#227) --- proxypool/processors/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index 50144590..fb50ce5e 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,3 +1,4 @@ +import hmac from flask import Flask, g, request from proxypool.exceptions import PoolEmptyException from proxypool.storages.redis import RedisClient @@ -22,7 +23,7 @@ def decorator(*args, **kwargs): else: return {"message": "Please provide an API key in header"}, 400 # Check if API key is correct and valid - if request.method == "GET" and api_key == API_KEY: + if request.method == "GET" and hmac.compare_digest(api_key, API_KEY): return func(*args, **kwargs) else: return {"message": "The provided API key is not valid"}, 403 From 91a1983ad4a98402871d0ac1a85c4fd4681e7b30 Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 4 Jul 2026 16:27:52 +0800 Subject: [PATCH 135/139] fix: resolve core runtime bugs in tester, scheduler and proxy parsing (#229) - utils/proxy: fix ValueError on multi-colon proxy strings in the str branch of convert_proxy_or_proxies, which made /random return 500 (#218) - scheduler: guard None process handles in finally so ENABLE_*=false no longer raises AttributeError (#62) - tester: replace deprecated get_event_loop()/asyncio.wait() with a fresh loop + asyncio.gather to fix 'Passing coroutines is forbidden' on Python 3.10+ (#191, #201) - tester: reuse one aiohttp session with a bounded connector instead of one per proxy, reducing memory growth and fd exhaustion (#119, #155) - tester: catch ClientResponseError/ContentTypeError + return_exceptions so bad httpbin responses no longer raise 'Task exception was never retrieved' (#208) --- proxypool/processors/tester.py | 183 +++++++++++++++++++-------------- proxypool/scheduler.py | 15 +-- proxypool/utils/proxy.py | 2 +- 3 files changed, 114 insertions(+), 86 deletions(-) diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 6937af8c..bdc26b07 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -5,7 +5,8 @@ from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ TEST_DONT_SET_MAX_SCORE -from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError +from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \ + ClientResponseError, ContentTypeError from asyncio import TimeoutError from proxypool.testers import __all__ as testers_cls @@ -16,6 +17,8 @@ ServerDisconnectedError, ClientOSError, ClientHttpProxyError, + ClientResponseError, + ContentTypeError, AssertionError ) @@ -30,83 +33,102 @@ def __init__(self): init redis """ self.redis = RedisClient() - self.loop = asyncio.get_event_loop() self.testers_cls = testers_cls self.testers = [tester_cls() for tester_cls in self.testers_cls] - async def test(self, proxy: Proxy): + async def test(self, proxy: Proxy, session: aiohttp.ClientSession): """ test single proxy :param proxy: Proxy object + :param session: shared aiohttp session :return: """ - async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: - try: - logger.debug(f'testing {proxy.string()}') - # if TEST_ANONYMOUS is True, make sure that - # the proxy has the effect of hiding the real IP - # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') - if TEST_ANONYMOUS: - url = 'https://httpbin.org/ip' - async with session.get(url, timeout=TEST_TIMEOUT) as response: - resp_json = await response.json() - origin_ip = resp_json['origin'] - # logger.debug(f'origin ip is {origin_ip}') - async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: - resp_json = await response.json() - anonymous_ip = resp_json['origin'] - logger.debug(f'anonymous ip is {anonymous_ip}') - assert origin_ip != anonymous_ip - assert proxy.host == anonymous_ip - async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, - allow_redirects=False) as response: - if response.status in TEST_VALID_STATUS: - if TEST_DONT_SET_MAX_SCORE: - logger.debug( - f'proxy {proxy.string()} is valid, remain current score') - else: - self.redis.max(proxy) - logger.debug( - f'proxy {proxy.string()} is valid, set max score') + try: + logger.debug(f'testing {proxy.string()}') + # if TEST_ANONYMOUS is True, make sure that + # the proxy has the effect of hiding the real IP + # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') + if TEST_ANONYMOUS: + url = 'https://httpbin.org/ip' + async with session.get(url, timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + origin_ip = resp_json['origin'] + # logger.debug(f'origin ip is {origin_ip}') + async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response: + resp_json = await response.json() + anonymous_ip = resp_json['origin'] + logger.debug(f'anonymous ip is {anonymous_ip}') + assert origin_ip != anonymous_ip + assert proxy.host == anonymous_ip + async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT, + allow_redirects=False) as response: + if response.status in TEST_VALID_STATUS: + if TEST_DONT_SET_MAX_SCORE: + logger.debug( + f'proxy {proxy.string()} is valid, remain current score') else: - self.redis.decrease(proxy) + self.redis.max(proxy) logger.debug( - f'proxy {proxy.string()} is invalid, decrease score') - # if independent tester class found, create new set of storage and do the extra test - for tester in self.testers: - key = tester.key - if self.redis.exists(proxy, key): - test_url = tester.test_url - headers = tester.headers() - cookies = tester.cookies() - async with session.get(test_url, proxy=f'http://{proxy.string()}', - timeout=TEST_TIMEOUT, - headers=headers, - cookies=cookies, - allow_redirects=False) as response: - resp_text = await response.text() - is_valid = await tester.parse(resp_text, test_url, proxy.string()) - if is_valid: - if tester.test_dont_set_max_score: - logger.info( - f'key[{key}] proxy {proxy.string()} is valid, remain current score') - else: - self.redis.max( - proxy, key, tester.proxy_score_max) - logger.info( - f'key[{key}] proxy {proxy.string()} is valid, set max score') + f'proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease(proxy) + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') + # if independent tester class found, create new set of storage and do the extra test + for tester in self.testers: + key = tester.key + if self.redis.exists(proxy, key): + test_url = tester.test_url + headers = tester.headers() + cookies = tester.cookies() + async with session.get(test_url, proxy=f'http://{proxy.string()}', + timeout=TEST_TIMEOUT, + headers=headers, + cookies=cookies, + allow_redirects=False) as response: + resp_text = await response.text() + is_valid = await tester.parse(resp_text, test_url, proxy.string()) + if is_valid: + if tester.test_dont_set_max_score: + logger.info( + f'key[{key}] proxy {proxy.string()} is valid, remain current score') else: - self.redis.decrease( - proxy, tester.key, tester.proxy_score_min) + self.redis.max( + proxy, key, tester.proxy_score_max) logger.info( - f'key[{key}] proxy {proxy.string()} is invalid, decrease score') + f'key[{key}] proxy {proxy.string()} is valid, set max score') + else: + self.redis.decrease( + proxy, tester.key, tester.proxy_score_min) + logger.info( + f'key[{key}] proxy {proxy.string()} is invalid, decrease score') - except EXCEPTIONS: - self.redis.decrease(proxy) - [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) - for tester in self.testers] + except EXCEPTIONS: + self.redis.decrease(proxy) + [self.redis.decrease(proxy, tester.key, tester.proxy_score_min) + for tester in self.testers] + logger.debug( + f'proxy {proxy.string()} is invalid, decrease score') + + async def run_tests(self): + """ + test all proxies in batches, reusing a single aiohttp session + :return: + """ + count = self.redis.count() + logger.debug(f'{count} proxies to test') + cursor = 0 + connector = aiohttp.TCPConnector(ssl=False, limit=TEST_BATCH) + async with aiohttp.ClientSession(connector=connector) as session: + while True: logger.debug( - f'proxy {proxy.string()} is invalid, decrease score') + f'testing proxies use cursor {cursor}, count {TEST_BATCH}') + cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) + if proxies: + tasks = [self.test(proxy, session) for proxy in proxies] + await asyncio.gather(*tasks, return_exceptions=True) + if not cursor: + break @logger.catch def run(self): @@ -116,26 +138,29 @@ def run(self): """ # event loop of aiohttp logger.info('stating tester...') - count = self.redis.count() - logger.debug(f'{count} proxies to test') - cursor = 0 - while True: - logger.debug( - f'testing proxies use cursor {cursor}, count {TEST_BATCH}') - cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) - if proxies: - tasks = [self.loop.create_task( - self.test(proxy)) for proxy in proxies] - self.loop.run_until_complete(asyncio.wait(tasks)) - if not cursor: - break + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self.run_tests()) + finally: + loop.close() def run_tester(): host = '96.113.165.182' port = '3128' - tasks = [tester.test(Proxy(host=host, port=port))] - tester.loop.run_until_complete(asyncio.wait(tasks)) + tester = Tester() + + async def _test(): + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: + await tester.test(Proxy(host=host, port=port), session) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(_test()) + finally: + loop.close() if __name__ == '__main__': diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py index a2d18abe..d0582268 100644 --- a/proxypool/scheduler.py +++ b/proxypool/scheduler.py @@ -129,12 +129,15 @@ def run(self): tester_process and tester_process.join() getter_process and getter_process.join() server_process and server_process.join() - logger.info( - f'tester is {"alive" if tester_process.is_alive() else "dead"}') - logger.info( - f'getter is {"alive" if getter_process.is_alive() else "dead"}') - logger.info( - f'server is {"alive" if server_process.is_alive() else "dead"}') + if tester_process: + logger.info( + f'tester is {"alive" if tester_process.is_alive() else "dead"}') + if getter_process: + logger.info( + f'getter is {"alive" if getter_process.is_alive() else "dead"}') + if server_process: + logger.info( + f'server is {"alive" if server_process.is_alive() else "dead"}') logger.info('proxy terminated') diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py index 5330ceb4..ed81ef8b 100644 --- a/proxypool/utils/proxy.py +++ b/proxypool/utils/proxy.py @@ -63,7 +63,7 @@ def convert_proxy_or_proxies(data): if is_auth_proxy(data): host, port = extract_auth_proxy(data) else: - host, port = data.split(':') + host, port, *_ = data.split(':') return Proxy(host=host, port=int(port)) From c355a413455e24538f78cc0ac14fd3d7732acadd Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 4 Jul 2026 16:28:08 +0800 Subject: [PATCH 136/139] chore: modernize dependencies and container runtime (#230) - requirements: move to Flask 2.x and drop the ancient itsdangerous==0.24 / MarkupSafe<2.1 pins that blocked installs on Python 3.10+; bump redis to 4/5, aiohttp>=3.9, lxml<6, gevent<25; add explicit Werkzeug<3 bound - Dockerfile: base image python:3.7-alpine (EOL) -> python:3.11-slim so gevent/lxml/aiohttp install from wheels instead of compiling under musl, fixing common build failures and enabling arm64 (e.g. Raspberry Pi) Refs #167, #183, #192, #178, #149, #158 --- Dockerfile | 14 +++++++------- requirements.txt | 17 ++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5ca5440..0de17573 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM python:3.7-alpine AS build +FROM python:3.11-slim AS build COPY requirements.txt . -RUN apk update &&\ - apk add --no-cache gcc g++ libffi-dev openssl-dev libxml2-dev libxslt-dev build-base musl-dev &&\ +RUN apt-get update &&\ + apt-get install -y --no-install-recommends gcc g++ libxml2-dev libxslt1-dev &&\ pip install -U pip &&\ - pip install --timeout 30 --user --no-cache-dir --no-warn-script-location -r requirements.txt + pip install --timeout 60 --user --no-cache-dir --no-warn-script-location -r requirements.txt &&\ + rm -rf /var/lib/apt/lists/* -FROM python:3.7-alpine +FROM python:3.11-slim ENV APP_ENV=prod ENV LOCAL_PKG="/root/.local" COPY --from=build ${LOCAL_PKG} ${LOCAL_PKG} -RUN apk update && apk add --no-cache libffi-dev openssl-dev libxslt-dev &&\ - ln -sf ${LOCAL_PKG}/bin/* /usr/local/bin/ +RUN ln -sf ${LOCAL_PKG}/bin/* /usr/local/bin/ WORKDIR /app COPY . . EXPOSE 5555 diff --git a/requirements.txt b/requirements.txt index 33f35c50..49828461 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,16 @@ -environs>=9.3.0,<10.0.0 -Flask>=1.1.2,<2.0.0 -attrs>=20.3.0,<21.0.0 +environs>=9.3.0,<12.0.0 +Flask>=2.0.0,<3.0.0 +Werkzeug>=2.0.0,<3.0.0 +attrs>=20.3.0,<24.0.0 retrying>=1.3.3,<2.0.0 -aiohttp>=3.8.1,<4.0.0 +aiohttp>=3.9.0,<4.0.0 requests>=2.25.1,<3.0.0 loguru>=0.5.3,<1.0.0 pyquery>=1.4.3,<2.0.0 supervisor>=4.2.1,<5.0.0 -redis>=3.5.3,<4.0.0 -lxml>=4.6.5,<5.0.0 +redis>=4.3.0,<6.0.0 +lxml>=4.6.5,<6.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.8.0,<24.0.0 +gevent>=22.10.2,<25.0.0 tornado>=6.0,<7.0 -itsdangerous==0.24 -MarkupSafe<2.1.0 From 40a326a8d7d383d26e68382f699846bc995716c9 Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 4 Jul 2026 16:44:39 +0800 Subject: [PATCH 137/139] security: validate key query param and fix /count return type (#231) - server: validate the key query parameter against a strict charset (letters, digits, _ : -, max 64) so the API cannot be used to probe arbitrary redis keys; invalid keys return 400 - server: /count without a key returned a bare int which Flask rejects; always return a string - README: document API_KEY and add a security note about exposing the pool publicly without authentication Refs #228 --- README.md | 3 +++ proxypool/processors/server.py | 27 ++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 474bf65b..a25d705f 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,9 @@ get random proxy 116.196.115.209:8080 - API_HOST：代理 Server 运行 Host，默认 0.0.0.0 - API_PORT：代理 Server 运行端口，默认 5555 - API_THREADED：代理 Server 是否使用多线程，默认 true +- API_KEY：API 访问鉴权密钥，默认空（即不鉴权）。设置后，调用 `/random`、`/all`、`/count` 需在请求头携带 `API-KEY`，详见下方「安全性」说明 + +> ⚠️ 安全提示：代理 Server 默认监听 `0.0.0.0` 且 `API_KEY` 默认为空，任何能访问该端口的人都可以调用 `/random`、`/all`、`/count`。如果将代理池**暴露到公网**，请务必设置 `API_KEY`，并配合防火墙/安全组限制来源。`key` 查询参数已做格式校验，仅允许字母、数字及 `_ : -`，最长 64 位。 ### 日志 diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index fb50ce5e..02241709 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -1,5 +1,6 @@ import hmac -from flask import Flask, g, request +import re +from flask import Flask, g, request, abort from proxypool.exceptions import PoolEmptyException from proxypool.storages.redis import RedisClient from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED @@ -11,6 +12,10 @@ if IS_DEV: app.debug = True +# allowed characters for the `key` query parameter that selects a redis sub-pool; +# restricts to a safe charset to avoid probing arbitrary redis keys via the API +VALID_KEY_PATTERN = re.compile(r'^[a-zA-Z0-9_:\-]{1,64}$') + def auth_required(func): @functools.wraps(func) @@ -41,6 +46,18 @@ def get_conn(): return g.redis +def get_request_key(): + """ + read the `key` query parameter and validate its format; + reject unexpected characters to avoid redis key probing/injection + :return: validated key or None + """ + key = request.args.get('key') + if key and not VALID_KEY_PATTERN.match(key): + abort(400, description='invalid key parameter') + return key + + @app.route('/') @auth_required def index(): @@ -59,7 +76,7 @@ def get_proxy(): if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool :return: get a random proxy """ - key = request.args.get('key') + key = get_request_key() conn = get_conn() # return conn.random(key).string() if key else conn.random().string() if key: @@ -78,7 +95,7 @@ def get_proxy_all(): get a random proxy :return: get a random proxy """ - key = request.args.get('key') + key = get_request_key() conn = get_conn() proxies = conn.all(key) if key else conn.all() @@ -98,8 +115,8 @@ def get_count(): :return: count, int """ conn = get_conn() - key = request.args.get('key') - return str(conn.count(key)) if key else conn.count() + key = get_request_key() + return str(conn.count(key)) if key else str(conn.count()) if __name__ == '__main__': From b915026bab0b7e706d0c730d75a10fb1c223b59f Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 4 Jul 2026 16:48:27 +0800 Subject: [PATCH 138/139] feat: filter proxies by country via ?area=CN (#84, #177) (#233) * feat: support fetching multiple proxies via /random?count=N - redis: add RedisClient.randoms(count, ...) which prefers max-score proxies and falls back to rank, returning a de-duplicated random sample - server: /random accepts a count parameter and returns N random proxies (one per line); count<=1 keeps the original single-proxy behavior; works with key too - README: document the new count parameter Closes #194 * feat: filter proxies by country via ?area=CN on /random and /all - utils/geo: add get_country_iso() backed by the bundled GeoLite2 db (previously an unused dependency); fails safe to None if unavailable - server: /random and /all accept an area parameter (country iso code) to return only proxies from that country; combines with count and key - README: document the area parameter Closes #84, Closes #177 --- README.md | 21 +++++++++++++++ proxypool/processors/server.py | 48 ++++++++++++++++++++++++++++++++-- proxypool/storages/redis.py | 23 +++++++++++++++- proxypool/utils/geo.py | 35 +++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 proxypool/utils/geo.py diff --git a/README.md b/README.md index a25d705f..e3268962 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,27 @@ get random proxy 116.196.115.209:8080 可以看到成功获取了代理，并请求 httpbin.org 验证了代理的可用性。 +### 获取多个代理 + +如果一次需要多个代理，可以给 `/random` 接口传入 `count` 参数，一次返回多个随机代理（每行一个）： + +``` +GET http://localhost:5555/random?count=5 +``` + +`count` 不传或为 1 时行为不变，仍返回单个代理；`count` 大于可用数量时返回全部可用代理。也可与 `key` 参数组合使用。 + +### 按地区（国家）筛选代理 + +可以给 `/random` 和 `/all` 接口传入 `area` 参数，按代理 IP 所属国家筛选（ISO 国家码，大小写不敏感），例如只获取国内（中国）代理： + +``` +GET http://localhost:5555/random?area=CN +GET http://localhost:5555/all?area=CN +``` + +国家信息由内置的 GeoLite2 离线库解析，无法解析归属地的代理会被排除。`area` 可与 `count`、`key` 参数组合使用。 + ## 可配置项 代理池可以通过设置环境变量来配置一些参数。 diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py index 02241709..f95f21dc 100644 --- a/proxypool/processors/server.py +++ b/proxypool/processors/server.py @@ -5,6 +5,8 @@ from proxypool.storages.redis import RedisClient from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED import functools +from random import choice, sample +from proxypool.utils.geo import get_country_iso __all__ = ['app'] @@ -58,6 +60,20 @@ def get_request_key(): return key +def filter_proxies_by_area(proxies, area): + """ + filter proxies by country iso code (e.g. 'CN', 'US'), case-insensitive; + proxies whose country cannot be resolved are excluded + :param proxies: list of Proxy + :param area: country iso code, or falsy to skip filtering + :return: filtered list of Proxy + """ + if not area: + return proxies + area = area.upper() + return [proxy for proxy in proxies if get_country_iso(proxy.host) == area] + + @app.route('/') @auth_required def index(): @@ -74,11 +90,37 @@ def get_proxy(): """ get a random proxy, can query the specific sub-pool according the (redis) key if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool + can pass a `count` parameter to get multiple random proxies at once + can pass an `area` parameter to only get proxies from a country (iso code, e.g. CN) :return: get a random proxy """ key = get_request_key() + count = request.args.get('count', type=int) + area = request.args.get('area') conn = get_conn() # return conn.random(key).string() if key else conn.random().string() + if area: + # area filtering needs the candidate set first, then filter by country + candidates = conn.all(key) if key else conn.all() + candidates = filter_proxies_by_area(candidates, area) + if not candidates and key and PROXY_RAND_KEY_DEGRADED: + candidates = filter_proxies_by_area(conn.all(), area) + if not candidates: + raise PoolEmptyException + if count and count > 1: + count = min(count, len(candidates)) + return '\n'.join(proxy.string() for proxy in sample(candidates, count)) + return choice(candidates).string() + if count and count > 1: + # return multiple random proxies, one per line + try: + proxies = conn.randoms(count, key) if key else conn.randoms(count) + except PoolEmptyException: + if key and PROXY_RAND_KEY_DEGRADED: + proxies = conn.randoms(count) + else: + raise + return '\n'.join(proxy.string() for proxy in proxies) if key: try: return conn.random(key).string() @@ -92,13 +134,15 @@ def get_proxy(): @auth_required def get_proxy_all(): """ - get a random proxy - :return: get a random proxy + get all proxies, optionally filtered by `area` (country iso code, e.g. CN) + :return: all proxies """ key = get_request_key() + area = request.args.get('area') conn = get_conn() proxies = conn.all(key) if key else conn.all() + proxies = filter_proxies_by_area(proxies, area) proxies_string = '' if proxies: for proxy in proxies: diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py index 2d052323..aed45029 100644 --- a/proxypool/storages/redis.py +++ b/proxypool/storages/redis.py @@ -3,7 +3,7 @@ from proxypool.schemas.proxy import Proxy from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \ PROXY_SCORE_INIT -from random import choice +from random import choice, sample from typing import List from loguru import logger from proxypool.utils.proxy import is_valid_proxy, convert_proxy_or_proxies @@ -70,6 +70,27 @@ def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_sco # else raise error raise PoolEmptyException + def randoms(self, count, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: + """ + get a batch of random proxies + firstly try to get proxies with max score, + if not enough, get proxies by rank (score from high to low) + if none exists, raise error + :param count: number of proxies to return + :return: list of proxies + """ + # try to get proxies with max score first + proxies = self.db.zrangebyscore( + redis_key, proxy_score_max, proxy_score_max) + if len(proxies) < count: + # not enough max-score proxies, fall back to all proxies by rank + proxies = self.db.zrevrangebyscore( + redis_key, proxy_score_max, proxy_score_min) + if not proxies: + raise PoolEmptyException + count = min(count, len(proxies)) + return convert_proxy_or_proxies(sample(proxies, count)) + def decrease(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN) -> int: """ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it diff --git a/proxypool/utils/geo.py b/proxypool/utils/geo.py new file mode 100644 index 00000000..82a105a0 --- /dev/null +++ b/proxypool/utils/geo.py @@ -0,0 +1,35 @@ +from loguru import logger + +# geolite2 provides an offline IP -> country database (bundled with the +# maxminddb_geolite2 dependency). loading it can fail if the optional +# dependency is missing, so degrade gracefully and disable area filtering. +try: + from geolite2 import geolite2 + + _reader = geolite2.reader() +except Exception as e: # pragma: no cover + _reader = None + logger.warning(f'geolite2 is unavailable, area filtering disabled: {e}') + + +def get_country_iso(ip): + """ + look up the ISO country code (e.g. 'CN', 'US') for an ip address + :param ip: ip address string + :return: uppercase iso code, or None if unknown/unavailable + """ + if _reader is None: + return None + try: + record = _reader.get(ip) + except Exception: + return None + if not record: + return None + country = record.get('country') or record.get('registered_country') or {} + return country.get('iso_code') + + +if __name__ == '__main__': + print('8.8.8.8', get_country_iso('8.8.8.8')) + print('114.114.114.114', get_country_iso('114.114.114.114')) From cabcd96cc9f30d7bdbc872bb8a8c52760023c142 Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 4 Jul 2026 16:48:36 +0800 Subject: [PATCH 139/139] feat: make the tester anonymity-check URL configurable via TEST_ANONYMOUS_URL (#234) - setting: add TEST_ANONYMOUS_URL (default https://httpbin.org/ip) - tester: use TEST_ANONYMOUS_URL instead of the hardcoded httpbin.org endpoint so users can self-host httpbin and avoid public rate limits - README: document TEST_ANONYMOUS_URL and TEST_ANONYMOUS Closes #109 --- README.md | 2 ++ proxypool/processors/tester.py | 4 ++-- proxypool/setting.py | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e3268962..4b6d4694 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ GET http://localhost:5555/all?area=CN - TEST_TIMEOUT：测试超时时间，默认 10 秒 - TEST_BATCH：批量测试数量，默认 20 个代理 - TEST_VALID_STATUS：测试有效的状态码 +- TEST_ANONYMOUS：是否只保留匿名代理，默认 true +- TEST_ANONYMOUS_URL：匿名 / 出口 IP 检测地址，默认 `https://httpbin.org/ip`，需返回 httpbin 格式的 JSON（`{"origin": "1.2.3.4"}`）。可指向自建 httpbin 服务以避免公共服务限流 - API_HOST：代理 Server 运行 Host，默认 0.0.0.0 - API_PORT：代理 Server 运行端口，默认 5555 - API_THREADED：代理 Server 是否使用多线程，默认 true diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index bdc26b07..5d70d238 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -4,7 +4,7 @@ from proxypool.schemas import Proxy from proxypool.storages.redis import RedisClient from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \ - TEST_DONT_SET_MAX_SCORE + TEST_DONT_SET_MAX_SCORE, TEST_ANONYMOUS_URL from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \ ClientResponseError, ContentTypeError from asyncio import TimeoutError @@ -49,7 +49,7 @@ async def test(self, proxy: Proxy, session: aiohttp.ClientSession): # the proxy has the effect of hiding the real IP # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}') if TEST_ANONYMOUS: - url = 'https://httpbin.org/ip' + url = TEST_ANONYMOUS_URL async with session.get(url, timeout=TEST_TIMEOUT) as response: resp_json = await response.json() origin_ip = resp_json['origin'] diff --git a/proxypool/setting.py b/proxypool/setting.py index a445667e..4d1cec56 100644 --- a/proxypool/setting.py +++ b/proxypool/setting.py @@ -75,6 +75,10 @@ TEST_BATCH = env.int('TEST_BATCH', 20) # only save anonymous proxy TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True) +# the url used to check the proxy anonymity and its exit ip; +# must return json like httpbin.org/ip ({"origin": "1.2.3.4"}); +# point this to a self-hosted httpbin to avoid public rate limits +TEST_ANONYMOUS_URL = env.str('TEST_ANONYMOUS_URL', 'https://httpbin.org/ip') # TEST_HEADERS = env.json('TEST_HEADERS', { # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', # })