From ac8842504f488d6139e10370ecbbf3806be399e8 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 16 Apr 2018 15:56:44 +0800 Subject: [PATCH 001/354] [update]jsonify --- Api/ProxyApi.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index 45db4843a..724dc35e6 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -14,16 +14,29 @@ __author__ = 'JHao' import sys +from werkzeug.wrappers import Response +from flask import Flask, jsonify, request sys.path.append('../') -from flask import Flask, jsonify, request from Util.GetConfig import GetConfig - from Manager.ProxyManager import ProxyManager app = Flask(__name__) + +class JsonResponse(Response): + + @classmethod + def force_type(cls, response, environ=None): + if isinstance(response, (dict, list)): + response = jsonify(response) + + return super(JsonResponse, cls).force_type(response, environ) + + +app.response_class = JsonResponse + api_list = { 'get': u'get an usable proxy', # 'refresh': u'refresh proxy pool', @@ -35,7 +48,7 @@ @app.route('/') def index(): - return jsonify(api_list) + return api_list @app.route('/get/') @@ -55,7 +68,7 @@ def refresh(): @app.route('/get_all/') def getAll(): proxies = ProxyManager().getAll() - return jsonify(proxies) + return proxies @app.route('/delete/', methods=['GET']) @@ -68,7 +81,7 @@ def delete(): @app.route('/get_status/') def getStatus(): status = ProxyManager().getNumber() - return jsonify(status) + return status def run(): From 34ef9cedb989feeb3ba1019596706c897c95a965 Mon Sep 17 00:00:00 2001 From: luocaodan Date: Fri, 27 Apr 2018 10:21:51 +0800 Subject: [PATCH 002/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E7=BD=91=E7=AB=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 5 +- ProxyGetter/getFreeProxy.py | 235 +++++++++++++++++++++++++++--------- Util/WebRequest.py | 20 ++- requirements.txt | 10 +- 4 files changed, 203 insertions(+), 67 deletions(-) diff --git a/Config.ini b/Config.ini index 14e785427..763007e23 100644 --- a/Config.ini +++ b/Config.ini @@ -17,9 +17,10 @@ freeProxyFifth = 1 freeProxySixth = 1 freeProxySeventh = 1 freeProxyEight = 1 +freeProxyNinth = 1 ;foreign website, outside the wall -;freeProxyWallFirst = 1 -;freeProxyWallSecond = 1 +freeProxyWallFirst = 1 +freeProxyWallSecond = 1 [HOST] ; API接口配置 http://127.0.0.1:5010 diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index ff9ee8197..4cd08ced5 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -21,7 +21,7 @@ reload(sys) sys.setdefaultencoding('utf-8') -sys.path.append('../') +sys.path.append('..') from Util.utilFunction import robustCrawl, getHtmlTree from Util.WebRequest import WebRequest @@ -32,14 +32,13 @@ """ 66ip.cn data5u.com - ip181.com xicidaili.com goubanjia.com xdaili.cn kuaidaili.com cn-proxy.com proxy-list.org - www.mimiip.com + www.mimiip.com to do """ @@ -55,13 +54,15 @@ def __init__(self): def freeProxyFirst(page=10): """ 抓取无忧代理 http://www.data5u.com/ + 几乎没有能用的 :param page: 页数 :return: """ - url_list = ['http://www.data5u.com/', - 'http://www.data5u.com/free/', - 'http://www.data5u.com/free/gngn/index.shtml', - 'http://www.data5u.com/free/gnpt/index.shtml'] + url_list = [ + 'http://www.data5u.com/', + 'http://www.data5u.com/free/gngn/index.shtml', + 'http://www.data5u.com/free/gnpt/index.shtml' + ] for url in url_list: html_tree = getHtmlTree(url) ul_list = html_tree.xpath('//ul[@class="l2"]') @@ -69,10 +70,10 @@ def freeProxyFirst(page=10): try: yield ':'.join(ul.xpath('.//li/text()')[0:2]) except Exception as e: - pass + print(e) @staticmethod - def freeProxySecond(proxy_number=100): + def deprecatedFreeProxySecond(proxy_number=100): """ 抓取代理66 http://www.66ip.cn/ :param proxy_number: 代理数量 @@ -85,6 +86,29 @@ def freeProxySecond(proxy_number=100): for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html): yield proxy + @staticmethod + def freeProxySecond(area=33): + """ + 修改抓取代理66 http://www.66ip.cn/ + :param page:抓取代理页数，page=1北京代理页，page=2上海代理页...... + :return: + """ + if area > 33: + page = 33 + for area_index in range(1, area + 1): + page_count = 5 + for i in range(1, page_count + 1): + url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i) + html_tree = getHtmlTree(url) + tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]") + if len(tr_list) == 0: + continue + for tr in tr_list: + yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0] + break + + ''' + 不能用了 @staticmethod def freeProxyThird(days=1): """ @@ -100,24 +124,28 @@ def freeProxyThird(days=1): yield ':'.join(tr.xpath('./td/text()')[0:2]) except Exception as e: pass + ''' @staticmethod - def freeProxyFourth(): + def freeProxyFourth(page_count=2): """ 抓取西刺代理 http://api.xicidaili.com/free2016.txt :return: """ - url_list = ['http://www.xicidaili.com/nn', # 高匿 - 'http://www.xicidaili.com/nt', # 透明 - ] + url_list = [ + 'http://www.xicidaili.com/nn/', # 高匿 + 'http://www.xicidaili.com/nt/', # 透明 + ] for each_url in url_list: - tree = getHtmlTree(each_url) - proxy_list = tree.xpath('.//table[@id="ip_list"]//tr') - for proxy in proxy_list: - try: - yield ':'.join(proxy.xpath('./td/text()')[0:2]) - except Exception as e: - pass + for i in range(1, page_count + 1): + page_url = each_url + str(i) + tree = getHtmlTree(page_url) + proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]') + for proxy in proxy_list: + try: + yield ':'.join(proxy.xpath('./td/text()')[0:2]) + except Exception as e: + pass @staticmethod def freeProxyFifth(): @@ -164,13 +192,17 @@ def freeProxySeventh(): """ 快代理免费https://www.kuaidaili.com/free/inha/1/ """ - url = 'https://www.kuaidaili.com/free/inha/{page}/' - for page in range(1, 10): - page_url = url.format(page=page) - tree = getHtmlTree(page_url) - proxy_list = tree.xpath('.//table//tr') - for tr in proxy_list[1:]: - yield ':'.join(tr.xpath('./td/text()')[0:2]) + url_list = [ + 'https://www.kuaidaili.com/free/inha/{page}/', + 'https://www.kuaidaili.com/free/intr/{page}/' + ] + for url in url_list: + for page in range(1, 5): + page_url = url.format(page=page) + tree = getHtmlTree(page_url) + proxy_list = tree.xpath('.//table//tr') + for tr in proxy_list[1:]: + yield ':'.join(tr.xpath('./td/text()')[0:2]) @staticmethod def freeProxyEight(): @@ -184,15 +216,33 @@ def freeProxyEight(): request = WebRequest() for url in url_list: - r = request.get(url) + r = request.get(url, use_proxy=True) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W].*(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) + + @staticmethod + def freeProxyNinth(): + """ + coderBusy + https://proxy.coderbusy.com/ + :return: + """ + urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] + request = WebRequest() + for url in urls: + r = request.get(url) + proxies = re.findall('data-ip="(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})".+?>(\d+)', r.text) + for proxy in proxies: + yield ':'.join(proxy) + + @staticmethod def freeProxyWallFirst(): """ 墙外网站 cn-proxy + 并没有被墙 :return: """ urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] @@ -205,6 +255,10 @@ def freeProxyWallFirst(): @staticmethod def freeProxyWallSecond(): + ''' + 并没有被墙 + :return: + ''' urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)] request = WebRequest() import base64 @@ -215,34 +269,101 @@ def freeProxyWallSecond(): yield base64.b64decode(proxy).decode() + @staticmethod + def freeProxyWallThird(): + urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1'] + request = WebRequest() + for url in urls: + r = request.get(url) + proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) + for proxy in proxies: + yield ':'.join(proxy) + + +import threading + +lock = threading.Lock() +success = 0 +total = 0 + + +def test_once(proxy): + ip_port = proxy.split(":") + ip = ip_port[0] + port = ip_port[1] + import requests + + req_url = "http://www.baidu.com" + proxies = { + "http": "http://%s:%s" % (ip, port), + "https": "https://%s:%s" % (ip, port) + } + + global total + + try: + response = requests.get(req_url, proxies=proxies, timeout=4) + if response.status_code != 200: + print("unknow error, status code:" + str(response.status_code)) + lock.acquire() + total += 1 + lock.release() + return 0 + print("success") + global success + lock.acquire() + success += 1 + total += 1 + lock.release() + return 1 + except requests.exceptions.Timeout: + print("timeout") + except requests.exceptions.ConnectionError: + print("poxy unusable") + except Exception: + print("request error") + + lock.acquire() + total += 1 + lock.release() + return 0 + + +def test_batch(iterator): + global success + global total + + for proxy in iterator: + t = threading.Thread(target=test_once, args=(proxy,)) + t.start() + t.join() + + print("success:" + str(success) + "\ttotal:" + str(total)) + + if __name__ == '__main__': gg = GetFreeProxy() - # for e in gg.freeProxyFirst(): - # print(e) - # - # for e in gg.freeProxySecond(): - # print(e) - # - # for e in gg.freeProxyThird(): - # print(e) - # - # for e in gg.freeProxyFourth(): - # print(e) - # - # for e in gg.freeProxyFifth(): - # print(e) - # - # for e in gg.freeProxySixth(): - # print(e) - # - # for e in gg.freeProxySeventh(): - # print(e) - # - # for e in gg.freeProxyEight(): - # print(e) - # - # for e in gg.freeProxyWallFirst(): - # print(e) - # - # for e in gg.freeProxyWallSecond(): - # print(e) + + # test_batch(gg.freeProxyFirst()) + + # test_batch(gg.freeProxySecond()) + + # test_batch(gg.freeProxyFourth()) + + # test_batch(gg.freeProxyFifth()) + + # test_batch(gg.freeProxySixth()) + + # test_batch(gg.freeProxySeventh()) + + # to do + test_batch(gg.freeProxyEight()) + # gg.freeProxyEight() + + # test_batch(gg.freeProxyNinth()) + + # test_batch(gg.freeProxyWallFirst()) + + # test_batch(gg.freeProxyWallSecond()) + + # test_batch(gg.freeProxyWallThird()) diff --git a/Util/WebRequest.py b/Util/WebRequest.py index abbdb17be..7f5011724 100644 --- a/Util/WebRequest.py +++ b/Util/WebRequest.py @@ -15,6 +15,7 @@ import requests import random import time +from requests.models import Response class WebRequest(object): @@ -51,7 +52,7 @@ def header(self): 'Accept-Language': 'zh-CN,zh;q=0.8'} def get(self, url, header=None, retry_time=5, timeout=30, - retry_flag=list(), retry_interval=5, *args, **kwargs): + retry_flag=list(), retry_interval=5, use_proxy=False, *args, **kwargs): """ get method :param url: target url @@ -60,6 +61,7 @@ def get(self, url, header=None, retry_time=5, timeout=30, :param timeout: network timeout :param retry_flag: if retry_flag in content. do retry :param retry_interval: retry interval(second) + :param use_proxy: 是否使用代理 :param args: :param kwargs: :return: @@ -69,7 +71,16 @@ def get(self, url, header=None, retry_time=5, timeout=30, headers.update(header) while True: try: - html = requests.get(url, headers=headers, timeout=timeout) + if use_proxy: + proxy_url = "http://127.0.0.1:5010/get" + ip_proxy = requests.get(proxy_url).text + proxies = { + "http": "http://" + ip_proxy, + "https": "https://" + ip_proxy + } + html = requests.get(url, headers=headers, timeout=timeout, proxies=proxies) + else: + html = requests.get(url, headers=headers, timeout=timeout) if any(f in html.content for f in retry_flag): raise Exception return html @@ -78,5 +89,8 @@ def get(self, url, header=None, retry_time=5, timeout=30, retry_time -= 1 if retry_time <= 0: # 多次请求失败时，返回百度页面 - return requests.get("https://www.baidu.com/") + resp = Response() + resp.status_code = 200 + return resp time.sleep(retry_interval) + diff --git a/requirements.txt b/requirements.txt index 698cc8197..c0db31b10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ APScheduler==3.2.0 -Flask==0.11.1 -requests==2.11.0 -lxml==3.7.1 +Flask +requests +lxml -pymongo==3.2.2 -redis==2.10.5 +pymongo +redis From de58a7e81fc9f7f33b8437f674c4bbd7245b2391 Mon Sep 17 00:00:00 2001 From: luocaodan Date: Fri, 27 Apr 2018 11:43:07 +0800 Subject: [PATCH 003/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E7=BD=91=E7=AB=99=20http://www.ip3366.net/fr?= =?UTF-8?q?ee/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 1 + ProxyGetter/getFreeProxy.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Config.ini b/Config.ini index 763007e23..322aaddd8 100644 --- a/Config.ini +++ b/Config.ini @@ -18,6 +18,7 @@ freeProxySixth = 1 freeProxySeventh = 1 freeProxyEight = 1 freeProxyNinth = 1 +freeProxyTen = 1 ;foreign website, outside the wall freeProxyWallFirst = 1 freeProxyWallSecond = 1 diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 4cd08ced5..1330d88fc 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -238,6 +238,17 @@ def freeProxyNinth(): yield ':'.join(proxy) + @staticmethod + def freeProxyTen(): + urls = ['http://www.ip3366.net/free/'] + request = WebRequest() + for url in urls: + r = request.get(url) + proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) + for proxy in proxies: + yield ":".join(proxy) + + @staticmethod def freeProxyWallFirst(): """ @@ -356,12 +367,12 @@ def test_batch(iterator): # test_batch(gg.freeProxySeventh()) - # to do - test_batch(gg.freeProxyEight()) - # gg.freeProxyEight() + # test_batch(gg.freeProxyEight()) # test_batch(gg.freeProxyNinth()) + # test_batch(gg.freeProxyTen()) + # test_batch(gg.freeProxyWallFirst()) # test_batch(gg.freeProxyWallSecond()) From e2433d39fc8f5802fedfbc6d6e8e9fc9e1ff7b15 Mon Sep 17 00:00:00 2001 From: luocaodan Date: Fri, 27 Apr 2018 12:41:15 +0800 Subject: [PATCH 004/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E7=BD=91=E7=AB=99=20http://www.iphai.com?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 2 ++ ProxyGetter/getFreeProxy.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Config.ini b/Config.ini index 322aaddd8..5f417badc 100644 --- a/Config.ini +++ b/Config.ini @@ -19,9 +19,11 @@ freeProxySeventh = 1 freeProxyEight = 1 freeProxyNinth = 1 freeProxyTen = 1 +freeProxyEleven = 1 ;foreign website, outside the wall freeProxyWallFirst = 1 freeProxyWallSecond = 1 +freeProxyWallThird = 1 [HOST] ; API接口配置 http://127.0.0.1:5010 diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 1330d88fc..b9f542b01 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -249,6 +249,22 @@ def freeProxyTen(): yield ":".join(proxy) + @staticmethod + def freeProxyEleven(): + urls = [ + 'http://www.iphai.com/free/ng', + 'http://www.iphai.com/free/np', + 'http://www.iphai.com/free/wg', + 'http://www.iphai.com/free/wp' + ] + request = WebRequest() + for url in urls: + r = request.get(url) + proxies = re.findall(r'\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?[\s\S]*?\s*?(\d+)\s*?', r.text) + for proxy in proxies: + yield ":".join(proxy) + + @staticmethod def freeProxyWallFirst(): """ @@ -373,6 +389,8 @@ def test_batch(iterator): # test_batch(gg.freeProxyTen()) + # test_batch(gg.freeProxyEleven()) + # test_batch(gg.freeProxyWallFirst()) # test_batch(gg.freeProxyWallSecond()) From 73afc225356d0577c35b6f2d127bf53f1d4ec3b1 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Fri, 27 Apr 2018 16:08:59 +0800 Subject: [PATCH 005/354] merge luocandan's code --- DB/SsdbClient.py | 4 + ProxyGetter/getFreeProxy.py | 131 ++++++-------------------- Test/.pytest_cache/v/cache/lastfailed | 3 + Test/.pytest_cache/v/cache/nodeids | 3 + Util/WebRequest.py | 21 +---- requirements.txt | 7 +- 6 files changed, 46 insertions(+), 123 deletions(-) create mode 100644 Test/.pytest_cache/v/cache/lastfailed create mode 100644 Test/.pytest_cache/v/cache/nodeids diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py index d9a4030f4..2522e0071 100644 --- a/DB/SsdbClient.py +++ b/DB/SsdbClient.py @@ -110,3 +110,7 @@ def getNumber(self): def changeTable(self, name): self.name = name + +if __name__ == '__main__': + c = SsdbClient('useful_proxy', '118.24.52.95', 8899) + print(c.getAll()) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index b9f542b01..78837d50a 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -53,7 +53,7 @@ def __init__(self): @staticmethod def freeProxyFirst(page=10): """ - 抓取无忧代理 http://www.data5u.com/ + 无忧代理 http://www.data5u.com/ 几乎没有能用的 :param page: 页数 :return: @@ -73,31 +73,16 @@ def freeProxyFirst(page=10): print(e) @staticmethod - def deprecatedFreeProxySecond(proxy_number=100): + def freeProxySecond(area=33, page=1): """ - 抓取代理66 http://www.66ip.cn/ - :param proxy_number: 代理数量 + 代理66 http://www.66ip.cn/ + :param area: 抓取代理页数，page=1北京代理页，page=2上海代理页...... + :param page: 翻页 :return: """ - url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format( - proxy_number) - request = WebRequest() - html = request.get(url).text - for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html): - yield proxy - - @staticmethod - def freeProxySecond(area=33): - """ - 修改抓取代理66 http://www.66ip.cn/ - :param page:抓取代理页数，page=1北京代理页，page=2上海代理页...... - :return: - """ - if area > 33: - page = 33 + area = 33 if area > 33 else area for area_index in range(1, area + 1): - page_count = 5 - for i in range(1, page_count + 1): + for i in range(1, page + 1): url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i) html_tree = getHtmlTree(url) tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]") @@ -107,12 +92,10 @@ def freeProxySecond(area=33): yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0] break - ''' - 不能用了 @staticmethod def freeProxyThird(days=1): """ - 抓取ip181 http://www.ip181.com/ + ip181 http://www.ip181.com/ 不能用了 :param days: :return: """ @@ -124,12 +107,11 @@ def freeProxyThird(days=1): yield ':'.join(tr.xpath('./td/text()')[0:2]) except Exception as e: pass - ''' @staticmethod def freeProxyFourth(page_count=2): """ - 抓取西刺代理 http://api.xicidaili.com/free2016.txt + 西刺代理 http://www.xicidaili.com :return: """ url_list = [ @@ -150,7 +132,7 @@ def freeProxyFourth(page_count=2): @staticmethod def freeProxyFifth(): """ - 抓取guobanjia http://www.goubanjia.com/ + guobanjia http://www.goubanjia.com/ :return: """ url = "http://www.goubanjia.com/" @@ -175,7 +157,7 @@ def freeProxyFifth(): @staticmethod def freeProxySixth(): """ - 抓取讯代理免费proxy http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10 + 讯代理 http://www.xdaili.cn/ :return: """ url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10' @@ -190,7 +172,7 @@ def freeProxySixth(): @staticmethod def freeProxySeventh(): """ - 快代理免费https://www.kuaidaili.com/free/inha/1/ + 快代理 https://www.kuaidaili.com """ url_list = [ 'https://www.kuaidaili.com/free/inha/{page}/', @@ -207,7 +189,7 @@ def freeProxySeventh(): @staticmethod def freeProxyEight(): """ - 秘密代理IP网站http://www.mimiip.com + 秘密代理 http://www.mimiip.com """ url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)] # 国内高匿 url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 10)] # 国内普匿 @@ -221,12 +203,10 @@ def freeProxyEight(): for proxy in proxies: yield ':'.join(proxy) - @staticmethod def freeProxyNinth(): """ - coderBusy - https://proxy.coderbusy.com/ + 码农代理 https://proxy.coderbusy.com/ :return: """ urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] @@ -237,9 +217,12 @@ def freeProxyNinth(): for proxy in proxies: yield ':'.join(proxy) - @staticmethod def freeProxyTen(): + """ + 云代理 http://www.ip3366.net/free/ + :return: + """ urls = ['http://www.ip3366.net/free/'] request = WebRequest() for url in urls: @@ -248,9 +231,12 @@ def freeProxyTen(): for proxy in proxies: yield ":".join(proxy) - @staticmethod def freeProxyEleven(): + """ + IP海 http://www.iphai.com/free/ng + :return: + """ urls = [ 'http://www.iphai.com/free/ng', 'http://www.iphai.com/free/np', @@ -260,16 +246,15 @@ def freeProxyEleven(): request = WebRequest() for url in urls: r = request.get(url) - proxies = re.findall(r'\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?[\s\S]*?\s*?(\d+)\s*?', r.text) + proxies = re.findall(r'\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?[\s\S]*?\s*?(\d+)\s*?', + r.text) for proxy in proxies: yield ":".join(proxy) - @staticmethod def freeProxyWallFirst(): """ 墙外网站 cn-proxy - 并没有被墙 :return: """ urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] @@ -282,10 +267,10 @@ def freeProxyWallFirst(): @staticmethod def freeProxyWallSecond(): - ''' - 并没有被墙 + """ + https://proxy-list.org/english/index.php :return: - ''' + """ urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)] request = WebRequest() import base64 @@ -295,7 +280,6 @@ def freeProxyWallSecond(): for proxy in proxies: yield base64.b64decode(proxy).decode() - @staticmethod def freeProxyWallThird(): urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1'] @@ -307,67 +291,6 @@ def freeProxyWallThird(): yield ':'.join(proxy) -import threading - -lock = threading.Lock() -success = 0 -total = 0 - - -def test_once(proxy): - ip_port = proxy.split(":") - ip = ip_port[0] - port = ip_port[1] - import requests - - req_url = "http://www.baidu.com" - proxies = { - "http": "http://%s:%s" % (ip, port), - "https": "https://%s:%s" % (ip, port) - } - - global total - - try: - response = requests.get(req_url, proxies=proxies, timeout=4) - if response.status_code != 200: - print("unknow error, status code:" + str(response.status_code)) - lock.acquire() - total += 1 - lock.release() - return 0 - print("success") - global success - lock.acquire() - success += 1 - total += 1 - lock.release() - return 1 - except requests.exceptions.Timeout: - print("timeout") - except requests.exceptions.ConnectionError: - print("poxy unusable") - except Exception: - print("request error") - - lock.acquire() - total += 1 - lock.release() - return 0 - - -def test_batch(iterator): - global success - global total - - for proxy in iterator: - t = threading.Thread(target=test_once, args=(proxy,)) - t.start() - t.join() - - print("success:" + str(success) + "\ttotal:" + str(total)) - - if __name__ == '__main__': gg = GetFreeProxy() diff --git a/Test/.pytest_cache/v/cache/lastfailed b/Test/.pytest_cache/v/cache/lastfailed new file mode 100644 index 000000000..65c9a06d6 --- /dev/null +++ b/Test/.pytest_cache/v/cache/lastfailed @@ -0,0 +1,3 @@ +{ + "testGetFreeProxy.py::testGetFreeProxy": true +} \ No newline at end of file diff --git a/Test/.pytest_cache/v/cache/nodeids b/Test/.pytest_cache/v/cache/nodeids new file mode 100644 index 000000000..0ce3684ce --- /dev/null +++ b/Test/.pytest_cache/v/cache/nodeids @@ -0,0 +1,3 @@ +[ + "testGetFreeProxy.py::testGetFreeProxy" +] \ No newline at end of file diff --git a/Util/WebRequest.py b/Util/WebRequest.py index 7f5011724..68db87500 100644 --- a/Util/WebRequest.py +++ b/Util/WebRequest.py @@ -12,10 +12,10 @@ """ __author__ = 'J_hao' +from requests.models import Response import requests import random import time -from requests.models import Response class WebRequest(object): @@ -37,7 +37,7 @@ def user_agent(self): 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)', 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', - ] + ] return random.choice(ua_list) @property @@ -52,7 +52,7 @@ def header(self): 'Accept-Language': 'zh-CN,zh;q=0.8'} def get(self, url, header=None, retry_time=5, timeout=30, - retry_flag=list(), retry_interval=5, use_proxy=False, *args, **kwargs): + retry_flag=list(), retry_interval=5, *args, **kwargs): """ get method :param url: target url @@ -61,7 +61,6 @@ def get(self, url, header=None, retry_time=5, timeout=30, :param timeout: network timeout :param retry_flag: if retry_flag in content. do retry :param retry_interval: retry interval(second) - :param use_proxy: 是否使用代理 :param args: :param kwargs: :return: @@ -71,16 +70,7 @@ def get(self, url, header=None, retry_time=5, timeout=30, headers.update(header) while True: try: - if use_proxy: - proxy_url = "http://127.0.0.1:5010/get" - ip_proxy = requests.get(proxy_url).text - proxies = { - "http": "http://" + ip_proxy, - "https": "https://" + ip_proxy - } - html = requests.get(url, headers=headers, timeout=timeout, proxies=proxies) - else: - html = requests.get(url, headers=headers, timeout=timeout) + html = requests.get(url, headers=headers, timeout=timeout) if any(f in html.content for f in retry_flag): raise Exception return html @@ -88,9 +78,8 @@ def get(self, url, header=None, retry_time=5, timeout=30, print(e) retry_time -= 1 if retry_time <= 0: - # 多次请求失败时，返回百度页面 + # 多次请求失败 resp = Response() resp.status_code = 200 return resp time.sleep(retry_interval) - diff --git a/requirements.txt b/requirements.txt index c0db31b10..5d00da69a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ APScheduler==3.2.0 -Flask -requests -lxml +werkzeug==0.11.15 +Flask==0.12 +requests==2.12.4 +lxml==3.7.2 pymongo redis From b9f3e148c64d9a50b7c23768c1007b4d993eead8 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Fri, 27 Apr 2018 16:14:40 +0800 Subject: [PATCH 006/354] [update] add contributor --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32a0ac968..8c59cd631 100644 --- a/README.md +++ b/README.md @@ -183,5 +183,5 @@ freeProxyCustom = 1 # 确保名字和你添加方法名字一致 ### Release Notes - [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md) + [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md) [@luocaodan](https://github.com/luocaodan) From f9a8bf54054f0e7435c7e9718da9033988e0958c Mon Sep 17 00:00:00 2001 From: highroom <827148@163.com> Date: Mon, 14 May 2018 23:39:30 +0800 Subject: [PATCH 007/354] =?UTF-8?q?=E5=A2=9E=E5=8A=A0fq=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E7=9A=84=E9=85=8D=E7=BD=AE=EF=BC=8C=E9=85=8D=E7=BD=AE=E5=90=8E?= =?UTF-8?q?=E8=AF=B7=E8=B0=83=E7=94=A8=E4=BB=A3=E7=90=86=E8=AE=BF=E9=97=AE?= =?UTF-8?q?wallproxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 4 ++++ ProxyGetter/getFreeProxy.py | 44 ++++++++++++++++++++++++++++++++++--- Util/WebRequest.py | 2 +- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/Config.ini b/Config.ini index 5f417badc..dae17c1a2 100644 --- a/Config.ini +++ b/Config.ini @@ -29,3 +29,7 @@ freeProxyWallThird = 1 ; API接口配置 http://127.0.0.1:5010 ip = 0.0.0.0 port = 5010 + +[WallProxy] +; fq代理配置 +; proxy = 127.0.0.1:1080 \ No newline at end of file diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 78837d50a..edb71fa8b 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -14,6 +14,12 @@ import re import sys import requests +import os + +try: + from configparser import ConfigParser # py3 +except: + from ConfigParser import ConfigParser # py2 try: from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 @@ -46,6 +52,15 @@ class GetFreeProxy(object): """ proxy getter """ + pwd = os.path.split(os.path.realpath(__file__))[0] + config_path = os.path.join(os.path.split(pwd)[0], 'Config.ini') + config_file = ConfigParser() + config_file.read(config_path) + if config_file.has_option('WallProxy', 'proxy'): + WallProxy = config_file.get('WallProxy', 'proxy') + wall_proxies = {"http": "http://{}".format(WallProxy), "https": "https://{}".format(WallProxy)} + else: + wall_proxies = None def __init__(self): pass @@ -257,10 +272,17 @@ def freeProxyWallFirst(): 墙外网站 cn-proxy :return: """ + kwargs = {} + if GetFreeProxy.wall_proxies: + kwargs['proxies'] = GetFreeProxy.wall_proxies + else: + return + urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] request = WebRequest() for url in urls: - r = request.get(url) + kwargs['url'] = url + r = request.get(**kwargs) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W](\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -271,21 +293,35 @@ def freeProxyWallSecond(): https://proxy-list.org/english/index.php :return: """ + kwargs = {} + if GetFreeProxy.wall_proxies: + kwargs['proxies'] = GetFreeProxy.wall_proxies + else: + return urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)] request = WebRequest() import base64 for url in urls: - r = request.get(url) + kwargs['url'] = url + r = request.get(**kwargs) proxies = re.findall(r"Proxy\('(.*?)'\)", r.text) for proxy in proxies: yield base64.b64decode(proxy).decode() @staticmethod def freeProxyWallThird(): + + kwargs = {} + if GetFreeProxy.wall_proxies: + kwargs['proxies'] = GetFreeProxy.wall_proxies + else: + return + urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1'] request = WebRequest() for url in urls: - r = request.get(url) + kwargs['url'] = url + r = request.get(**kwargs) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -319,3 +355,5 @@ def freeProxyWallThird(): # test_batch(gg.freeProxyWallSecond()) # test_batch(gg.freeProxyWallThird()) + for e in gg.freeProxyWallThird(): + print(e) diff --git a/Util/WebRequest.py b/Util/WebRequest.py index 68db87500..47286a225 100644 --- a/Util/WebRequest.py +++ b/Util/WebRequest.py @@ -70,7 +70,7 @@ def get(self, url, header=None, retry_time=5, timeout=30, headers.update(header) while True: try: - html = requests.get(url, headers=headers, timeout=timeout) + html = requests.get(url, headers=headers, timeout=timeout, **kwargs) if any(f in html.content for f in retry_flag): raise Exception return html From 876b1ec4de23924f8e940772a8690d2373c1a402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8D=A3=E9=A3=9E=20=E5=BE=90?= Date: Wed, 23 May 2018 15:23:41 +0800 Subject: [PATCH 008/354] add http://ip.jiangxianli.com/ --- Config.ini | 1 + ProxyGetter/getFreeProxy.py | 37 +++++++++++++++++++++++++++++++++++++ Test/testGetFreeProxy.py | 16 ++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Config.ini b/Config.ini index 5f417badc..ca011a01f 100644 --- a/Config.ini +++ b/Config.ini @@ -20,6 +20,7 @@ freeProxyEight = 1 freeProxyNinth = 1 freeProxyTen = 1 freeProxyEleven = 1 +freeProxyTwelve = 1 ;foreign website, outside the wall freeProxyWallFirst = 1 freeProxyWallSecond = 1 diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 78837d50a..54d09c1b0 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -15,6 +15,7 @@ import sys import requests + try: from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 except: @@ -25,6 +26,7 @@ from Util.utilFunction import robustCrawl, getHtmlTree from Util.WebRequest import WebRequest +from Util.utilFunction import verifyProxyFormat # for debug to disable insecureWarning requests.packages.urllib3.disable_warnings() @@ -251,6 +253,24 @@ def freeProxyEleven(): for proxy in proxies: yield ":".join(proxy) + @staticmethod + def freeProxyTwelve(page_count=8): + """ + guobanjia http://ip.jiangxianli.com/?page= + 免费代理库 + 超多量 + :return: + """ + for i in range(1, page_count + 1): + url = 'http://ip.jiangxianli.com/?page={}'.format(i) + # print(url) + html_tree = getHtmlTree(url) + tr_list = html_tree.xpath("/html/body/div[1]/div/div[1]/div[2]/table/tbody/tr") + if len(tr_list) == 0: + continue + for tr in tr_list: + yield tr.xpath("./td[2]/text()")[0] + ":" + tr.xpath("./td[3]/text()")[0] + @staticmethod def freeProxyWallFirst(): """ @@ -314,6 +334,23 @@ def freeProxyWallThird(): # test_batch(gg.freeProxyEleven()) + proxy_iter = gg.freeProxyTwelve() + proxy_set = set() + for proxy in proxy_iter: + proxy = proxy.strip() + if proxy and verifyProxyFormat(proxy): + #self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) + proxy_set.add(proxy) + #else: + #self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy)) + + # store + for proxy in proxy_set: + print(proxy) + + + # test_batch(gg.freeProxyTwelve()) + # test_batch(gg.freeProxyWallFirst()) # test_batch(gg.freeProxyWallSecond()) diff --git a/Test/testGetFreeProxy.py b/Test/testGetFreeProxy.py index df99c79f3..33c3f9e46 100644 --- a/Test/testGetFreeProxy.py +++ b/Test/testGetFreeProxy.py @@ -12,6 +12,18 @@ """ __author__ = 'J_hao' +import re +import sys +import requests + + +try: + from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 +except: + reload(sys) + sys.setdefaultencoding('utf-8') + +sys.path.append('..') from ProxyGetter.getFreeProxy import GetFreeProxy from Util.GetConfig import GetConfig @@ -28,9 +40,9 @@ def testGetFreeProxy(): proxy_count = 0 for proxy in getattr(GetFreeProxy, proxyGetter.strip())(): if proxy: - print('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) + print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,proxy_count=proxy_count)) proxy_count += 1 - assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter) + #assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter) if __name__ == '__main__': From 413e41b2973e41742e55ab7bb7a1d642fe6ada8d Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 10 Jul 2018 16:50:31 +0800 Subject: [PATCH 009/354] =?UTF-8?q?[update]=20=E4=BF=AE=E6=94=B9ProxyGette?= =?UTF-8?q?r=E6=A3=80=E6=9F=A5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 4 -- ProxyGetter/CheckProxy.py | 72 ++++++++++++++++++++++++++++ ProxyGetter/getFreeProxy.py | 96 ++++--------------------------------- 3 files changed, 81 insertions(+), 91 deletions(-) create mode 100644 ProxyGetter/CheckProxy.py diff --git a/Config.ini b/Config.ini index 95e33400d..ca011a01f 100644 --- a/Config.ini +++ b/Config.ini @@ -30,7 +30,3 @@ freeProxyWallThird = 1 ; API接口配置 http://127.0.0.1:5010 ip = 0.0.0.0 port = 5010 - -[WallProxy] -; fq代理配置 -; proxy = 127.0.0.1:1080 \ No newline at end of file diff --git a/ProxyGetter/CheckProxy.py b/ProxyGetter/CheckProxy.py new file mode 100644 index 000000000..f6ba9b66a --- /dev/null +++ b/ProxyGetter/CheckProxy.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： CheckProxy + Description : used for check getFreeProxy.py + Author : JHao + date： 2018/7/10 +------------------------------------------------- + Change Activity: + 2018/7/10: CheckProxy +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys +from getFreeProxy import GetFreeProxy +from Util.utilFunction import verifyProxyFormat + +sys.path.append('../') + +from Util.LogHandler import LogHandler + +log = LogHandler('check_proxy', file=False) + + +class CheckProxy(object): + + @staticmethod + def checkAllGetProxyFunc(): + """ + 检查getFreeProxy所有代理获取函数运行情况 + Returns: + None + """ + import inspect + member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction) + proxy_count_dict = dict() + for func_name, func in member_list: + log.info(u"开始运行 {}".format(func_name)) + try: + proxy_list = [_ for _ in func() if verifyProxyFormat(_)] + proxy_count_dict[func_name] = len(proxy_list) + except Exception as e: + log.info(u"代理获取函数 {} 运行出错!".format(func_name)) + log.error(str(e)) + log.info(u"所有函数运行完毕 " + "***" * 5) + for func_name, func in member_list: + log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0))) + + @staticmethod + def checkGetProxyFunc(func): + """ + 检查指定的getFreeProxy某个function运行情况 + Args: + func: getFreeProxy中某个可调用方法 + + Returns: + None + """ + func_name = getattr(func, '__name__', "None") + log.info("start running func: {}".format(func_name)) + count = 0 + for proxy in func(): + if verifyProxyFormat(proxy): + log.info("fetch proxy: {}".format(proxy)) + count += 1 + log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count)) + + +if __name__ == '__main__': + CheckProxy.checkAllGetProxyFunc() + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 23542a5b7..bf2e03f61 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -14,13 +14,6 @@ import re import sys import requests -import os - -try: - from configparser import ConfigParser # py3 -except: - from ConfigParser import ConfigParser # py2 - try: from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 @@ -30,8 +23,8 @@ sys.path.append('..') -from Util.utilFunction import robustCrawl, getHtmlTree from Util.WebRequest import WebRequest +from Util.utilFunction import getHtmlTree from Util.utilFunction import verifyProxyFormat # for debug to disable insecureWarning @@ -54,15 +47,6 @@ class GetFreeProxy(object): """ proxy getter """ - pwd = os.path.split(os.path.realpath(__file__))[0] - config_path = os.path.join(os.path.split(pwd)[0], 'Config.ini') - config_file = ConfigParser() - config_file.read(config_path) - if config_file.has_option('WallProxy', 'proxy'): - WallProxy = config_file.get('WallProxy', 'proxy') - wall_proxies = {"http": "http://{}".format(WallProxy), "https": "https://{}".format(WallProxy)} - else: - wall_proxies = None def __init__(self): pass @@ -215,7 +199,7 @@ def freeProxyEight(): request = WebRequest() for url in url_list: - r = request.get(url, use_proxy=True) + r = request.get(url) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W].*(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -278,7 +262,6 @@ def freeProxyTwelve(page_count=8): """ for i in range(1, page_count + 1): url = 'http://ip.jiangxianli.com/?page={}'.format(i) - # print(url) html_tree = getHtmlTree(url) tr_list = html_tree.xpath("/html/body/div[1]/div/div[1]/div[2]/table/tbody/tr") if len(tr_list) == 0: @@ -292,17 +275,10 @@ def freeProxyWallFirst(): 墙外网站 cn-proxy :return: """ - kwargs = {} - if GetFreeProxy.wall_proxies: - kwargs['proxies'] = GetFreeProxy.wall_proxies - else: - return - urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] request = WebRequest() for url in urls: - kwargs['url'] = url - r = request.get(**kwargs) + r = request.get(url) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W](\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -313,84 +289,30 @@ def freeProxyWallSecond(): https://proxy-list.org/english/index.php :return: """ - kwargs = {} - if GetFreeProxy.wall_proxies: - kwargs['proxies'] = GetFreeProxy.wall_proxies - else: - return urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)] request = WebRequest() import base64 for url in urls: - kwargs['url'] = url - r = request.get(**kwargs) + r = request.get(url) proxies = re.findall(r"Proxy\('(.*?)'\)", r.text) for proxy in proxies: yield base64.b64decode(proxy).decode() @staticmethod def freeProxyWallThird(): - - kwargs = {} - if GetFreeProxy.wall_proxies: - kwargs['proxies'] = GetFreeProxy.wall_proxies - else: - return - urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1'] request = WebRequest() for url in urls: - kwargs['url'] = url - r = request.get(**kwargs) + r = request.get(url) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) if __name__ == '__main__': - gg = GetFreeProxy() - - # test_batch(gg.freeProxyFirst()) - - # test_batch(gg.freeProxySecond()) - - # test_batch(gg.freeProxyFourth()) - - # test_batch(gg.freeProxyFifth()) - - # test_batch(gg.freeProxySixth()) - - # test_batch(gg.freeProxySeventh()) - - # test_batch(gg.freeProxyEight()) - - # test_batch(gg.freeProxyNinth()) - - # test_batch(gg.freeProxyTen()) - - # test_batch(gg.freeProxyEleven()) - - proxy_iter = gg.freeProxyTwelve() - proxy_set = set() - for proxy in proxy_iter: - proxy = proxy.strip() - if proxy and verifyProxyFormat(proxy): - #self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) - proxy_set.add(proxy) - #else: - #self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy)) - - # store - for proxy in proxy_set: - print(proxy) - - - # test_batch(gg.freeProxyTwelve()) - - # test_batch(gg.freeProxyWallFirst()) + from CheckProxy import CheckProxy - # test_batch(gg.freeProxyWallSecond()) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) - # test_batch(gg.freeProxyWallThird()) - for e in gg.freeProxyWallThird(): - print(e) + CheckProxy.checkAllGetProxyFunc() From edac60ce8ea6340834e1e4afa53d37f3e1a783a8 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 10 Jul 2018 16:54:13 +0800 Subject: [PATCH 010/354] [update] readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c59cd631..8dc30eee9 100644 --- a/README.md +++ b/README.md @@ -178,10 +178,10 @@ freeProxyCustom = 1 # 确保名字和你添加方法名字一致 　　这里感谢以下contributor的无私奉献： -　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom) +　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom)| [@luocaodan](https://github.com/luocaodan) ### Release Notes - [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md) [@luocaodan](https://github.com/luocaodan) + [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md) From 62b05856fbed3f104842010defe0f46fd5e5c242 Mon Sep 17 00:00:00 2001 From: YeClimEric Date: Wed, 10 Oct 2018 17:50:34 +0800 Subject: [PATCH 011/354] =?UTF-8?q?1.flask=E6=94=AF=E6=8C=81=E5=A4=9A?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E5=A4=84=E7=90=86=E4=BB=BB=E5=8A=A1=202.?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20proxy=20=E9=87=87=E9=9B=86=E3=80=81?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E6=B5=81=E7=A8=8B=EF=BC=8C=E5=8A=A0=E5=BF=AB?= =?UTF-8?q?=20userfull=20proxy=20=E6=A0=A1=E9=AA=8C=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Api/ProxyApi.py | 9 ++++----- Config.ini | 8 +++++--- Manager/ProxyManager.py | 32 ++++++++++++-------------------- Schedule/ProxyRefreshSchedule.py | 23 +++++++++++++---------- Util/GetConfig.py | 11 ++++++++--- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index 724dc35e6..2e3733013 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -2,13 +2,13 @@ # !/usr/bin/env python """ ------------------------------------------------- - File Name： ProxyApi.py - Description : + File Name： ProxyApi.py + Description : Author : JHao date： 2016/12/4 ------------------------------------------------- Change Activity: - 2016/12/4: + 2016/12/4: ------------------------------------------------- """ __author__ = 'JHao' @@ -26,7 +26,6 @@ class JsonResponse(Response): - @classmethod def force_type(cls, response, environ=None): if isinstance(response, (dict, list)): @@ -86,7 +85,7 @@ def getStatus(): def run(): config = GetConfig() - app.run(host=config.host_ip, port=config.host_port) + app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes) if __name__ == '__main__': diff --git a/Config.ini b/Config.ini index ca011a01f..d1ab07bb4 100644 --- a/Config.ini +++ b/Config.ini @@ -9,11 +9,11 @@ name = proxy [ProxyGetter] ;register the proxy getter function -freeProxyFirst = 1 +freeProxyFirst = 1 freeProxySecond = 1 ;freeProxyThird = 1 freeProxyFourth = 1 -freeProxyFifth = 1 +freeProxyFifth = 1 freeProxySixth = 1 freeProxySeventh = 1 freeProxyEight = 1 @@ -26,7 +26,9 @@ freeProxyWallFirst = 1 freeProxyWallSecond = 1 freeProxyWallThird = 1 -[HOST] +[API] ; API接口配置 http://127.0.0.1:5010 ip = 0.0.0.0 port = 5010 +; flask多进程处理请求 +processes = 10 diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py index 6131c089a..33aa76b39 100644 --- a/Manager/ProxyManager.py +++ b/Manager/ProxyManager.py @@ -2,13 +2,13 @@ # !/usr/bin/env python """ ------------------------------------------------- - File Name： ProxyManager.py - Description : + File Name： ProxyManager.py + Description : Author : JHao date： 2016/12/3 ------------------------------------------------- Change Activity: - 2016/12/3: + 2016/12/3: ------------------------------------------------- """ __author__ = 'JHao' @@ -40,30 +40,22 @@ def refresh(self): fetch proxy into Db by ProxyGetter :return: """ + self.db.changeTable(self.raw_proxy_queue) for proxyGetter in self.config.proxy_getter_functions: # fetch - proxy_set = set() try: self.log.info("{func}: fetch proxy start".format(func=proxyGetter)) - proxy_iter = [_ for _ in getattr(GetFreeProxy, proxyGetter.strip())()] + for proxy in getattr(GetFreeProxy, proxyGetter.strip())(): + # 挨个存储 proxy，优化raw 队列的 push 速度，进而加快 check proxy 的速度 + proxy = proxy.strip() + if proxy and verifyProxyFormat(proxy): + self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) + self.db.put(proxy) + else: + self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy)) except Exception as e: self.log.error("{func}: fetch proxy fail".format(func=proxyGetter)) continue - for proxy in proxy_iter: - proxy = proxy.strip() - if proxy and verifyProxyFormat(proxy): - self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) - proxy_set.add(proxy) - else: - self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy)) - - # store - for proxy in proxy_set: - self.db.changeTable(self.useful_proxy_queue) - if self.db.exists(proxy): - continue - self.db.changeTable(self.raw_proxy_queue) - self.db.put(proxy) def get(self): """ diff --git a/Schedule/ProxyRefreshSchedule.py b/Schedule/ProxyRefreshSchedule.py index 7dac2aa34..6088fcb0a 100644 --- a/Schedule/ProxyRefreshSchedule.py +++ b/Schedule/ProxyRefreshSchedule.py @@ -18,7 +18,8 @@ import time import logging from threading import Thread -from apscheduler.schedulers.blocking import BlockingScheduler +# 使用后台调度，不使用阻塞式~ +from apscheduler.schedulers.background import BackgroundScheduler as Sch sys.path.append('../') @@ -73,12 +74,7 @@ def refreshPool(): pp.validProxy() -def main(process_num=30): - p = ProxyRefreshSchedule() - - # 获取新代理 - p.refresh() - +def batch_refresh(process_num=30): # 检验新代理 pl = [] for num in range(process_num): @@ -93,11 +89,18 @@ def main(process_num=30): pl[num].join() +def fetch_all(): + p = ProxyRefreshSchedule() + # 获取新代理 + p.refresh() + + def run(): - main() - sch = BlockingScheduler() - sch.add_job(main, 'interval', minutes=10) # 每10分钟抓取一次 + sch = Sch() + sch.add_job(fetch_all, 'interval', minutes=5) # 每5分钟抓取一次 + sch.add_job(batch_refresh, "interval", minutes=1) # 每分钟检查一次 sch.start() + fetch_all() if __name__ == '__main__': diff --git a/Util/GetConfig.py b/Util/GetConfig.py index 24b003f28..8ea57be56 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -2,7 +2,7 @@ # !/usr/bin/env python """ ------------------------------------------------- - File Name： GetConfig.py + File Name： GetConfig.py Description : fetch config from config.ini Author : JHao date： 2016/12/3 @@ -51,11 +51,15 @@ def proxy_getter_functions(self): @LazyProperty def host_ip(self): - return self.config_file.get('HOST','ip') + return self.config_file.get('API','ip') @LazyProperty def host_port(self): - return int(self.config_file.get('HOST', 'port')) + return int(self.config_file.get('API', 'port')) + + @LazyProperty + def processes(self): + return int(self.config_file.get('API', 'processes')) if __name__ == '__main__': gg = GetConfig() @@ -66,3 +70,4 @@ def host_port(self): print(gg.proxy_getter_functions) print(gg.host_ip) print(gg.host_port) + print(gg.processes) From a0b152a968e073c0c35f8dc03d862f783ba4ee86 Mon Sep 17 00:00:00 2001 From: YeClimEric Date: Wed, 10 Oct 2018 18:15:47 +0800 Subject: [PATCH 012/354] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c815a4e7..d97495489 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,28 +3,25 @@ WORKDIR /usr/src/app COPY . . ENV DEBIAN_FRONTEND noninteractive ENV TZ Asia/Shanghai -RUN pip install --no-cache-dir -r requirements.txt && \ - apt-get update && \ - apt-get install -y --force-yes git make gcc g++ autoconf && apt-get clean && \ - git clone --depth 1 https://github.com/ideawu/ssdb.git ssdb && \ - cd ssdb && make && make install && cp ssdb-server /usr/bin && \ - apt-get remove -y --force-yes git make gcc g++ autoconf && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - cp ssdb.conf /etc && cd .. && yes | rm -r ssdb && \ - mkdir -p /var/lib/ssdb && \ - sed \ - -e 's@home.*@home /var/lib@' \ - -e 's/loglevel.*/loglevel info/' \ - -e 's@work_dir = .*@work_dir = /var/lib/ssdb@' \ - -e 's@pidfile = .*@pidfile = /run/ssdb.pid@' \ - -e 's@level:.*@level: info@' \ - -e 's@ip:.*@ip: 0.0.0.0@' \ - -i /etc/ssdb.conf && \ - echo "# ! /bin/sh " > /usr/src/app/run.sh && \ - echo "cd Run" >> /usr/src/app/run.sh && \ - echo "/usr/bin/ssdb-server /etc/ssdb.conf &" >> /usr/src/app/run.sh && \ - echo "python main.py" >> /usr/src/app/run.sh && \ - chmod 777 run.sh + +RUN apt-get update +RUN apt-get install vim -y + +RUN apt-get install -y redis-server +RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf \ + && sed -i 's/^\(databases .*\)$/databases 1/' /etc/redis/redis.conf \ + && sed -i 's/^\(daemonize .*\)$/daemonize yes/' /etc/redis/redis.conf +# && sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/redis/redis.conf \ +# && sed -i 's/^\(logfile .*\)$/# \1/' /etc/redis/redis.conf + +RUN pip install --no-cache-dir -r requirements.txt + + +RUN echo "# ! /bin/sh " > run.sh \ + && echo "redis-server /etc/redis/redis.conf&" >> run.sh \ + && echo "cd Run" >> run.sh \ + && echo "python main.py" >> run.sh \ + && chmod 777 run.sh + EXPOSE 5010 CMD [ "sh", "run.sh" ] From 5de6b7d3793337f7c5aa05dd3539c7db3b31fc9e Mon Sep 17 00:00:00 2001 From: YeClimEric Date: Wed, 10 Oct 2018 19:29:58 +0800 Subject: [PATCH 013/354] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Schedule/ProxyRefreshSchedule.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Schedule/ProxyRefreshSchedule.py b/Schedule/ProxyRefreshSchedule.py index 6088fcb0a..38668072d 100644 --- a/Schedule/ProxyRefreshSchedule.py +++ b/Schedule/ProxyRefreshSchedule.py @@ -102,6 +102,9 @@ def run(): sch.start() fetch_all() + while True: + time.sleep(1) + if __name__ == '__main__': run() From 2086a52ecc21c3099c328fa0df40281399feebaf Mon Sep 17 00:00:00 2001 From: jhao104 Date: Wed, 17 Oct 2018 14:21:09 +0800 Subject: [PATCH 014/354] [fix] fix198 --- Api/ProxyApi.py | 5 ++++- Config.ini | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index 2e3733013..b8977f9ca 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -85,7 +85,10 @@ def getStatus(): def run(): config = GetConfig() - app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes) + if sys.platform.startswith("win"): + app.run(host=config.host_ip, port=config.host_port) + else: + app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes) if __name__ == '__main__': diff --git a/Config.ini b/Config.ini index d1ab07bb4..9394f744e 100644 --- a/Config.ini +++ b/Config.ini @@ -27,8 +27,7 @@ freeProxyWallSecond = 1 freeProxyWallThird = 1 [API] -; API接口配置 http://127.0.0.1:5010 +; API config http://127.0.0.1:5010 ip = 0.0.0.0 port = 5010 -; flask多进程处理请求 processes = 10 From 7449f7dabb9449a6eedf67f2ff4d20df39a9e5ae Mon Sep 17 00:00:00 2001 From: vc5 Date: Thu, 25 Oct 2018 00:01:36 +0800 Subject: [PATCH 015/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=86=E7=A0=81?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 1 + DB/DbClient.py | 3 ++- DB/SsdbClient.py | 4 ++-- Util/GetConfig.py | 9 +++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Config.ini b/Config.ini index 9394f744e..24f570f01 100644 --- a/Config.ini +++ b/Config.ini @@ -6,6 +6,7 @@ host = 127.0.0.1 port = 6379 ;port = 8888 name = proxy +#password = yourpassword [ProxyGetter] ;register the proxy getter function diff --git a/DB/DbClient.py b/DB/DbClient.py index 68c5db7a7..0036434ae 100644 --- a/DB/DbClient.py +++ b/DB/DbClient.py @@ -75,7 +75,8 @@ def __initDbClient(self): assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type) self.client = getattr(__import__(__type), __type)(name=self.config.db_name, host=self.config.db_host, - port=self.config.db_port) + port=self.config.db_port, + password=self.config.db_password) def get(self, key, **kwargs): return self.client.get(key, **kwargs) diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py index 2522e0071..2249fdcc1 100644 --- a/DB/SsdbClient.py +++ b/DB/SsdbClient.py @@ -32,7 +32,7 @@ class SsdbClient(object): """ - def __init__(self, name, host, port): + def __init__(self, name, **kwargs): """ init :param name: hash name @@ -41,7 +41,7 @@ def __init__(self, name, host, port): :return: """ self.name = name - self.__conn = Redis(connection_pool=BlockingConnectionPool(host=host, port=port)) + self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs)) def get(self, proxy): """ diff --git a/Util/GetConfig.py b/Util/GetConfig.py index 8ea57be56..c4c31ab0e 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -45,6 +45,15 @@ def db_host(self): def db_port(self): return int(self.config_file.get('DB', 'port')) + @LazyProperty + def db_password(self): + try: + password = self.config_file.get('DB', 'password') + except Exception: + password = None + return password + + @LazyProperty def proxy_getter_functions(self): return self.config_file.options('ProxyGetter') From 0238d9f931425736c9d72e4ea3e429ff4f03ef64 Mon Sep 17 00:00:00 2001 From: J_hao104 Date: Mon, 29 Oct 2018 09:44:48 +0800 Subject: [PATCH 016/354] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dc30eee9..47480fb92 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ freeProxyCustom = 1 # 确保名字和你添加方法名字一致 　　这里感谢以下contributor的无私奉献： -　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom)| [@luocaodan](https://github.com/luocaodan) +　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom)| [@luocaodan](https://github.com/luocaodan)| [@vc5](https://github.com/vc5) ### Release Notes From 8ac170e981fb08a892c27552782b4528d67f64eb Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 8 Nov 2018 21:35:31 +0800 Subject: [PATCH 017/354] =?UTF-8?q?=E5=AE=8C=E5=96=84Redis=E5=92=8CMongodb?= =?UTF-8?q?=E9=AA=8C=E8=AF=81=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加Config.ini的用户和密码 为username参数做兼容处理 --- Config.ini | 3 ++- DB/DbClient.py | 1 + DB/MongodbClient.py | 4 ++-- DB/RedisClient.py | 8 ++++++-- DB/SsdbClient.py | 7 +++++-- Util/GetConfig.py | 7 +++++++ 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Config.ini b/Config.ini index 24f570f01..cf3f8ded2 100644 --- a/Config.ini +++ b/Config.ini @@ -6,7 +6,8 @@ host = 127.0.0.1 port = 6379 ;port = 8888 name = proxy -#password = yourpassword +;username = your_username (Only Mongodb) +;password = your_password [ProxyGetter] ;register the proxy getter function diff --git a/DB/DbClient.py b/DB/DbClient.py index 0036434ae..40127cc11 100644 --- a/DB/DbClient.py +++ b/DB/DbClient.py @@ -76,6 +76,7 @@ def __initDbClient(self): self.client = getattr(__import__(__type), __type)(name=self.config.db_name, host=self.config.db_host, port=self.config.db_port, + username=self.config.db_username, password=self.config.db_password) def get(self, key, **kwargs): diff --git a/DB/MongodbClient.py b/DB/MongodbClient.py index bd0647f51..a30ef6cf1 100644 --- a/DB/MongodbClient.py +++ b/DB/MongodbClient.py @@ -17,9 +17,9 @@ class MongodbClient(object): - def __init__(self, name, host, port): + def __init__(self, name, host, port, **kwargs): self.name = name - self.client = MongoClient(host, port) + self.client = MongoClient(host, port, **kwargs) self.db = self.client.proxy def changeTable(self, name): diff --git a/DB/RedisClient.py b/DB/RedisClient.py index 7d9af4386..1983d855e 100644 --- a/DB/RedisClient.py +++ b/DB/RedisClient.py @@ -22,7 +22,11 @@ class RedisClient(object): Reids client """ - def __init__(self, name, host, port): + # 为了保持DbClient的标准 + # 在RedisClient里面接受username参数, 但不进行使用. + # 因为不能将username通过kwargs传进redis.Redis里面, 会报错: + # TypeError: __init__() got an unexpected keyword argument 'username' + def __init__(self, name, host, port, username, **kwargs): """ init :param name: @@ -31,7 +35,7 @@ def __init__(self, name, host, port): :return: """ self.name = name - self.__conn = redis.Redis(host=host, port=port, db=0) + self.__conn = redis.Redis(host=host, port=port, db=0, **kwargs) def get(self): """ diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py index 2249fdcc1..202ddaa8f 100644 --- a/DB/SsdbClient.py +++ b/DB/SsdbClient.py @@ -31,8 +31,11 @@ class SsdbClient(object): 验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1； """ - - def __init__(self, name, **kwargs): + # 为了保持DbClient的标准 + # 在SsdbClient里面接受username参数, 但不进行使用. + # 因为不能将username通过kwargs传进redis.Redis里面, 会报错: + # TypeError: __init__() got an unexpected keyword argument 'username' + def __init__(self, name, username, **kwargs): """ init :param name: hash name diff --git a/Util/GetConfig.py b/Util/GetConfig.py index c4c31ab0e..c26b00f1e 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -53,6 +53,13 @@ def db_password(self): password = None return password + @LazyProperty + def db_username(self): + try: + username = self.config_file.get('DB', 'username') + except Exception: + username = None + return username @LazyProperty def proxy_getter_functions(self): From 4eaaa7dc12a5e318368f8eb4f1bb08ef8ee7ca48 Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 8 Nov 2018 22:22:43 +0800 Subject: [PATCH 018/354] =?UTF-8?q?=E4=BC=98=E5=8C=96Docker=E4=BD=BF?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 标准化Dockerfile 2. 添加Docker-compose的部署方式 3. 整理Docker相关的文件 --- Docker/Dockerfile | 13 +++++++++++++ Dockerfile => Docker/Dockerfile.develop | 0 Docker/docker-compose.yml | 14 ++++++++++++++ README.md | 17 +++++++++++++++++ Run/main.py | 3 ++- 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 Docker/Dockerfile rename Dockerfile => Docker/Dockerfile.develop (100%) create mode 100644 Docker/docker-compose.yml diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 000000000..6ad6f5f53 --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.6 +WORKDIR /usr/src/app +COPY . . + +ENV DEBIAN_FRONTEND noninteractive +ENV TZ Asia/Shanghai + +RUN pip install --no-cache-dir -r requirements.txt + +EXPOSE 5010 + +WORKDIR /usr/src/app/ +CMD [ "python", "Run/main.py" ] diff --git a/Dockerfile b/Docker/Dockerfile.develop similarity index 100% rename from Dockerfile rename to Docker/Dockerfile.develop diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml new file mode 100644 index 000000000..9529745d5 --- /dev/null +++ b/Docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' +services: + proxy_pool: + volumes: + - ..:/usr/src/app + ports: + - "5010:5010" + links: + - proxy_redis + image: "proxy_pool" + proxy_redis: + ports: + - "6379:6379" + image: "redis" \ No newline at end of file diff --git a/README.md b/README.md index 47480fb92..e5cece52a 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,23 @@ port = 5010 # 监听端口 # 依次到Api下启动ProxyApi.py,Schedule下启动ProxyRefreshSchedule.py和ProxyValidSchedule.py即可. ``` +* 生产环境 Docker/docker-compose + +```shell +# Workdir proxy_pool +docker build -t proxy_pool . +pip install docker-compose +docker-compose -f Docker/docker-compose.yml up -d +``` + +* 开发环境 Docker + +```shell +# Workdir proxy_pool +docker build -t proxy_pool . +docker run -it --rm -v $(pwd):/usr/src/app -p 5010:5010 proxy_pool +``` + ### 使用 　　启动过几分钟后就能看到抓取到的代理IP，你可以直接到数据库中查看，推荐一个[SSDB可视化工具](https://github.com/jhao104/SSDBAdmin)。 diff --git a/Run/main.py b/Run/main.py index 6b07654ee..fcd84f6f4 100644 --- a/Run/main.py +++ b/Run/main.py @@ -15,7 +15,8 @@ import sys from multiprocessing import Process -sys.path.append('../') +sys.path.append('.') +sys.path.append('..') from Api.ProxyApi import run as ProxyApiRun from Schedule.ProxyValidSchedule import run as ValidRun From 935929db18effd7cd319a7de1dc0871419ba3267 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 9 Nov 2018 15:49:08 +0800 Subject: [PATCH 019/354] [fix] The Requests package through 2.19.1 before 2018-09-14 for Python sends an HTTP Authorization header to an http URI upon receiving a same-hostname https-to-http redirect, which makes it easier for remote attackers to discover credentials by sniffing the network. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5d00da69a..bc3581ff5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ APScheduler==3.2.0 werkzeug==0.11.15 Flask==0.12 -requests==2.12.4 +requests==2.20.0 lxml==3.7.2 pymongo From dcfa0e03777ee833ba06967c33b6cd39e0371384 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 9 Nov 2018 16:29:29 +0800 Subject: [PATCH 020/354] =?UTF-8?q?[update]=20=E4=BC=98=E5=8C=96=E6=8A=93?= =?UTF-8?q?=E5=8E=BB=E5=87=BD=E6=95=B0=EF=BC=8C=E6=AF=8F=E6=AC=A1=E5=B0=91?= =?UTF-8?q?=E6=8A=93=E4=B8=80=E4=BA=9B=20=E5=87=8F=E5=B0=91=E8=80=97?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/getFreeProxy.py | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index bf2e03f61..a560dc700 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -15,17 +15,10 @@ import sys import requests -try: - from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 -except: - reload(sys) - sys.setdefaultencoding('utf-8') - sys.path.append('..') from Util.WebRequest import WebRequest from Util.utilFunction import getHtmlTree -from Util.utilFunction import verifyProxyFormat # for debug to disable insecureWarning requests.packages.urllib3.disable_warnings() @@ -48,9 +41,6 @@ class GetFreeProxy(object): proxy getter """ - def __init__(self): - pass - @staticmethod def freeProxyFirst(page=10): """ @@ -164,7 +154,7 @@ def freeProxySixth(): url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10' request = WebRequest() try: - res = request.get(url).json() + res = request.get(url, timeout=10).json() for row in res['RESULT']['rows']: yield '{}:{}'.format(row['ip'], row['port']) except Exception as e: @@ -180,7 +170,7 @@ def freeProxySeventh(): 'https://www.kuaidaili.com/free/intr/{page}/' ] for url in url_list: - for page in range(1, 5): + for page in range(1, 2): page_url = url.format(page=page) tree = getHtmlTree(page_url) proxy_list = tree.xpath('.//table//tr') @@ -192,14 +182,14 @@ def freeProxyEight(): """ 秘密代理 http://www.mimiip.com """ - url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)] # 国内高匿 - url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 10)] # 国内普匿 - url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 10)] # 国内透明 + url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 2)] # 国内高匿 + url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 2)] # 国内普匿 + url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 2)] # 国内透明 url_list = url_gngao + url_gnpu + url_gntou request = WebRequest() for url in url_list: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W].*(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -213,7 +203,7 @@ def freeProxyNinth(): urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] request = WebRequest() for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall('data-ip="(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})".+?>(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -227,7 +217,7 @@ def freeProxyTen(): urls = ['http://www.ip3366.net/free/'] request = WebRequest() for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) for proxy in proxies: yield ":".join(proxy) @@ -246,14 +236,14 @@ def freeProxyEleven(): ] request = WebRequest() for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r'\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?[\s\S]*?\s*?(\d+)\s*?', r.text) for proxy in proxies: yield ":".join(proxy) @staticmethod - def freeProxyTwelve(page_count=8): + def freeProxyTwelve(page_count=2): """ guobanjia http://ip.jiangxianli.com/?page= 免费代理库 @@ -278,7 +268,7 @@ def freeProxyWallFirst(): urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] request = WebRequest() for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W](\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -293,7 +283,7 @@ def freeProxyWallSecond(): request = WebRequest() import base64 for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r"Proxy\('(.*?)'\)", r.text) for proxy in proxies: yield base64.b64decode(proxy).decode() @@ -303,7 +293,7 @@ def freeProxyWallThird(): urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1'] request = WebRequest() for url in urls: - r = request.get(url) + r = request.get(url, timeout=10) proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', r.text) for proxy in proxies: yield ':'.join(proxy) @@ -312,7 +302,17 @@ def freeProxyWallThird(): if __name__ == '__main__': from CheckProxy import CheckProxy - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth) - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyThird) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFourth) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySixth) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySeventh) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEight) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyNinth) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTen) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve) - CheckProxy.checkAllGetProxyFunc() + # CheckProxy.checkAllGetProxyFunc() From f203ae19b6436b88d84d181a8f392c4044e04e09 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 9 Nov 2018 16:30:02 +0800 Subject: [PATCH 021/354] =?UTF-8?q?[update]=20=E6=A3=80=E6=9F=A5=20getter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/CheckProxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProxyGetter/CheckProxy.py b/ProxyGetter/CheckProxy.py index f6ba9b66a..f29824723 100644 --- a/ProxyGetter/CheckProxy.py +++ b/ProxyGetter/CheckProxy.py @@ -62,7 +62,7 @@ def checkGetProxyFunc(func): count = 0 for proxy in func(): if verifyProxyFormat(proxy): - log.info("fetch proxy: {}".format(proxy)) + log.info("{} fetch proxy: {}".format(func_name, proxy)) count += 1 log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count)) From 69eafeabdd11451adf2b6f42dac1620e729dcba3 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 9 Nov 2018 16:30:37 +0800 Subject: [PATCH 022/354] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B0=E5=8F=AF?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E4=BB=A3=E7=90=86=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Config.ini b/Config.ini index 24f570f01..1d46fc857 100644 --- a/Config.ini +++ b/Config.ini @@ -1,10 +1,9 @@ [DB] ;Configure the database information -;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB +;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB type = SSDB host = 127.0.0.1 port = 6379 -;port = 8888 name = proxy #password = yourpassword @@ -15,17 +14,17 @@ freeProxySecond = 1 ;freeProxyThird = 1 freeProxyFourth = 1 freeProxyFifth = 1 -freeProxySixth = 1 +;freeProxySixth = 1 freeProxySeventh = 1 -freeProxyEight = 1 -freeProxyNinth = 1 +;freeProxyEight = 1 +;freeProxyNinth = 1 freeProxyTen = 1 freeProxyEleven = 1 freeProxyTwelve = 1 ;foreign website, outside the wall -freeProxyWallFirst = 1 -freeProxyWallSecond = 1 -freeProxyWallThird = 1 +;freeProxyWallFirst = 1 +;freeProxyWallSecond = 1 +;freeProxyWallThird = 1 [API] ; API config http://127.0.0.1:5010 From d77e1110e99c49bbe0d81a2beb3beb3f0bbe3205 Mon Sep 17 00:00:00 2001 From: 1again Date: Fri, 9 Nov 2018 20:34:35 +0800 Subject: [PATCH 023/354] =?UTF-8?q?[refine]=20Refine=20GetConfig=20?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基于配置化的管理思想, 假设项目的任何地方都需要使用GetConfig 于是可以在GetConfig模块里生成一个config对象. 任何地方需要只要import即可. --- Api/ProxyApi.py | 3 +-- DB/DbClient.py | 19 +++++++++---------- Manager/ProxyManager.py | 5 ++--- Util/GetConfig.py | 2 ++ 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index b8977f9ca..99a0953a0 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -19,7 +19,7 @@ sys.path.append('../') -from Util.GetConfig import GetConfig +from Util.GetConfig import config from Manager.ProxyManager import ProxyManager app = Flask(__name__) @@ -84,7 +84,6 @@ def getStatus(): def run(): - config = GetConfig() if sys.platform.startswith("win"): app.run(host=config.host_ip, port=config.host_port) else: diff --git a/DB/DbClient.py b/DB/DbClient.py index 0036434ae..869c93af1 100644 --- a/DB/DbClient.py +++ b/DB/DbClient.py @@ -16,7 +16,7 @@ import os import sys -from Util.GetConfig import GetConfig +from Util.GetConfig import config from Util.utilClass import Singleton sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -55,7 +55,6 @@ def __init__(self): init :return: """ - self.config = GetConfig() self.__initDbClient() def __initDbClient(self): @@ -64,19 +63,19 @@ def __initDbClient(self): :return: """ __type = None - if "SSDB" == self.config.db_type: + if "SSDB" == config.db_type: __type = "SsdbClient" - elif "REDIS" == self.config.db_type: + elif "REDIS" == config.db_type: __type = "RedisClient" - elif "MONGODB" == self.config.db_type: + elif "MONGODB" == config.db_type: __type = "MongodbClient" else: pass - assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type) - self.client = getattr(__import__(__type), __type)(name=self.config.db_name, - host=self.config.db_host, - port=self.config.db_port, - password=self.config.db_password) + assert __type, 'type error, Not support DB type: {}'.format(config.db_type) + self.client = getattr(__import__(__type), __type)(name=config.db_name, + host=config.db_host, + port=config.db_port, + password=config.db_password) def get(self, key, **kwargs): return self.client.get(key, **kwargs) diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py index 33aa76b39..a2f39b3c5 100644 --- a/Manager/ProxyManager.py +++ b/Manager/ProxyManager.py @@ -17,7 +17,7 @@ from Util import EnvUtil from DB.DbClient import DbClient -from Util.GetConfig import GetConfig +from Util.GetConfig import config from Util.LogHandler import LogHandler from Util.utilFunction import verifyProxyFormat from ProxyGetter.getFreeProxy import GetFreeProxy @@ -30,7 +30,6 @@ class ProxyManager(object): def __init__(self): self.db = DbClient() - self.config = GetConfig() self.raw_proxy_queue = 'raw_proxy' self.log = LogHandler('proxy_manager') self.useful_proxy_queue = 'useful_proxy' @@ -41,7 +40,7 @@ def refresh(self): :return: """ self.db.changeTable(self.raw_proxy_queue) - for proxyGetter in self.config.proxy_getter_functions: + for proxyGetter in config.proxy_getter_functions: # fetch try: self.log.info("{func}: fetch proxy start".format(func=proxyGetter)) diff --git a/Util/GetConfig.py b/Util/GetConfig.py index c4c31ab0e..efbbe5077 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -70,6 +70,8 @@ def host_port(self): def processes(self): return int(self.config_file.get('API', 'processes')) +config = GetConfig() + if __name__ == '__main__': gg = GetConfig() print(gg.db_type) From 40861f429011c53e25693e62daede4b47c253dd2 Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 12 Nov 2018 10:00:38 +0800 Subject: [PATCH 024/354] [update] config annotation --- Config.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Config.ini b/Config.ini index 1d46fc857..54f690397 100644 --- a/Config.ini +++ b/Config.ini @@ -27,7 +27,10 @@ freeProxyTwelve = 1 ;freeProxyWallThird = 1 [API] -; API config http://127.0.0.1:5010 +# API config http://127.0.0.1:5010 +# The ip specified when starting the web API ip = 0.0.0.0 +# he port on which to run the web API port = 5010 +# Flask processes option processes = 10 From 2591918c874a001435b3ff0af8604e5070b8ff58 Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 12 Nov 2018 10:32:01 +0800 Subject: [PATCH 025/354] [update] formatting code --- Util/GetConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Util/GetConfig.py b/Util/GetConfig.py index efbbe5077..5dfae9912 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -53,14 +53,13 @@ def db_password(self): password = None return password - @LazyProperty def proxy_getter_functions(self): return self.config_file.options('ProxyGetter') @LazyProperty def host_ip(self): - return self.config_file.get('API','ip') + return self.config_file.get('API', 'ip') @LazyProperty def host_port(self): @@ -70,6 +69,7 @@ def host_port(self): def processes(self): return int(self.config_file.get('API', 'processes')) + config = GetConfig() if __name__ == '__main__': From 8a0404521ddcf17031a5975f83c7b6b5a8e3b662 Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 12 Nov 2018 11:27:13 +0800 Subject: [PATCH 026/354] [update] set default pwd option --- Util/GetConfig.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Util/GetConfig.py b/Util/GetConfig.py index 5dfae9912..0f60fcd2f 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -47,11 +47,7 @@ def db_port(self): @LazyProperty def db_password(self): - try: - password = self.config_file.get('DB', 'password') - except Exception: - password = None - return password + return self.config_file.get('DB', 'password', fallback="default pwd") @LazyProperty def proxy_getter_functions(self): @@ -82,3 +78,4 @@ def processes(self): print(gg.host_ip) print(gg.host_port) print(gg.processes) + print(gg.db_password) From 6525ea8e09f3a128f0e2652d5d333005b41196c2 Mon Sep 17 00:00:00 2001 From: jhao Date: Tue, 13 Nov 2018 10:28:31 +0800 Subject: [PATCH 027/354] =?UTF-8?q?[update]=20=E8=B0=83=E6=95=B4=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Schedule/ProxyRefreshSchedule.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Schedule/ProxyRefreshSchedule.py b/Schedule/ProxyRefreshSchedule.py index 38668072d..a61cc5a25 100644 --- a/Schedule/ProxyRefreshSchedule.py +++ b/Schedule/ProxyRefreshSchedule.py @@ -18,8 +18,7 @@ import time import logging from threading import Thread -# 使用后台调度，不使用阻塞式~ -from apscheduler.schedulers.background import BackgroundScheduler as Sch +from apscheduler.schedulers.background import BackgroundScheduler sys.path.append('../') @@ -74,7 +73,7 @@ def refreshPool(): pp.validProxy() -def batch_refresh(process_num=30): +def batchRefresh(process_num=30): # 检验新代理 pl = [] for num in range(process_num): @@ -89,21 +88,23 @@ def batch_refresh(process_num=30): pl[num].join() -def fetch_all(): +def fetchAll(): p = ProxyRefreshSchedule() # 获取新代理 p.refresh() def run(): - sch = Sch() - sch.add_job(fetch_all, 'interval', minutes=5) # 每5分钟抓取一次 - sch.add_job(batch_refresh, "interval", minutes=1) # 每分钟检查一次 - sch.start() - fetch_all() + scheduler = BackgroundScheduler() + # 不用太快, 网站更新速度比较慢, 太快会加大验证压力, 导致raw_proxy积压 + scheduler.add_job(fetchAll, 'interval', minutes=10, id="fetch_proxy") + scheduler.add_job(batchRefresh, "interval", minutes=1) # 每分钟检查一次 + scheduler.start() + + fetchAll() while True: - time.sleep(1) + time.sleep(3) if __name__ == '__main__': From c1e74b4237971caf9dfefede1405e0516c27fe7a Mon Sep 17 00:00:00 2001 From: jhao Date: Tue, 13 Nov 2018 10:29:14 +0800 Subject: [PATCH 028/354] =?UTF-8?q?[update]=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Manager/ProxyManager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py index a2f39b3c5..c770b6224 100644 --- a/Manager/ProxyManager.py +++ b/Manager/ProxyManager.py @@ -36,7 +36,7 @@ def __init__(self): def refresh(self): """ - fetch proxy into Db by ProxyGetter + fetch proxy into Db by ProxyGetter/getFreeProxy.py :return: """ self.db.changeTable(self.raw_proxy_queue) @@ -45,7 +45,7 @@ def refresh(self): try: self.log.info("{func}: fetch proxy start".format(func=proxyGetter)) for proxy in getattr(GetFreeProxy, proxyGetter.strip())(): - # 挨个存储 proxy，优化raw 队列的 push 速度，进而加快 check proxy 的速度 + # 直接存储代理, 不用在代码中排重, hash 结构本身具有排重功能 proxy = proxy.strip() if proxy and verifyProxyFormat(proxy): self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) From e41a9cbe796744f91e395e4064ebb5c9ef82e39c Mon Sep 17 00:00:00 2001 From: jhao Date: Tue, 13 Nov 2018 10:30:01 +0800 Subject: [PATCH 029/354] [update] dbclient --- DB/DbClient.py | 10 ++++------ DB/SsdbClient.py | 14 ++++++-------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/DB/DbClient.py b/DB/DbClient.py index 869c93af1..f79fc8511 100644 --- a/DB/DbClient.py +++ b/DB/DbClient.py @@ -44,7 +44,7 @@ class DbClient(object): 所有方法需要相应类去具体实现： SSDB：SsdbClient.py - REDIS:RedisClient.py + REDIS:RedisClient.py 停用 统一使用SsdbClient.py """ @@ -66,7 +66,7 @@ def __initDbClient(self): if "SSDB" == config.db_type: __type = "SsdbClient" elif "REDIS" == config.db_type: - __type = "RedisClient" + __type = "SsdbClient" elif "MONGODB" == config.db_type: __type = "MongodbClient" else: @@ -107,7 +107,5 @@ def getNumber(self): if __name__ == "__main__": account = DbClient() - print(account.get()) - account.changeTable('use') - account.put('ac') - print(account.get()) + account.changeTable('useful_proxy') + print(account.pop()) diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py index 202ddaa8f..4ceedd1df 100644 --- a/DB/SsdbClient.py +++ b/DB/SsdbClient.py @@ -31,16 +31,13 @@ class SsdbClient(object): 验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1； """ - # 为了保持DbClient的标准 - # 在SsdbClient里面接受username参数, 但不进行使用. - # 因为不能将username通过kwargs传进redis.Redis里面, 会报错: - # TypeError: __init__() got an unexpected keyword argument 'username' - def __init__(self, name, username, **kwargs): + def __init__(self, name, **kwargs): """ init :param name: hash name - :param host: ssdb host - :param port: ssdb port + :param host: host + :param port: port + :param password: password :return: """ self.name = name @@ -114,6 +111,7 @@ def getNumber(self): def changeTable(self, name): self.name = name + if __name__ == '__main__': - c = SsdbClient('useful_proxy', '118.24.52.95', 8899) + c = SsdbClient(name='useful_proxy', host='127.0.0.1', port=8899, password=None) print(c.getAll()) From 428359c8dada998481f038dbdc8d3923e5850c0e Mon Sep 17 00:00:00 2001 From: jhao Date: Tue, 13 Nov 2018 14:02:03 +0800 Subject: [PATCH 030/354] Merge branch 'jhao104/master' of https://github.com/1again/proxy_pool into 1again-jhao104/master # Conflicts: # DB/DbClient.py # Util/GetConfig.py --- Config.ini | 2 +- README.md | 2 +- Util/GetConfig.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Config.ini b/Config.ini index 44ef085d2..c8a9cc266 100644 --- a/Config.ini +++ b/Config.ini @@ -3,7 +3,7 @@ ;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB type = SSDB host = 127.0.0.1 -port = 6379 +port = 8888 name = proxy ;username = your_username (Only Mongodb) ;password = your_password diff --git a/README.md b/README.md index e5cece52a..8bdca40c5 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ freeProxyCustom = 1 # 确保名字和你添加方法名字一致 　　这里感谢以下contributor的无私奉献： -　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom)| [@luocaodan](https://github.com/luocaodan)| [@vc5](https://github.com/vc5) +　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom)| [@luocaodan](https://github.com/luocaodan)| [@vc5](https://github.com/vc5)| [@1again](https://github.com/1again) ### Release Notes diff --git a/Util/GetConfig.py b/Util/GetConfig.py index 0f60fcd2f..cd354e20f 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -47,7 +47,7 @@ def db_port(self): @LazyProperty def db_password(self): - return self.config_file.get('DB', 'password', fallback="default pwd") + return self.config_file.get('DB', 'password', fallback=None) @LazyProperty def proxy_getter_functions(self): From 3c3ddaff09a346680c4bcfceb52fb5db0e690d1b Mon Sep 17 00:00:00 2001 From: incoding Date: Wed, 14 Nov 2018 13:17:00 +0800 Subject: [PATCH 031/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9B=B4=E5=8A=A0?= =?UTF-8?q?=E4=B8=A5=E8=B0=A8=E7=9A=84=E4=BB=A3=E7=90=86=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E8=A7=84=E5=88=99=EF=BC=88=E4=B8=80=E4=BA=9B=E9=9D=9E=E6=B3=95?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E4=B9=9F=E4=BC=9A=E8=BF=94=E5=9B=9E200?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Util/utilFunction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/utilFunction.py b/Util/utilFunction.py index fc26a59b1..ec86c1fe3 100644 --- a/Util/utilFunction.py +++ b/Util/utilFunction.py @@ -100,7 +100,7 @@ def validUsefulProxy(proxy): try: # 超过20秒的代理就不要了 r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False) - if r.status_code == 200: + if r.status_code == 200 and r.headers['content-type'].lower().find('application/json') != -1 and r.json()['origin']: # logger.info('%s is ok' % proxy) return True except Exception as e: From e5c1b89c919bae95fcb14e715d7b2e91115dfbe3 Mon Sep 17 00:00:00 2001 From: incoding Date: Wed, 14 Nov 2018 13:33:47 +0800 Subject: [PATCH 032/354] =?UTF-8?q?=E6=B7=BB=E5=8A=A0my=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=B9=EF=BC=8C=E4=BF=9D=E5=AD=98=E5=AE=9A=E5=88=B6=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my/Config.ini | 37 +++++++++++++++++++++++++++++++++++++ my/Dockerfile | 16 ++++++++++++++++ my/build.sh | 1 + my/run.sh | 14 ++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 my/Config.ini create mode 100644 my/Dockerfile create mode 100755 my/build.sh create mode 100755 my/run.sh diff --git a/my/Config.ini b/my/Config.ini new file mode 100644 index 000000000..627092c1a --- /dev/null +++ b/my/Config.ini @@ -0,0 +1,37 @@ +[DB] +;Configure the database information +;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB +type = PROXY_POOL_DB_TYPE +host = PROXY_POOL_DB_HOST +port = PROXY_POOL_DB_PORT +name = proxy +;username = your_username (Only Mongodb) +;password = your_password + +[ProxyGetter] +;register the proxy getter function +freeProxyFirst = 1 +freeProxySecond = 1 +;freeProxyThird = 1 +freeProxyFourth = 1 +freeProxyFifth = 1 +;freeProxySixth = 1 +freeProxySeventh = 1 +;freeProxyEight = 1 +;freeProxyNinth = 1 +freeProxyTen = 1 +freeProxyEleven = 1 +freeProxyTwelve = 1 +;foreign website, outside the wall +;freeProxyWallFirst = 1 +;freeProxyWallSecond = 1 +;freeProxyWallThird = 1 + +[API] +# API config http://127.0.0.1:5010 +# The ip specified when starting the web API +ip = 0.0.0.0 +# he port on which to run the web API +port = 8080 +# Flask processes option +processes = 10 diff --git a/my/Dockerfile b/my/Dockerfile new file mode 100644 index 000000000..cb042627c --- /dev/null +++ b/my/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.6 + +WORKDIR /usr/src/app + +ENV TZ=Asia/Shanghai \ + PROXY_POOL_DB_TYPE=SSDB \ + PROXY_POOL_DB_HOST=redis \ + PROXY_POOL_DB_PORT=6379 + +COPY . . + +RUN pip install --no-cache-dir -r requirements.txt && cp my/Config.ini ./ + +CMD [ "my/run.sh" ] + +EXPOSE 8080 diff --git a/my/build.sh b/my/build.sh new file mode 100755 index 000000000..328e9449d --- /dev/null +++ b/my/build.sh @@ -0,0 +1 @@ +docker build -t registry.cn-beijing.aliyuncs.com/ryttech/proxy_pool:1.12.20181114 -f my/Dockerfile . \ No newline at end of file diff --git a/my/run.sh b/my/run.sh new file mode 100755 index 000000000..441ace853 --- /dev/null +++ b/my/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +for var in \ + PROXY_POOL_DB_TYPE \ + PROXY_POOL_DB_HOST \ + PROXY_POOL_DB_PORT \ +; do + val="${!var}" + if [ "$val" ]; then + sed -ri "s/$var/$val/" Config.ini + fi +done + +python Run/main.py \ No newline at end of file From 110b0df1e29529346314378155890d740064ca0b Mon Sep 17 00:00:00 2001 From: jhao Date: Wed, 14 Nov 2018 16:51:41 +0800 Subject: [PATCH 033/354] Merge branch 'jhao104/master' of https://github.com/1again/proxy_pool into 1again-jhao104/master # Conflicts: # DB/DbClient.py # Util/GetConfig.py --- Api/ProxyApi.py | 5 +---- Config.ini | 4 +--- Schedule/ProxyValidSchedule.py | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index 99a0953a0..fc759a363 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -84,10 +84,7 @@ def getStatus(): def run(): - if sys.platform.startswith("win"): - app.run(host=config.host_ip, port=config.host_port) - else: - app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes) + app.run(host=config.host_ip, port=config.host_port) if __name__ == '__main__': diff --git a/Config.ini b/Config.ini index c8a9cc266..5bdf095a1 100644 --- a/Config.ini +++ b/Config.ini @@ -32,6 +32,4 @@ freeProxyTwelve = 1 # The ip specified when starting the web API ip = 0.0.0.0 # he port on which to run the web API -port = 5010 -# Flask processes option -processes = 10 +port = 8080 diff --git a/Schedule/ProxyValidSchedule.py b/Schedule/ProxyValidSchedule.py index 9b075cf90..098c8a336 100644 --- a/Schedule/ProxyValidSchedule.py +++ b/Schedule/ProxyValidSchedule.py @@ -32,7 +32,7 @@ def __init__(self): self.queue = Queue() self.proxy_item = dict() - def __validProxy(self, threads=10): + def __validProxy(self, threads=20): """ 验证useful_proxy代理 :param threads: 线程数 From a3ba910f391fd0220f357f926ef2b5ab6e0a973f Mon Sep 17 00:00:00 2001 From: windhw Date: Thu, 6 Dec 2018 12:48:10 +0800 Subject: [PATCH 034/354] Update main.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加对SIGTERM的处理，这样在后台运行的时候，如果kill掉主进程，子进程也能kill --- Run/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Run/main.py b/Run/main.py index fcd84f6f4..cce7b6142 100644 --- a/Run/main.py +++ b/Run/main.py @@ -12,7 +12,7 @@ """ __author__ = 'JHao' -import sys +import sys，signal from multiprocessing import Process sys.path.append('.') @@ -31,6 +31,14 @@ def run(): p_list.append(p2) p3 = Process(target=RefreshRun, name='RefreshRun') p_list.append(p3) + + def kill_child_processes(signum,frame): + for p in p_list: + p.terminate() + sys.exit(1) + + signal.signal(signal.SIGTERM, kill_child_processes) + for p in p_list: p.daemon = True From 2260c6d02f2374d7b4952787cac964f648ffd2b2 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 7 Dec 2018 14:21:51 +0800 Subject: [PATCH 035/354] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B0httpbin?= =?UTF-8?q?=E6=A3=80=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Util/utilFunction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/utilFunction.py b/Util/utilFunction.py index ec86c1fe3..f4e802263 100644 --- a/Util/utilFunction.py +++ b/Util/utilFunction.py @@ -100,7 +100,7 @@ def validUsefulProxy(proxy): try: # 超过20秒的代理就不要了 r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False) - if r.status_code == 200 and r.headers['content-type'].lower().find('application/json') != -1 and r.json()['origin']: + if r.status_code == 200 and r.json().get("origin"): # logger.info('%s is ok' % proxy) return True except Exception as e: From 26aaf1851a5b9bf4bc84ab344835d37d857ab6d7 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 7 Dec 2018 14:23:49 +0800 Subject: [PATCH 036/354] =?UTF-8?q?=E3=80=90del=E3=80=91delete=20un=20use?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my/Config.ini | 37 ------------------------------------- my/Dockerfile | 16 ---------------- my/build.sh | 1 - my/run.sh | 14 -------------- 4 files changed, 68 deletions(-) delete mode 100644 my/Config.ini delete mode 100644 my/Dockerfile delete mode 100755 my/build.sh delete mode 100755 my/run.sh diff --git a/my/Config.ini b/my/Config.ini deleted file mode 100644 index 627092c1a..000000000 --- a/my/Config.ini +++ /dev/null @@ -1,37 +0,0 @@ -[DB] -;Configure the database information -;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB -type = PROXY_POOL_DB_TYPE -host = PROXY_POOL_DB_HOST -port = PROXY_POOL_DB_PORT -name = proxy -;username = your_username (Only Mongodb) -;password = your_password - -[ProxyGetter] -;register the proxy getter function -freeProxyFirst = 1 -freeProxySecond = 1 -;freeProxyThird = 1 -freeProxyFourth = 1 -freeProxyFifth = 1 -;freeProxySixth = 1 -freeProxySeventh = 1 -;freeProxyEight = 1 -;freeProxyNinth = 1 -freeProxyTen = 1 -freeProxyEleven = 1 -freeProxyTwelve = 1 -;foreign website, outside the wall -;freeProxyWallFirst = 1 -;freeProxyWallSecond = 1 -;freeProxyWallThird = 1 - -[API] -# API config http://127.0.0.1:5010 -# The ip specified when starting the web API -ip = 0.0.0.0 -# he port on which to run the web API -port = 8080 -# Flask processes option -processes = 10 diff --git a/my/Dockerfile b/my/Dockerfile deleted file mode 100644 index cb042627c..000000000 --- a/my/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.6 - -WORKDIR /usr/src/app - -ENV TZ=Asia/Shanghai \ - PROXY_POOL_DB_TYPE=SSDB \ - PROXY_POOL_DB_HOST=redis \ - PROXY_POOL_DB_PORT=6379 - -COPY . . - -RUN pip install --no-cache-dir -r requirements.txt && cp my/Config.ini ./ - -CMD [ "my/run.sh" ] - -EXPOSE 8080 diff --git a/my/build.sh b/my/build.sh deleted file mode 100755 index 328e9449d..000000000 --- a/my/build.sh +++ /dev/null @@ -1 +0,0 @@ -docker build -t registry.cn-beijing.aliyuncs.com/ryttech/proxy_pool:1.12.20181114 -f my/Dockerfile . \ No newline at end of file diff --git a/my/run.sh b/my/run.sh deleted file mode 100755 index 441ace853..000000000 --- a/my/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -for var in \ - PROXY_POOL_DB_TYPE \ - PROXY_POOL_DB_HOST \ - PROXY_POOL_DB_PORT \ -; do - val="${!var}" - if [ "$val" ]; then - sed -ri "s/$var/$val/" Config.ini - fi -done - -python Run/main.py \ No newline at end of file From 223f57d1eb8d243b1d69e28b90a39f0529ec4407 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 7 Dec 2018 15:29:55 +0800 Subject: [PATCH 037/354] [fix] fix password --- Util/GetConfig.py | 9 ++------- Util/utilClass.py | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Util/GetConfig.py b/Util/GetConfig.py index cd354e20f..c25035504 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -26,7 +26,7 @@ class GetConfig(object): def __init__(self): self.pwd = os.path.split(os.path.realpath(__file__))[0] self.config_path = os.path.join(os.path.split(self.pwd)[0], 'Config.ini') - self.config_file = ConfigParse() + self.config_file = ConfigParse(defaults={"password": None}) self.config_file.read(self.config_path) @LazyProperty @@ -47,7 +47,7 @@ def db_port(self): @LazyProperty def db_password(self): - return self.config_file.get('DB', 'password', fallback=None) + return self.config_file.get('DB', 'password') @LazyProperty def proxy_getter_functions(self): @@ -61,10 +61,6 @@ def host_ip(self): def host_port(self): return int(self.config_file.get('API', 'port')) - @LazyProperty - def processes(self): - return int(self.config_file.get('API', 'processes')) - config = GetConfig() @@ -77,5 +73,4 @@ def processes(self): print(gg.proxy_getter_functions) print(gg.host_ip) print(gg.host_port) - print(gg.processes) print(gg.db_password) diff --git a/Util/utilClass.py b/Util/utilClass.py index 89112ffd8..b3a35f141 100644 --- a/Util/utilClass.py +++ b/Util/utilClass.py @@ -44,8 +44,8 @@ class ConfigParse(ConfigParser): rewrite ConfigParser, for support upper option """ - def __init__(self): - ConfigParser.__init__(self) + def __init__(self, *args, **kwargs): + ConfigParser.__init__(self, *args, **kwargs) def optionxform(self, optionstr): return optionstr From d49a66a6a1051e2eb86231e03a6a0ab3875dee1e Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 15 Feb 2019 16:02:02 +0800 Subject: [PATCH 038/354] =?UTF-8?q?[update]=20=E4=BD=BF=E7=94=A8setting.py?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2Config.ini=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Api/ProxyApi.py | 2 +- Config.ini | 2 +- Config/ConfigGetter.py | 71 ++++++++++++++++++++++++ {Test => Config}/__init__.py | 11 ++-- Config/setting.py | 54 ++++++++++++++++++ DB/DbClient.py | 2 +- DB/SsdbClient.py | 4 +- Manager/ProxyManager.py | 2 +- Schedule/ProxyValidSchedule.py | 4 +- Test/.pytest_cache/v/cache/lastfailed | 3 - Test/.pytest_cache/v/cache/nodeids | 3 - Test/{testGetConfig.py => testConfig.py} | 22 ++++---- Util/GetConfig.py | 7 +-- Util/utilClass.py | 19 ------- test.py | 5 +- 15 files changed, 154 insertions(+), 57 deletions(-) create mode 100644 Config/ConfigGetter.py rename {Test => Config}/__init__.py (56%) create mode 100644 Config/setting.py delete mode 100644 Test/.pytest_cache/v/cache/lastfailed delete mode 100644 Test/.pytest_cache/v/cache/nodeids rename Test/{testGetConfig.py => testConfig.py} (60%) diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py index fc759a363..91df76f88 100644 --- a/Api/ProxyApi.py +++ b/Api/ProxyApi.py @@ -19,7 +19,7 @@ sys.path.append('../') -from Util.GetConfig import config +from Config.ConfigGetter import config from Manager.ProxyManager import ProxyManager app = Flask(__name__) diff --git a/Config.ini b/Config.ini index 5bdf095a1..ee13eaf2c 100644 --- a/Config.ini +++ b/Config.ini @@ -1,6 +1,6 @@ [DB] ;Configure the database information -;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB +;type: SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB type = SSDB host = 127.0.0.1 port = 8888 diff --git a/Config/ConfigGetter.py b/Config/ConfigGetter.py new file mode 100644 index 000000000..56c766c0d --- /dev/null +++ b/Config/ConfigGetter.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： ConfigGetter + Description : 读取配置 + Author : JHao + date： 2019/2/15 +------------------------------------------------- + Change Activity: + 2019/2/15: +------------------------------------------------- +""" +__author__ = 'JHao' + + +from Util.utilClass import LazyProperty +from Config.setting import * + + +class ConfigGetter(object): + """ + get config + """ + + def __init__(self): + pass + + @LazyProperty + def db_type(self): + return DATABASES.get("default", {}).get("TYPE", "SSDB") + + @LazyProperty + def db_name(self): + return DATABASES.get("default", {}).get("NAME", "proxy") + + @LazyProperty + def db_host(self): + return DATABASES.get("default", {}).get("HOST", "127.0.0.1") + + @LazyProperty + def db_port(self): + return DATABASES.get("default", {}).get("PORT", 8080) + + @LazyProperty + def db_password(self): + return DATABASES.get("default", {}).get("PASSWORD", "") + + @LazyProperty + def proxy_getter_functions(self): + return PROXY_GETTER + + @LazyProperty + def host_ip(self): + return SERVER_API.get("HOST", "127.0.0.1") + + @LazyProperty + def host_port(self): + return SERVER_API.get("PORT", 5010) + + +config = ConfigGetter() + +if __name__ == '__main__': + print(config.db_type) + print(config.db_name) + print(config.db_host) + print(config.db_port) + print(config.proxy_getter_functions) + print(config.host_ip) + print(config.host_port) + print(config.db_password) diff --git a/Test/__init__.py b/Config/__init__.py similarity index 56% rename from Test/__init__.py rename to Config/__init__.py index 898942953..9a7d547ee 100644 --- a/Test/__init__.py +++ b/Config/__init__.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------- - File Name： __init__.py - Description : - Author : J_hao - date： 2017/7/31 + File Name： __init__ + Description : + Author : JHao + date： 2019/2/15 ------------------------------------------------- Change Activity: - 2017/7/31: + 2019/2/15: ------------------------------------------------- """ -__author__ = 'J_hao' diff --git a/Config/setting.py b/Config/setting.py new file mode 100644 index 000000000..39ae36748 --- /dev/null +++ b/Config/setting.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： setting.py + Description : 配置文件 + Author : JHao + date： 2019/2/15 +------------------------------------------------- + Change Activity: + 2019/2/15: +------------------------------------------------- +""" + +# database config + +DATABASES = { + "default": { + "TYPE": "SSDB", # TYPE SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB + "HOST": "127.0.0.1", + "PORT": 8888, + "NAME": "proxy", + "PASSWORD": "" + + } +} + +# register the proxy getter function + +PROXY_GETTER = [ + "freeProxyFirst", + "freeProxySecond", + # "freeProxyThird", + "freeProxyFourth", + "freeProxyFifth", + # "freeProxySixth" + "freeProxySeventh", + # "freeProxyEight", + # "freeProxyNinth", + "freeProxyTen", + "freeProxyEleven", + "freeProxyTwelve", + # foreign website, outside the wall + "freeProxyWallFirst", + "freeProxyWallSecond", + "freeProxyWallThird" +] + + +# # API config http://127.0.0.1:5010 + +SERVER_API = { + "HOST": "0.0.0.0", # The ip specified which starting the web API + "PORT": 5010 # port number to which the server listens to +} \ No newline at end of file diff --git a/DB/DbClient.py b/DB/DbClient.py index f79fc8511..baa1f79fc 100644 --- a/DB/DbClient.py +++ b/DB/DbClient.py @@ -16,7 +16,7 @@ import os import sys -from Util.GetConfig import config +from Config.ConfigGetter import config from Util.utilClass import Singleton sys.path.append(os.path.dirname(os.path.abspath(__file__))) diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py index 4ceedd1df..85545b355 100644 --- a/DB/SsdbClient.py +++ b/DB/SsdbClient.py @@ -3,7 +3,7 @@ """ ------------------------------------------------- File Name： SsdbClient.py - Description : 封装SSDB操作 + Description : 封装SSDB/Redis操作 Author : JHao date： 2016/12/2 ------------------------------------------------- @@ -27,7 +27,7 @@ class SsdbClient(object): SSDB client SSDB中代理存放的容器为hash： - 原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为为None,以后扩展可能会加入代理属性； + 原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为None,以后扩展可能会加入代理属性； 验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1； """ diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py index c770b6224..fd007773b 100644 --- a/Manager/ProxyManager.py +++ b/Manager/ProxyManager.py @@ -17,7 +17,7 @@ from Util import EnvUtil from DB.DbClient import DbClient -from Util.GetConfig import config +from Config.ConfigGetter import config from Util.LogHandler import LogHandler from Util.utilFunction import verifyProxyFormat from ProxyGetter.getFreeProxy import GetFreeProxy diff --git a/Schedule/ProxyValidSchedule.py b/Schedule/ProxyValidSchedule.py index 098c8a336..6b1fa6485 100644 --- a/Schedule/ProxyValidSchedule.py +++ b/Schedule/ProxyValidSchedule.py @@ -56,8 +56,8 @@ def main(self): self.log.info("Start valid useful proxy") self.__validProxy() else: - self.log.info('Valid Complete! sleep 5 minutes.') - time.sleep(60 * 5) + self.log.info('Valid Complete! sleep 5 sec.') + time.sleep(5) self.putQueue() def putQueue(self): diff --git a/Test/.pytest_cache/v/cache/lastfailed b/Test/.pytest_cache/v/cache/lastfailed deleted file mode 100644 index 65c9a06d6..000000000 --- a/Test/.pytest_cache/v/cache/lastfailed +++ /dev/null @@ -1,3 +0,0 @@ -{ - "testGetFreeProxy.py::testGetFreeProxy": true -} \ No newline at end of file diff --git a/Test/.pytest_cache/v/cache/nodeids b/Test/.pytest_cache/v/cache/nodeids deleted file mode 100644 index 0ce3684ce..000000000 --- a/Test/.pytest_cache/v/cache/nodeids +++ /dev/null @@ -1,3 +0,0 @@ -[ - "testGetFreeProxy.py::testGetFreeProxy" -] \ No newline at end of file diff --git a/Test/testGetConfig.py b/Test/testConfig.py similarity index 60% rename from Test/testGetConfig.py rename to Test/testConfig.py index 7f44fa6b4..7ed759387 100644 --- a/Test/testGetConfig.py +++ b/Test/testConfig.py @@ -12,22 +12,22 @@ """ __author__ = 'J_hao' -from Util.GetConfig import GetConfig +from Config.ConfigGetter import config # noinspection PyPep8Naming -def testGetConfig(): +def testConfig(): """ - test class GetConfig in Util/GetConfig :return: """ - gg = GetConfig() - print(gg.db_type) - print(gg.db_name) - print(gg.db_host) - print(gg.db_port) - assert isinstance(gg.proxy_getter_functions, list) - print(gg.proxy_getter_functions) + print(config.db_type) + print(config.db_name) + print(config.db_host) + print(config.db_port) + print(config.db_password) + assert isinstance(config.proxy_getter_functions, list) + print(config.proxy_getter_functions) + if __name__ == '__main__': - testGetConfig() + testConfig() diff --git a/Util/GetConfig.py b/Util/GetConfig.py index c25035504..65554b317 100644 --- a/Util/GetConfig.py +++ b/Util/GetConfig.py @@ -13,8 +13,6 @@ """ __author__ = 'JHao' -import os -from Util.utilClass import ConfigParse from Util.utilClass import LazyProperty @@ -24,10 +22,7 @@ class GetConfig(object): """ def __init__(self): - self.pwd = os.path.split(os.path.realpath(__file__))[0] - self.config_path = os.path.join(os.path.split(self.pwd)[0], 'Config.ini') - self.config_file = ConfigParse(defaults={"password": None}) - self.config_file.read(self.config_path) + pass @LazyProperty def db_type(self): diff --git a/Util/utilClass.py b/Util/utilClass.py index b3a35f141..cffe72443 100644 --- a/Util/utilClass.py +++ b/Util/utilClass.py @@ -9,7 +9,6 @@ ------------------------------------------------- Change Activity: 2016/12/3: Class LazyProperty - 2016/12/4: rewrite ConfigParser ------------------------------------------------- """ __author__ = 'JHao' @@ -33,24 +32,6 @@ def __get__(self, instance, owner): return value -try: - from configparser import ConfigParser # py3 -except: - from ConfigParser import ConfigParser # py2 - - -class ConfigParse(ConfigParser): - """ - rewrite ConfigParser, for support upper option - """ - - def __init__(self, *args, **kwargs): - ConfigParser.__init__(self, *args, **kwargs) - - def optionxform(self, optionstr): - return optionstr - - class Singleton(type): """ Singleton Metaclass diff --git a/test.py b/test.py index 518710d3b..d636535a9 100644 --- a/test.py +++ b/test.py @@ -12,4 +12,7 @@ """ __author__ = 'JHao' -from Schedule import ProxyRefreshSchedule \ No newline at end of file +from Test import testConfig + +if __name__ == '__main__': + testConfig.testConfig() From 2b54d4af03c96515198fada0ee630cf98ea52cf9 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 15 Feb 2019 16:06:33 +0800 Subject: [PATCH 039/354] =?UTF-8?q?[update]=20=E4=BD=BF=E7=94=A8setting.py?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2Config.ini=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Test/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 Test/__init__.py diff --git a/Test/__init__.py b/Test/__init__.py new file mode 100644 index 000000000..9b16c75ff --- /dev/null +++ b/Test/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： __init__ + Description : + Author : JHao + date： 2019/2/15 +------------------------------------------------- + Change Activity: + 2019/2/15: +------------------------------------------------- +""" +__author__ = 'JHao' \ No newline at end of file From f00a4569d26ef963656cf9b7617cec9f8780e666 Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 15 Feb 2019 16:24:37 +0800 Subject: [PATCH 040/354] =?UTF-8?q?[update]=20=E4=BD=BF=E7=94=A8setting.py?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2Config.ini=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 61 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 8bdca40c5..4f253af81 100644 --- a/README.md +++ b/README.md @@ -39,25 +39,41 @@ git clone git@github.com:jhao104/proxy_pool.git pip install -r requirements.txt ``` -* 配置Config.ini: +* 配置Config/setting.py: ```shell -# Config.ini 为项目配置文件 -# 配置DB -type = SSDB # 如果使用SSDB或redis数据库，均配置为SSDB -host = localhost # db host -port = 8888 # db port -name = proxy # 默认配置 +# Config/setting.py 为项目配置文件 + +# 配置DB +DATABASES = { + "default": { + "TYPE": "SSDB", # 如果使用SSDB或redis数据库，均配置为SSDB + "HOST": "127.0.0.1", # db host + "PORT": 8888, # db port + "NAME": "proxy", # 默认配置 + "PASSWORD": "" # db password + + } +} + # 配置 ProxyGetter -freeProxyFirst = 1 # 这里是启动的抓取函数，可在ProxyGetter/getFreeProxy.py 扩展 -freeProxySecond = 1 -.... -# 配置 HOST (api服务) -ip = 127.0.0.1 # 监听ip,0.0.0.0开启外网访问 -port = 5010 # 监听端口 -# 上面配置启动后，代理api地址为 http://127.0.0.1:5010 +PROXY_GETTER = [ + "freeProxyFirst", # 这里是启用的代理抓取函数名，可在ProxyGetter/getFreeProxy.py 扩展 + "freeProxySecond", + .... +] + + +# 配置 API服务 + +SERVER_API = { + "HOST": "0.0.0.0", # 监听ip, 0.0.0.0 监听所有IP + "PORT": 5010 # 监听端口 +} + +# 上面配置启动后，代理池访问地址为 http://127.0.0.1:5010 ``` @@ -164,18 +180,17 @@ class GetFreeProxy(object): # 确保每个proxy都是 host:ip正确的格式就行 ``` -* 2、添加好方法后，修改Config.ini文件中的`[ProxyGetter]`项： +* 2、添加好方法后，修改Config/setting.py文件中的`PROXY_GETTER`项： -　　在`Config.ini`的`[ProxyGetter]`下添加自定义的方法的名字: +　　在`PROXY_GETTER`下添加自定义的方法的名字: ```shell - -[ProxyGetter] -;register the proxy getter function -freeProxyFirst = 0 # 如果要取消某个方法，将其删除或赋为0即可 -.... -freeProxyCustom = 1 # 确保名字和你添加方法名字一致 - +PROXY_GETTER = [ + "freeProxyFirst", + "freeProxySecond", + .... + "freeProxyCustom" # # 确保名字和你添加方法名字一致 +] ``` From 16c5a04ba43c05608261581a6affeee1a9d1728f Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 15 Feb 2019 16:29:33 +0800 Subject: [PATCH 041/354] =?UTF-8?q?[update]=20=E4=BD=BF=E7=94=A8setting.py?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2Config.ini=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config.ini | 35 -------------------- Config/setting.py | 2 +- Test/testConfig.py | 2 +- Test/testGetFreeProxy.py | 11 +++---- Util/GetConfig.py | 71 ---------------------------------------- 5 files changed, 7 insertions(+), 114 deletions(-) delete mode 100644 Config.ini delete mode 100644 Util/GetConfig.py diff --git a/Config.ini b/Config.ini deleted file mode 100644 index ee13eaf2c..000000000 --- a/Config.ini +++ /dev/null @@ -1,35 +0,0 @@ -[DB] -;Configure the database information -;type: SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB -type = SSDB -host = 127.0.0.1 -port = 8888 -name = proxy -;username = your_username (Only Mongodb) -;password = your_password - -[ProxyGetter] -;register the proxy getter function -freeProxyFirst = 1 -freeProxySecond = 1 -;freeProxyThird = 1 -freeProxyFourth = 1 -freeProxyFifth = 1 -;freeProxySixth = 1 -freeProxySeventh = 1 -;freeProxyEight = 1 -;freeProxyNinth = 1 -freeProxyTen = 1 -freeProxyEleven = 1 -freeProxyTwelve = 1 -;foreign website, outside the wall -;freeProxyWallFirst = 1 -;freeProxyWallSecond = 1 -;freeProxyWallThird = 1 - -[API] -# API config http://127.0.0.1:5010 -# The ip specified when starting the web API -ip = 0.0.0.0 -# he port on which to run the web API -port = 8080 diff --git a/Config/setting.py b/Config/setting.py index 39ae36748..8b87191fa 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -51,4 +51,4 @@ SERVER_API = { "HOST": "0.0.0.0", # The ip specified which starting the web API "PORT": 5010 # port number to which the server listens to -} \ No newline at end of file +} diff --git a/Test/testConfig.py b/Test/testConfig.py index 7ed759387..ebfd1171f 100644 --- a/Test/testConfig.py +++ b/Test/testConfig.py @@ -2,7 +2,7 @@ """ ------------------------------------------------- File Name： testGetConfig - Description : test all function in GetConfig.py + Description : testGetConfig Author : J_hao date： 2017/7/31 ------------------------------------------------- diff --git a/Test/testGetFreeProxy.py b/Test/testGetFreeProxy.py index 33c3f9e46..854172773 100644 --- a/Test/testGetFreeProxy.py +++ b/Test/testGetFreeProxy.py @@ -16,7 +16,6 @@ import sys import requests - try: from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 except: @@ -25,7 +24,7 @@ sys.path.append('..') from ProxyGetter.getFreeProxy import GetFreeProxy -from Util.GetConfig import GetConfig +from Config.ConfigGetter import config # noinspection PyPep8Naming @@ -34,15 +33,15 @@ def testGetFreeProxy(): test class GetFreeProxy in ProxyGetter/GetFreeProxy :return: """ - gc = GetConfig() - proxy_getter_functions = gc.proxy_getter_functions + proxy_getter_functions = config.proxy_getter_functions for proxyGetter in proxy_getter_functions: proxy_count = 0 for proxy in getattr(GetFreeProxy, proxyGetter.strip())(): if proxy: - print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,proxy_count=proxy_count)) + print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy, + proxy_count=proxy_count)) proxy_count += 1 - #assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter) + # assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter) if __name__ == '__main__': diff --git a/Util/GetConfig.py b/Util/GetConfig.py deleted file mode 100644 index 65554b317..000000000 --- a/Util/GetConfig.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： GetConfig.py - Description : fetch config from config.ini - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: get db property func -------------------------------------------------- -""" -__author__ = 'JHao' - -from Util.utilClass import LazyProperty - - -class GetConfig(object): - """ - to get config from config.ini - """ - - def __init__(self): - pass - - @LazyProperty - def db_type(self): - return self.config_file.get('DB', 'type') - - @LazyProperty - def db_name(self): - return self.config_file.get('DB', 'name') - - @LazyProperty - def db_host(self): - return self.config_file.get('DB', 'host') - - @LazyProperty - def db_port(self): - return int(self.config_file.get('DB', 'port')) - - @LazyProperty - def db_password(self): - return self.config_file.get('DB', 'password') - - @LazyProperty - def proxy_getter_functions(self): - return self.config_file.options('ProxyGetter') - - @LazyProperty - def host_ip(self): - return self.config_file.get('API', 'ip') - - @LazyProperty - def host_port(self): - return int(self.config_file.get('API', 'port')) - - -config = GetConfig() - -if __name__ == '__main__': - gg = GetConfig() - print(gg.db_type) - print(gg.db_name) - print(gg.db_host) - print(gg.db_port) - print(gg.proxy_getter_functions) - print(gg.host_ip) - print(gg.host_port) - print(gg.db_password) From 55e71981168e57658371e27f7b9517011cca653f Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 18 Feb 2019 10:53:03 +0800 Subject: [PATCH 042/354] =?UTF-8?q?[update]=20=E4=BD=BF=E7=94=A8setting.py?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2Config.ini=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 4f253af81..b62864f2d 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,27 @@ PROXY_GETTER = [ 　　`ProxyRefreshSchedule`会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。 +### 代理采集 + + 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): + + | 厂商名称 | 状态 | 更新速度 | 可用率 | 是否被墙 | 地址 | + | ----- | ---- | -------- | ------ | --------- | ----- | + | 无忧代理 | 可用 | 几分钟一次 | * | 否 | [地址](http://www.data5u.com/free/index.html) | + | 66代理 | 可用 | 更新很慢 | * | 否 | [地址](http://www.66ip.cn/) | + | 西刺代理 | 可用 | 几分钟一次 | * | 否 | [地址](http://www.xicidaili.com)| + | 全网代理 | 可用 | 几分钟一次 | * | 否 | [地址](http://www.goubanjia.com/)| + | 训代理 | 已关闭免费代理 | * | * | 否 | [地址](http://www.xdaili.cn/)| + | 快代理 | 可用 |几分钟一次| * | 否 | [地址](https://www.kuaidaili.com/)| + | 云代理 | 可用 |几分钟一次| * | 否 | [地址](http://www.ip3366.net/)| + | IP海 | 可用 |几小时一次| * | 否 | [地址](http://www.iphai.com/)| + | 免费IP代理库 | 可用 |快| * | 否 | [地址](http://ip.jiangxianli.com/)| + | 中国IP地址 | 可用 |几分钟一次| * | 是 | [地址](http://cn-proxy.com/)| + | Proxy List | 可用 |几分钟一次| * | 是 | [地址](https://proxy-list.org/chinese/index.php)| + | ProxyList+ | 可用 |几分钟一次| * | 是 | [地址](https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1)| + + 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 + ### 问题反馈 　　任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈，如果没有账号可以去 我的[博客](http://www.spiderpy.cn/blog/message)中留言。 From 086074c4288167871a3c23b34346ab59db01f29c Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 18 Feb 2019 11:17:38 +0800 Subject: [PATCH 043/354] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B066?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E9=87=87=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/getFreeProxy.py | 48 ++++++++++++++----------------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index a560dc700..caa5b6e9c 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -23,18 +23,6 @@ # for debug to disable insecureWarning requests.packages.urllib3.disable_warnings() -""" - 66ip.cn - data5u.com - xicidaili.com - goubanjia.com - xdaili.cn - kuaidaili.com - cn-proxy.com - proxy-list.org - www.mimiip.com to do -""" - class GetFreeProxy(object): """ @@ -64,24 +52,24 @@ def freeProxyFirst(page=10): print(e) @staticmethod - def freeProxySecond(area=33, page=1): + def freeProxySecond(count=20): """ 代理66 http://www.66ip.cn/ - :param area: 抓取代理页数，page=1北京代理页，page=2上海代理页...... - :param page: 翻页 + :param count: 提取数量 :return: """ - area = 33 if area > 33 else area - for area_index in range(1, area + 1): - for i in range(1, page + 1): - url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i) - html_tree = getHtmlTree(url) - tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]") - if len(tr_list) == 0: - continue - for tr in tr_list: - yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0] - break + urls = [ + "http://www.66ip.cn/mo.php?sxb=&tqsl={count}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=", + "http://www.66ip.cn/nmtq.php?getnum={count}" + "&isp=0&anonymoustype=0&start=&ports=&export=&ipaddress=&area=1&proxytype=2&api=66ip", + ] + request = WebRequest() + for _ in urls: + url = _.format(count=count) + html = request.get(url).content + ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html) + for ip in ips: + yield ip.strip() @staticmethod def freeProxyThird(days=1): @@ -180,7 +168,7 @@ def freeProxySeventh(): @staticmethod def freeProxyEight(): """ - 秘密代理 http://www.mimiip.com + 秘密代理 http://www.mimiip.com 不能用 """ url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 2)] # 国内高匿 url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 2)] # 国内普匿 @@ -197,7 +185,7 @@ def freeProxyEight(): @staticmethod def freeProxyNinth(): """ - 码农代理 https://proxy.coderbusy.com/ + 码农代理 https://proxy.coderbusy.com/ 不能用 :return: """ urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] @@ -303,7 +291,7 @@ def freeProxyWallThird(): from CheckProxy import CheckProxy # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) - # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyThird) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFourth) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth) @@ -313,6 +301,6 @@ def freeProxyWallThird(): # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyNinth) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTen) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve) # CheckProxy.checkAllGetProxyFunc() From 792fd13e780205823e872d1370daa46a8b088e97 Mon Sep 17 00:00:00 2001 From: jhao Date: Mon, 18 Feb 2019 14:54:44 +0800 Subject: [PATCH 044/354] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B0=E4=BB=A3?= =?UTF-8?q?=E7=90=86IP=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config/setting.py | 10 +++++----- ProxyGetter/CheckProxy.py | 2 -- ProxyGetter/getFreeProxy.py | 28 +++++++++++++--------------- Test/testGetFreeProxy.py | 11 ----------- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/Config/setting.py b/Config/setting.py index 8b87191fa..63b4f6153 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -29,10 +29,10 @@ PROXY_GETTER = [ "freeProxyFirst", "freeProxySecond", - # "freeProxyThird", + # "freeProxyThird", # 网站已不能访问 "freeProxyFourth", "freeProxyFifth", - # "freeProxySixth" + # "freeProxySixth" # 不再提供免费代理 "freeProxySeventh", # "freeProxyEight", # "freeProxyNinth", @@ -40,9 +40,9 @@ "freeProxyEleven", "freeProxyTwelve", # foreign website, outside the wall - "freeProxyWallFirst", - "freeProxyWallSecond", - "freeProxyWallThird" + # "freeProxyWallFirst", + # "freeProxyWallSecond", + # "freeProxyWallThird" ] diff --git a/ProxyGetter/CheckProxy.py b/ProxyGetter/CheckProxy.py index f29824723..2b3fc6a29 100644 --- a/ProxyGetter/CheckProxy.py +++ b/ProxyGetter/CheckProxy.py @@ -12,11 +12,9 @@ """ __author__ = 'JHao' -import sys from getFreeProxy import GetFreeProxy from Util.utilFunction import verifyProxyFormat -sys.path.append('../') from Util.LogHandler import LogHandler diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index caa5b6e9c..cdfa843a0 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -88,7 +88,7 @@ def freeProxyThird(days=1): pass @staticmethod - def freeProxyFourth(page_count=2): + def freeProxyFourth(page_count=1): """ 西刺代理 http://www.xicidaili.com :return: @@ -136,7 +136,7 @@ def freeProxyFifth(): @staticmethod def freeProxySixth(): """ - 讯代理 http://www.xdaili.cn/ + 讯代理 http://www.xdaili.cn/ 已停用 :return: """ url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10' @@ -154,21 +154,19 @@ def freeProxySeventh(): 快代理 https://www.kuaidaili.com """ url_list = [ - 'https://www.kuaidaili.com/free/inha/{page}/', - 'https://www.kuaidaili.com/free/intr/{page}/' + 'https://www.kuaidaili.com/free/inha/', + 'https://www.kuaidaili.com/free/intr/' ] for url in url_list: - for page in range(1, 2): - page_url = url.format(page=page) - tree = getHtmlTree(page_url) - proxy_list = tree.xpath('.//table//tr') - for tr in proxy_list[1:]: - yield ':'.join(tr.xpath('./td/text()')[0:2]) + tree = getHtmlTree(url) + proxy_list = tree.xpath('.//table//tr') + for tr in proxy_list[1:]: + yield ':'.join(tr.xpath('./td/text()')[0:2]) @staticmethod def freeProxyEight(): """ - 秘密代理 http://www.mimiip.com 不能用 + 秘密代理 http://www.mimiip.com 已停用 """ url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 2)] # 国内高匿 url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 2)] # 国内普匿 @@ -185,7 +183,7 @@ def freeProxyEight(): @staticmethod def freeProxyNinth(): """ - 码农代理 https://proxy.coderbusy.com/ 不能用 + 码农代理 https://proxy.coderbusy.com/ 已停用 :return: """ urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] @@ -233,7 +231,7 @@ def freeProxyEleven(): @staticmethod def freeProxyTwelve(page_count=2): """ - guobanjia http://ip.jiangxianli.com/?page= + http://ip.jiangxianli.com/?page= 免费代理库 超多量 :return: @@ -291,7 +289,7 @@ def freeProxyWallThird(): from CheckProxy import CheckProxy # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyThird) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFourth) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth) @@ -300,7 +298,7 @@ def freeProxyWallThird(): # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEight) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyNinth) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTen) - # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve) # CheckProxy.checkAllGetProxyFunc() diff --git a/Test/testGetFreeProxy.py b/Test/testGetFreeProxy.py index 854172773..5074945b4 100644 --- a/Test/testGetFreeProxy.py +++ b/Test/testGetFreeProxy.py @@ -12,22 +12,11 @@ """ __author__ = 'J_hao' -import re -import sys -import requests -try: - from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 -except: - reload(sys) - sys.setdefaultencoding('utf-8') - -sys.path.append('..') from ProxyGetter.getFreeProxy import GetFreeProxy from Config.ConfigGetter import config -# noinspection PyPep8Naming def testGetFreeProxy(): """ test class GetFreeProxy in ProxyGetter/GetFreeProxy From 07f9845017836d2776272e87551b55fb4a677f1a Mon Sep 17 00:00:00 2001 From: jhao Date: Tue, 19 Feb 2019 15:24:23 +0800 Subject: [PATCH 045/354] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B0=E4=BB=A3?= =?UTF-8?q?=E7=90=86IP=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/release_notes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/release_notes.md b/doc/release_notes.md index 0871a2db5..36e097726 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -1,5 +1,11 @@ ## Release Notes +* 1.13 (2019.02) + + 1.使用.py文件替换.ini作为配置文件； + + 2.更新代理采集部分； + * 1.12 (2018.4) 1.优化代理格式检查; From 0c48d9dc1a0e3dcb2f166882ea29ed7ad3213a21 Mon Sep 17 00:00:00 2001 From: J_hao104 Date: Tue, 5 Mar 2019 10:05:06 +0800 Subject: [PATCH 046/354] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b62864f2d..48edb4c98 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ * 支持版本: ![](https://img.shields.io/badge/Python-2.x-green.svg) ![](https://img.shields.io/badge/Python-3.x-blue.svg) -* 测试地址: http://123.207.35.36:5010 (单机勿压。感谢) +* 测试地址: http://118.24.52.95:5010 (单机勿压。感谢) ### 下载安装 From b568bd2092fc4aa405314968ead1102b1216f18d Mon Sep 17 00:00:00 2001 From: weak_ptr Date: Sun, 10 Mar 2019 17:21:54 +0800 Subject: [PATCH 047/354] =?UTF-8?q?[refine]=20=E5=85=81=E8=AE=B8=20docker-?= =?UTF-8?q?compose=20up=20=E7=9B=B4=E6=8E=A5=E8=BF=90=E8=A1=8C=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E8=80=8C=E6=97=A0=E9=9C=80=E4=BF=AE=E6=94=B9=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以下为修改内容。 - 移除现在看起来无用的 Dockerfile.develop - 将 Dockerfile 和 docker-compose.yml 移动到项目根目录下，删除 Docker 目录 - 修改 docker-compose.yml 内容，令 docker-compose 自行构建 proxy_pool，通过环境变量传递数据库类型和域名、端口等配置信息，不再暴露 redis 端口到 host 主机 - 修改 Dockerfile 内容，先复制 requirements.txt，完成依赖安装后，再复制代码文件，避免开发迭代时每次都要等 pip install - 修改 Config.setting 模块，先尝试通过环境变量获取配置信息，并提供未配置环境变量时的默认值。 --- Config/setting.py | 24 ++++++++++++++++++++---- Docker/Dockerfile.develop | 27 --------------------------- Docker/docker-compose.yml | 14 -------------- Docker/Dockerfile => Dockerfile | 9 +++------ docker-compose.yml | 14 ++++++++++++++ requirements.txt | 3 --- 6 files changed, 37 insertions(+), 54 deletions(-) delete mode 100644 Docker/Dockerfile.develop delete mode 100644 Docker/docker-compose.yml rename Docker/Dockerfile => Dockerfile (89%) create mode 100644 docker-compose.yml diff --git a/Config/setting.py b/Config/setting.py index 63b4f6153..a74e69a32 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -12,12 +12,29 @@ """ # database config +from os import getenv + + +class ConfigError(BaseException): + pass + + +DB_TYPE = getenv('db_type', 'SSDB') + +if DB_TYPE == 'SSDB': + DB_HOST = getenv('ssdb_host', '127.0.0.1') + DB_PORT = getenv('ssdb_port', '6379') +elif DB_TYPE == 'MONGODB': + DB_HOST = getenv('mongodb_host', '127.0.0.1') + DB_PORT = getenv('mongodb_host', '27017') +else: + raise ConfigError('Unknown database type, your environment variable `db_type` should be one of SSDB/MONGODB.') DATABASES = { "default": { - "TYPE": "SSDB", # TYPE SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB - "HOST": "127.0.0.1", - "PORT": 8888, + "TYPE": DB_TYPE, # TYPE SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB + "HOST": DB_HOST, + "PORT": DB_PORT, "NAME": "proxy", "PASSWORD": "" @@ -45,7 +62,6 @@ # "freeProxyWallThird" ] - # # API config http://127.0.0.1:5010 SERVER_API = { diff --git a/Docker/Dockerfile.develop b/Docker/Dockerfile.develop deleted file mode 100644 index d97495489..000000000 --- a/Docker/Dockerfile.develop +++ /dev/null @@ -1,27 +0,0 @@ -FROM python:3.6 -WORKDIR /usr/src/app -COPY . . -ENV DEBIAN_FRONTEND noninteractive -ENV TZ Asia/Shanghai - -RUN apt-get update -RUN apt-get install vim -y - -RUN apt-get install -y redis-server -RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf \ - && sed -i 's/^\(databases .*\)$/databases 1/' /etc/redis/redis.conf \ - && sed -i 's/^\(daemonize .*\)$/daemonize yes/' /etc/redis/redis.conf -# && sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/redis/redis.conf \ -# && sed -i 's/^\(logfile .*\)$/# \1/' /etc/redis/redis.conf - -RUN pip install --no-cache-dir -r requirements.txt - - -RUN echo "# ! /bin/sh " > run.sh \ - && echo "redis-server /etc/redis/redis.conf&" >> run.sh \ - && echo "cd Run" >> run.sh \ - && echo "python main.py" >> run.sh \ - && chmod 777 run.sh - -EXPOSE 5010 -CMD [ "sh", "run.sh" ] diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml deleted file mode 100644 index 9529745d5..000000000 --- a/Docker/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '2' -services: - proxy_pool: - volumes: - - ..:/usr/src/app - ports: - - "5010:5010" - links: - - proxy_redis - image: "proxy_pool" - proxy_redis: - ports: - - "6379:6379" - image: "redis" \ No newline at end of file diff --git a/Docker/Dockerfile b/Dockerfile similarity index 89% rename from Docker/Dockerfile rename to Dockerfile index 6ad6f5f53..abe8ddb07 100644 --- a/Docker/Dockerfile +++ b/Dockerfile @@ -1,13 +1,10 @@ FROM python:3.6 -WORKDIR /usr/src/app -COPY . . - ENV DEBIAN_FRONTEND noninteractive ENV TZ Asia/Shanghai - +WORKDIR /usr/src/app +COPY ./requirements.txt . RUN pip install --no-cache-dir -r requirements.txt - +COPY . . EXPOSE 5010 - WORKDIR /usr/src/app/ CMD [ "python", "Run/main.py" ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..1c7f24659 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' +services: + proxy_pool: + build: . + ports: + - "5010:5010" + links: + - proxy_redis + environment: + db_type: SSDB + ssdb_host: proxy_redis + ssdb_port: 6379 + proxy_redis: + image: "redis" diff --git a/requirements.txt b/requirements.txt index bc3581ff5..3da935240 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,5 @@ werkzeug==0.11.15 Flask==0.12 requests==2.20.0 lxml==3.7.2 - pymongo redis - - From 595b08861abfa0e3a4e8dfa16132686292a5815c Mon Sep 17 00:00:00 2001 From: baiyan Date: Sun, 24 Mar 2019 00:42:18 +0800 Subject: [PATCH 048/354] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=A0=E5=BF=A7?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E8=A7=A3=E6=9E=90=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/getFreeProxy.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index cdfa843a0..470cbb3c2 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -33,7 +33,10 @@ class GetFreeProxy(object): def freeProxyFirst(page=10): """ 无忧代理 http://www.data5u.com/ - 几乎没有能用的 + 无忧代理有反爬虫机制。 + 需要获得元素的 classname。 + 匹配classname中每个字符在key中的位置，组合得到一个整数。 + 最后将整数右移3位得到的才是正确的端口号。 :param page: 页数 :return: """ @@ -42,12 +45,21 @@ def freeProxyFirst(page=10): 'http://www.data5u.com/free/gngn/index.shtml', 'http://www.data5u.com/free/gnpt/index.shtml' ] + key = 'ABCDEFGHIZ' for url in url_list: html_tree = getHtmlTree(url) ul_list = html_tree.xpath('//ul[@class="l2"]') for ul in ul_list: try: - yield ':'.join(ul.xpath('.//li/text()')[0:2]) + ip = ul.xpath('./span[1]/li/text()')[0] + classnames = ul.xpath('./span[2]/li/attribute::class')[0] + classname = classnames.split(' ')[1] + port_sum = 0 + for c in classname: + port_sum *= 10 + port_sum += key.index(c) + port = port_sum >> 3 + yield '{}:{}'.format(ip, port) except Exception as e: print(e) From 35467fb3bc8ac5c63b6939df84aa027f820f3421 Mon Sep 17 00:00:00 2001 From: Oddcc Date: Fri, 29 Mar 2019 14:19:19 +0800 Subject: [PATCH 049/354] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新文档中生产环境部署命令 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 48edb4c98..2bee689f4 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ SERVER_API = { # Workdir proxy_pool docker build -t proxy_pool . pip install docker-compose -docker-compose -f Docker/docker-compose.yml up -d +docker-compose -f docker-compose.yml up -d ``` * 开发环境 Docker From f8d039e61e0dc88ebfee43f96f9a584f07c9ca90 Mon Sep 17 00:00:00 2001 From: houbaron Date: Wed, 8 May 2019 21:40:11 +0800 Subject: [PATCH 050/354] =?UTF-8?q?[refine]=E5=85=81=E8=AE=B8=20docker-com?= =?UTF-8?q?pose.yml=20=E5=AE=9A=E4=B9=89=E5=AF=86=E7=A0=81=E8=80=8C?= =?UTF-8?q?=E6=97=A0=E9=A1=BB=E4=BF=AE=E6=94=B9=20setting.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config/setting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Config/setting.py b/Config/setting.py index a74e69a32..66b8f0866 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -24,9 +24,11 @@ class ConfigError(BaseException): if DB_TYPE == 'SSDB': DB_HOST = getenv('ssdb_host', '127.0.0.1') DB_PORT = getenv('ssdb_port', '6379') + DB_PASSWORD = getenv('ssdb_password', '6379') elif DB_TYPE == 'MONGODB': DB_HOST = getenv('mongodb_host', '127.0.0.1') DB_PORT = getenv('mongodb_host', '27017') + DB_PASSWORD = getenv('mongodb_password', '6379') else: raise ConfigError('Unknown database type, your environment variable `db_type` should be one of SSDB/MONGODB.') @@ -36,7 +38,7 @@ class ConfigError(BaseException): "HOST": DB_HOST, "PORT": DB_PORT, "NAME": "proxy", - "PASSWORD": "" + "PASSWORD": DB_PASSWORD } } From bb4a7b9367a74645d1bfecbf92299260ef4bde0f Mon Sep 17 00:00:00 2001 From: houbaron Date: Wed, 8 May 2019 21:44:56 +0800 Subject: [PATCH 051/354] =?UTF-8?q?[refine]=E8=AE=BE=E7=BD=AE=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config/setting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Config/setting.py b/Config/setting.py index 66b8f0866..358b0bfbc 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -24,11 +24,11 @@ class ConfigError(BaseException): if DB_TYPE == 'SSDB': DB_HOST = getenv('ssdb_host', '127.0.0.1') DB_PORT = getenv('ssdb_port', '6379') - DB_PASSWORD = getenv('ssdb_password', '6379') + DB_PASSWORD = getenv('ssdb_password', '') elif DB_TYPE == 'MONGODB': DB_HOST = getenv('mongodb_host', '127.0.0.1') DB_PORT = getenv('mongodb_host', '27017') - DB_PASSWORD = getenv('mongodb_password', '6379') + DB_PASSWORD = getenv('mongodb_password', '') else: raise ConfigError('Unknown database type, your environment variable `db_type` should be one of SSDB/MONGODB.') From f5a4317bbc96f6396d85337bba735545c437fecd Mon Sep 17 00:00:00 2001 From: hero Date: Sat, 11 May 2019 20:09:56 +0800 Subject: [PATCH 052/354] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=85=A8=E7=BD=91?= =?UTF-8?q?=E4=BB=A3=E7=90=86port=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/getFreeProxy.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index cdfa843a0..330bf090a 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -128,8 +128,20 @@ def freeProxyFifth(): try: # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port ip_addr = ''.join(each_proxy.xpath(xpath_str)) - port = each_proxy.xpath(".//span[contains(@class, 'port')]/text()")[0] - yield '{}:{}'.format(ip_addr, port) + + # HTML中的port是随机数，真正的端口编码在class后面的字母中。 + # 比如这个： + # 9054 + # CFACE解码后对应的是3128。 + port = 0 + for _ in each_proxy.xpath(".//span[contains(@class, 'port')]" + "/attribute::class")[0]. \ + replace("port ", ""): + port *= 10 + port += (ord(_) - ord('A')) + port /= 8 + + yield '{}:{}'.format(ip_addr, int(port)) except Exception as e: pass From 35f43ecbe67ba869fcb3b7f044185f79a7452699 Mon Sep 17 00:00:00 2001 From: jhao Date: Wed, 10 Jul 2019 17:17:32 +0800 Subject: [PATCH 053/354] [update] fix 272 --- Schedule/ProxyCheck.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Schedule/ProxyCheck.py b/Schedule/ProxyCheck.py index 4300f7bf7..782d993d1 100644 --- a/Schedule/ProxyCheck.py +++ b/Schedule/ProxyCheck.py @@ -15,6 +15,12 @@ import sys from threading import Thread + +try: + from Queue import Empty # py3 +except: + from queue import Empty # py2 + sys.path.append('../') from Util.utilFunction import validUsefulProxy @@ -35,7 +41,10 @@ def __init__(self, queue, item_dict): def run(self): self.db.changeTable(self.useful_proxy_queue) while self.queue.qsize(): - proxy = self.queue.get() + try: + proxy = self.queue.get() + except Empty: + break count = self.item_dict[proxy] if validUsefulProxy(proxy): # 验证通过计数器减1 @@ -53,8 +62,3 @@ def run(self): self.db.put(proxy, num=int(count) + 1) self.queue.task_done() - -if __name__ == '__main__': - # p = ProxyCheck() - # p.run() - pass From 2f39dedbf36c3838233f452323f18ddad25f9e7b Mon Sep 17 00:00:00 2001 From: jhao Date: Thu, 11 Jul 2019 16:39:23 +0800 Subject: [PATCH 054/354] =?UTF-8?q?[update]=20=E4=BB=A3=E7=90=86=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E7=B1=BB=E5=9E=8B=E5=B0=81=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyHelper/Proxy.py | 104 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 ProxyHelper/Proxy.py diff --git a/ProxyHelper/Proxy.py b/ProxyHelper/Proxy.py new file mode 100644 index 000000000..dce009e96 --- /dev/null +++ b/ProxyHelper/Proxy.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： Proxy + Description : 代理对象类型封装 + Author : JHao + date： 2019/7/11 +------------------------------------------------- + Change Activity: + 2019/7/11: 代理对象类型封装 +------------------------------------------------- +""" +__author__ = 'JHao' + + +class Proxy(object): + + def __init__(self, proxy): + if isinstance(proxy, basestring): + self._proxy = proxy + self._fail_count = 0 + self._region = "" + self._type = "" + self._last_status = "" + self._last_time = "" + + elif isinstance(proxy, dict): + self._proxy = proxy.get("proxy") + self._fail_count = proxy.get("fail_count") + self._region = proxy.get("region") + self._type = proxy.get("type") + self._last_status = proxy.get("last_status") + self._last_time = proxy.get("last_time") + + else: + raise TypeError("proxy arg invalid") + + @property + def proxy(self): + """ 代理 ip:port """ + return self._proxy + + @property + def fail_count(self): + """ 检测失败次数 """ + return self._fail_count + + @property + def region(self): + """ 地理位置(国家/城市) """ + return self._region + + @property + def type(self): + """ 透明/匿名/高匿 """ + return self._type + + @property + def last_status(self): + """ 最后一次检测结果 """ + return self._last_status + + @property + def last_time(self): + """ 最后一次检测时间 """ + return self._last_time + + # --- proxy method --- + @fail_count.setter + def fail_count(self, value): + self._fail_count = value + + @region.setter + def region(self, value): + self._region = value + + @type.setter + def type(self, value): + self._type = value + + @last_status.setter + def last_status(self, value): + self._last_status = value + + @last_time.setter + def last_time(self, value): + self._last_time = value + + +def proxy2Json(proxy): + return {"proxy": proxy.proxy, + "fail_count": proxy.fail_count, + "region": proxy.region, + "type": proxy.type, + "last_status": proxy.last_status, + "last_time": proxy.last_time} + + +if __name__ == '__main__': + p = Proxy("127.0.0.1:8080") + + import json + + print json.dumps(p, default=proxy2Json) From 964061e8e80baf2534652e290385f7131f880447 Mon Sep 17 00:00:00 2001 From: jhao Date: Thu, 11 Jul 2019 17:03:31 +0800 Subject: [PATCH 055/354] =?UTF-8?q?[update]=20=E6=97=A0=E5=BF=A7=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ProxyGetter/getFreeProxy.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 330bf090a..60dd884c1 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -30,17 +30,14 @@ class GetFreeProxy(object): """ @staticmethod - def freeProxyFirst(page=10): + def freeProxy01(): """ 无忧代理 http://www.data5u.com/ 几乎没有能用的 - :param page: 页数 :return: """ url_list = [ 'http://www.data5u.com/', - 'http://www.data5u.com/free/gngn/index.shtml', - 'http://www.data5u.com/free/gnpt/index.shtml' ] for url in url_list: html_tree = getHtmlTree(url) @@ -300,7 +297,7 @@ def freeProxyWallThird(): if __name__ == '__main__': from CheckProxy import CheckProxy - # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01()) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyThird) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFourth) @@ -310,7 +307,7 @@ def freeProxyWallThird(): # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEight) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyNinth) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTen) - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) + # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven) # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve) # CheckProxy.checkAllGetProxyFunc() From f0c7a0f918cad270a508bed8296cd644b5dc1722 Mon Sep 17 00:00:00 2001 From: jhao Date: Thu, 18 Jul 2019 10:00:04 +0800 Subject: [PATCH 056/354] =?UTF-8?q?[update]=20=E7=A0=B4=E8=A7=A3=E4=BB=A3?= =?UTF-8?q?=E7=90=8666=20=E5=8A=A0=E9=80=9F=E4=B9=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Config/setting.py | 4 +-- ProxyGetter/CheckProxy.py | 2 +- ProxyGetter/getFreeProxy.py | 56 +++++++++++++++++++++++++++---------- Util/utilFunction.py | 12 -------- requirements.txt | 1 + 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/Config/setting.py b/Config/setting.py index 358b0bfbc..4ef1b76eb 100644 --- a/Config/setting.py +++ b/Config/setting.py @@ -46,8 +46,8 @@ class ConfigError(BaseException): # register the proxy getter function PROXY_GETTER = [ - "freeProxyFirst", - "freeProxySecond", + "freeProxy01", + "freeProxy02", # "freeProxyThird", # 网站已不能访问 "freeProxyFourth", "freeProxyFifth", diff --git a/ProxyGetter/CheckProxy.py b/ProxyGetter/CheckProxy.py index 2b3fc6a29..d15be49c9 100644 --- a/ProxyGetter/CheckProxy.py +++ b/ProxyGetter/CheckProxy.py @@ -67,4 +67,4 @@ def checkGetProxyFunc(func): if __name__ == '__main__': CheckProxy.checkAllGetProxyFunc() - CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst) + CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01) \ No newline at end of file diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py index 60dd884c1..1b1277af4 100644 --- a/ProxyGetter/getFreeProxy.py +++ b/ProxyGetter/getFreeProxy.py @@ -49,24 +49,52 @@ def freeProxy01(): print(e) @staticmethod - def freeProxySecond(count=20): + def freeProxy02(count=20): """ 代理66 http://www.66ip.cn/ :param count: 提取数量 :return: """ urls = [ - "http://www.66ip.cn/mo.php?sxb=&tqsl={count}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=", - "http://www.66ip.cn/nmtq.php?getnum={count}" - "&isp=0&anonymoustype=0&start=&ports=&export=&ipaddress=&area=1&proxytype=2&api=66ip", - ] - request = WebRequest() - for _ in urls: - url = _.format(count=count) - html = request.get(url).content - ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html) - for ip in ips: - yield ip.strip() + "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=", + "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s" + "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip" + ] + + try: + import execjs + import requests + + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', + 'Accept': '*/*', + 'Connection': 'keep-alive', + 'Accept-Language': 'zh-CN,zh;q=0.8'} + session = requests.session() + src = session.get("http://www.66ip.cn/", headers=headers).text + src = src.split("")[0] + '}' + src = src.replace("")[0] + '}' + src = src.replace("")[0] + '}' + src = src.replace("")[0] + '}' - src = src.replace("")[0] + '}' +# src = src.replace("")[0] + '}' -# src = src.replace("")[0] + '}' - src = src.replace("8080' % script_content + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response(tree=tree) + result = list(FreeProxyListFetcher().fetch()) + # 注意：实际 JS 解码逻辑可能需要更复杂的 mock + # 这里主要验证 fetch() 不报错且返回列表 + assert isinstance(result, list) + + +class TestKuaidailiFetcher(object): + + @patch("fetcher.sources.kuaidaili.WebRequest") + @patch("fetcher.sources.kuaidaili.sleep", return_value=None) + def test_fetch(self, mock_sleep, mock_wr): + from fetcher.sources.kuaidaili import KuaidailiFetcher + # kuaidaili 使用 proxy_list[1:] 跳过第一行 + html = _html_table([("IP", "Port"), ("1.2.3.4", "8080")]) + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response(tree=tree) + result = list(KuaidailiFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestFreeVPNNodeFetcher(object): + + @patch("fetcher.sources.freevpnnode.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.freevpnnode import FreeVPNNodeFetcher + html = _html_table([("1.2.3.4", "8080")]) + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response( + tree=tree, text="1.2.3.4:8080 5.6.7.8:3128") + result = list(FreeVPNNodeFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + +class TestScdnFetcher(object): + + @patch("fetcher.sources.scdn.WebRequest") + def test_fetch_json(self, mock_wr): + from fetcher.sources.scdn import ScdnFetcher + json_data = {"data": [{"ip": "1.2.3.4", "port": "8080"}]} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ScdnFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.scdn.WebRequest") + def test_fetch_table_html(self, mock_wr): + from fetcher.sources.scdn import ScdnFetcher + table_html = '1.2.3.48080' + json_data = {"table_html": table_html} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ScdnFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestZdayeFetcher(object): + + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.sleep", return_value=None) + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_recent(self, mock_dt, mock_sleep, mock_wr): + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 模拟最新帖子时间在5分钟内 + mock_dt.now.return_value = real_datetime(2026, 5, 31, 12, 0, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 31, 11, 58, 0) + + index_tree = etree.HTML( + '2026/05/31 11:58:00' + '

test

') + detail_tree = etree.HTML(_html_table([("1.2.3.4", "8080")])) + + def side_effect(url, **kwargs): + resp = MagicMock() + if "free" in url: + resp.tree = index_tree + else: + resp.tree = detail_tree + return resp + + mock_wr.return_value.get.side_effect = side_effect + result = list(ZdayeFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_old_returns_empty(self, mock_dt, mock_wr): + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 模拟最新帖子时间超过5分钟 + mock_dt.now.return_value = real_datetime(2026, 5, 31, 12, 0, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 31, 10, 0, 0) + + index_tree = etree.HTML( + '2026/05/31 10:00:00' + '

test

') + mock_wr.return_value.get.return_value = _make_response(tree=index_tree) + result = list(ZdayeFetcher().fetch()) + assert result == [] + + +class TestIhuanFetcher(object): + + @patch("fetcher.sources.ihuan.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.ihuan import IhuanFetcher + ti_tree = etree.HTML( + '
') + post_tree = etree.HTML(_html_table([("1.2.3.4", "8080")])) + + ti_resp = _make_response(tree=ti_tree, text="") + post_resp = _make_response(tree=post_tree, text="1.2.3.4:8080") + + mock_instance = MagicMock() + mock_instance.get.return_value = ti_resp + mock_instance.post.return_value = post_resp + mock_wr.return_value = mock_instance + + result = list(IhuanFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.ihuan.WebRequest") + def test_fetch_no_key_returns_empty(self, mock_wr): + from fetcher.sources.ihuan import IhuanFetcher + ti_tree = etree.HTML('
') + ti_resp = _make_response(tree=ti_tree, text="no key here") + mock_wr.return_value.get.return_value = ti_resp + result = list(IhuanFetcher().fetch()) + assert result == [] From 7db2a8490c13bbe1b1138e568fc5101534605c10 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Sun, 31 May 2026 18:15:46 +0800 Subject: [PATCH 330/354] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E4=BB=A5=E5=8F=8D=E6=98=A0=E4=BB=A3=E7=90=86=E9=87=87?= =?UTF-8?q?=E9=9B=86=E6=A8=A1=E5=9D=97=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 重写扩展代理源文档，改为 BaseFetcher 基类 + 独立文件模式 - 更新项目结构文档，fetcher 目录从单文件改为 sources/ 插件目录 - 更新配置文档，PROXY_FETCHER 改为自动扫描 + PROXY_FETCHER_EXCLUDE 黑名单 - 更新变更日志，记录重构内容 - 更新 CLAUDE.md，同步架构、配置、命名规范、测试结构 --- CLAUDE.md | 19 +++--- docs/changelog.md | 7 +++ docs/configuration.md | 20 ++++--- docs/extending/fetcher.md | 122 +++++++++++++++++++++++++++++++------- docs/project-structure.md | 15 +++-- 5 files changed, 141 insertions(+), 42 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 25f087abe..1f63b0ae2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ Python (3.8–3.11)、Flask (API)、Redis/SSDB (存储)、APScheduler (调度) - 安装依赖：`pip install -r requirements.txt` - 运行代理爬取/验证调度器：`python proxyPool.py schedule` - 运行 API 服务器：`python proxyPool.py server` +- 查看启用的代理源：`python proxyPool.py show` - 运行单元测试：`pytest tests/unit/` - 运行 API 测试：`pytest tests/api/` - 运行集成测试（需真实 Redis）：`pytest tests/integration/ -m integration` @@ -25,7 +26,9 @@ tests/ │ ├── test_proxy.py # Proxy 类：构造、序列化、setter、add_source │ ├── test_db_client.py # DbClient.parseDbConn URI 解析 │ ├── test_config.py # ConfigHandler 环境变量覆盖 -│ └── test_validator.py # formatValidator 正则匹配 +│ ├── test_validator.py # formatValidator 正则匹配 +│ ├── test_base_fetcher.py # BaseFetcher 基类解析方法 +│ └── test_fetcher_sources.py # 各代理源 fetcher yield 逻辑 ├── api/ # Flask 测试客户端，mock ProxyHandler │ └── test_proxy_api.py # /get /pop /all /count /delete 全路由 └── integration/ # 需要真实 Redis，标记 @pytest.mark.integration @@ -50,7 +53,7 @@ tests/ 免费代理池项目，爬取公开代理源、验证代理可用性、持久化存储到 Redis/SSDB，并通过 Flask RESTful API 提供代理服务。 ### 核心组件 -- **爬取器** (`fetcher/proxyFetcher.py`)：`ProxyFetcher` 类，每个代理源对应一个静态方法，yield 出 `host:port` 字符串。通过 `setting.py` 中的 `PROXY_FETCHER` 列表启用对应爬取器。 +- **爬取器** (`fetcher/`)：插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类（提供 `parseProxiesFromText`/`parseProxiesFromJson`/`parseProxiesFromTree`/`yieldUniqueProxies` 共享方法，约定 `name`/`url`/`enabled` 属性和 `fetch()` 方法）。每个代理源在 `sources/` 目录下独立文件，继承 `BaseFetcher`。调度器自动扫描目录加载 `enabled=True` 的源。`setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 - **数据库层** (`db/`)：抽象 `dbClient` 接口，包含 Redis (`redisClient.py`) 和 SSDB (`ssdbClient.py`) 两种实现。通过 `setting.py` 中的 `DB_CONN` 配置连接（格式：`redis://:pwd@ip:port/db` 或 `ssdb://:pwd@ip:port`）。 - **调度器** (`helper/scheduler.py`)：基于 APScheduler 的定时任务，驱动爬取器运行并触发验证。时区通过 `setting.py` 中的 `TIMEZONE` 配置。 - **验证器** (`helper/validator.py`)：使用 `HTTP_URL` (http://httpbin.org) 和 `HTTPS_URL` (https://www.qq.com) 测试代理，超时时间由 `VERIFY_TIMEOUT` 指定（默认 10 秒）。超过 `MAX_FAIL_COUNT` 的代理会被移除。当代理池数量低于 `POOL_SIZE_MIN`（默认 20）时触发重新爬取。 @@ -64,14 +67,14 @@ tests/ - **命令行入口** (`proxyPool.py`)：基于 click 的命令行工具，包含 `schedule` 和 `server` 两个子命令。 ### 扩展代理源 -1. 在 `fetcher/proxyFetcher.py` 的 `ProxyFetcher` 类中新增一个静态方法，yield 出 `host:port` 字符串。 -2. 将该方名添加到 `setting.py` 的 `PROXY_FETCHER` 列表中。调度器会自动识别并启用新的代理源。 +1. 在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher`，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法 yield 出 `host:port` 字符串。 +2. 调度器下一轮采集自动发现并启用，无需修改配置。可用 `python proxyPool.py show` 查看启用列表。 ## 关键配置 所有运行时配置均在 `setting.py` 中： - `HOST`/`PORT`：API 绑定的地址和端口 - `DB_CONN`：数据库连接字符串 -- `PROXY_FETCHER`：已启用的爬取器方法名列表 +- `PROXY_FETCHER_EXCLUDE`：爬取器黑名单（自动扫描 `enabled=True` 的源，排除黑名单中的） - `HTTP_URL`/`HTTPS_URL`：验证目标 URL - `VERIFY_TIMEOUT`：验证超时时间（默认 10 秒） - `MAX_FAIL_COUNT`：代理被移除前允许的最大失败次数 @@ -99,9 +102,9 @@ tests/ - **缩进**：4 个空格（Python 标准） - **文件命名**：驼峰命名，如 `proxyFetcher.py`、`dbClient.py`、`redisClient.py`、`webRequest.py` - **类命名**：帕斯卡命名，如 `ProxyFetcher`、`RedisClient`、`SsdbClient`、`ProxyValidator` -- **方法命名**：混合风格——数据库/爬取器方法使用驼峰命名（`getAll`、`getCount`、`changeTable`、`freeProxy01`），属性和辅助方法使用下划线命名（`user_agent`、`fail_count`、`check_count`） -- **爬取器方法**：命名为 `freeProxy` + 两位数字（如 `freeProxy01`、`freeProxy02`）。新增爬取器必须遵循此模式 -- **常量**（在 `setting.py` 中）：大写下划线命名（`DB_CONN`、`PROXY_FETCHER`、`HTTP_URL`、`MAX_FAIL_COUNT`） +- **方法命名**：混合风格——数据库/爬取器方法使用驼峰命名（`getAll`、`getCount`、`changeTable`、`parseProxiesFromText`），属性和辅助方法使用下划线命名（`user_agent`、`fail_count`、`check_count`） +- **爬取器文件**：小写命名（如 `zdaye.py`、`kuaidaili.py`），类名 PascalCase（如 `ZdayeFetcher`、`KuaidailiFetcher`） +- **常量**（在 `setting.py` 中）：大写下划线命名（`DB_CONN`、`PROXY_FETCHER_EXCLUDE`、`HTTP_URL`、`MAX_FAIL_COUNT`） - **变量**：下划线命名（`proxy_obj`、`proxy_str`、`https`） - **注释/文档字符串**：源文件头部和行内注释通常使用中文（普通话） - **单例模式**：使用自定义 `Singleton` 元类（`util/singleton.py`）结合 `six.withMetaclass` 实现 diff --git a/docs/changelog.md b/docs/changelog.md index 1369ea2b7..c38033298 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -8,6 +8,13 @@ 4. 优化CI配置, 避免PR时重复触发测试; (2026-05-26) 5. 迁移文档从 Sphinx/ReadTheDocs 到 MkDocs Material/GitHub Pages; (2026-05-27) 6. **重写测试套件**: 使用pytest重构全部测试, 覆盖unit/api/integration三层; (2026-05-28) +7. **重构代理采集模块**: 将 `proxyFetcher.py` 拆分为独立文件 + `BaseFetcher` 基类插件架构，支持自动扫描和运行时热更新; (2026-05-31) + - 每个代理源独立文件（`fetcher/sources/`），继承 `BaseFetcher` 基类 + - 自动扫描启用的代理源，无需在配置中列举 + - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 + - 新增 `proxyPool.py show` 命令查看启用的代理源 + - 每次调度打印 active fetchers 列表 + - 删除旧 `proxyFetcher.py` ## 2.4.2 (2024-01-18) diff --git a/docs/configuration.md b/docs/configuration.md index 2baa07ec4..6387df37f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -49,22 +49,26 @@ DB_CONN = 'ssdb://:123456@127.0.0.1:8888' ## 采集配置 -### `PROXY_FETCHER` +代理采集采用插件架构，调度器自动扫描 `fetcher/sources/` 目录，加载所有 `enabled=True` 的代理源。新增代理源只需在 `sources/` 下创建文件，无需修改配置。 -启用的代理采集方法名列表。代理采集方法位于 `fetcher/proxyFetcher.py` 类中。 +查看当前启用的代理源： -由于各个代理源的稳定性不容易掌握，当某个代理采集方法失效时，可以在该配置中注释掉其名称。如果有增加某些代理采集方法，也请在该配置中添加其方法名，具体请参考 [扩展代理源](extending/fetcher.md)。 +```bash +python proxyPool.py show +``` + +### `PROXY_FETCHER_EXCLUDE` -调度程序每次执行采集任务时都会再次加载该配置，保证每次运行的采集方法都是有效的。 +代理源黑名单。列表中的类名对应的代理源不会被加载，即使 `enabled=True`。适用于临时禁用某个代理源而不修改其源文件。 ```python -PROXY_FETCHER = [ - "freeProxy01", - "freeProxy02", - # .... +PROXY_FETCHER_EXCLUDE = [ + # "BinglxFetcher", # 临时禁用冰凌代理 ] ``` +如需永久禁用，建议直接在源文件中设置 `enabled = False`。 + ## 校验配置 ### `HTTP_URL` diff --git a/docs/extending/fetcher.md b/docs/extending/fetcher.md index c63b7547b..63b8a3f1e 100644 --- a/docs/extending/fetcher.md +++ b/docs/extending/fetcher.md @@ -4,39 +4,117 @@ ## 添加新的代理源 -### 第一步：编写获取方法 +### 第一步：创建代理源文件 -在 `ProxyFetcher` 类中添加自定义的获取代理的静态方法，该方法需要以生成器（yield）形式返回 `host:port` 格式的代理字符串： +在 `fetcher/sources/` 目录下新建一个 `.py` 文件，继承 `BaseFetcher` 基类，实现 `fetch()` 方法： ```python -# fetcher/proxyFetcher.py - -class ProxyFetcher(object): - # .... - # 自定义代理源获取方法 - @staticmethod - def freeProxyCustom01(): # 命名不和已有重复即可 - # 通过某网站或者某接口或某数据库获取代理 - # 假设你已经拿到了一个代理列表 - proxies = ["x.x.x.x:3128", "x.x.x.x:80"] - for proxy in proxies: +# fetcher/sources/mySource.py + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class MySourceFetcher(BaseFetcher): + """我的代理源 https://example.com/proxy""" + + name = "mysource" # 唯一标识，用于日志 + url = "https://example.com/proxy" # 源网站首页 + enabled = True # 设为 False 可禁用 + + def fetch(self): + """yield "host:port" 格式的代理字符串""" + r = WebRequest().get(self.url, timeout=10) + for proxy in self.parseProxiesFromText(r.text): yield proxy - # 确保每个 proxy 都是 host:port 正确的格式返回 + + +if __name__ == '__main__': + for proxy in MySourceFetcher().fetch(): + print(proxy) +``` + +添加完成后，调度器下一轮采集（默认 4 分钟）会自动发现并启用新源，**无需修改任何配置文件**。 + +### 第二步：验证 + +独立调试代理源： + +```bash +python -m fetcher.sources.mySource +``` + +查看当前启用的代理源： + +```bash +python proxyPool.py show +``` + +## 禁用代理源 + +有两种方式禁用某个代理源： + +**方式一：修改源文件**（推荐） + +在对应文件中将 `enabled` 设为 `False`： + +```python +class MySourceFetcher(BaseFetcher): + enabled = False # 禁用该源 ``` -### 第二步：注册到配置 +**方式二：黑名单配置** -修改配置文件 `setting.py` 中的 `PROXY_FETCHER` 项，加入刚才添加的自定义方法的名字： +在 `setting.py` 的 `PROXY_FETCHER_EXCLUDE` 列表中添加类名，无需修改源文件： ```python -PROXY_FETCHER = [ - # .... - "freeProxyCustom01" # 确保名字和你添加的方法名字一致 -] +PROXY_FETCHER_EXCLUDE = ["MySourceFetcher"] ``` -调度程序每次执行采集任务时都会重新加载该配置，添加后会自动启用新的代理源。 +## BaseFetcher 基类 + +所有代理源必须继承 `BaseFetcher`，基类提供以下约定和工具： + +### 必须声明的属性 + +| 属性 | 类型 | 说明 | +|------|------|------| +| `name` | str | 唯一标识，用于日志和代理来源标记 | +| `url` | str | 源网站首页 URL | + +### 可选属性 + +| 属性 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| `enabled` | bool | `True` | 是否启用 | + +### 必须实现的方法 + +| 方法 | 说明 | +|------|------| +| `fetch(self)` | 生成器，yield `"host:port"` 格式字符串 | + +### 共享解析工具 + +| 方法 | 说明 | +|------|------| +| `parseProxiesFromText(text)` | 从纯文本中用正则提取 ip:port | +| `parseProxiesFromJson(data)` | 从 JSON 结构中递归提取 ip:port | +| `parseProxiesFromTree(tree)` | 从 lxml tree 的 table 行中提取 ip:port | +| `yieldUniqueProxies(proxies)` | 去重 yield | + +## 运行时热更新 + +调度器每轮采集时会重新扫描 `fetcher/sources/` 目录并 reload 模块，因此： + +- 新增文件 → 下一轮自动启用 +- 修改文件内容 → 下一轮自动加载新版本 +- 删除文件 → 下一轮自动移除（建议先从 `PROXY_FETCHER_EXCLUDE` 或 `enabled` 中禁用） ## 命名规范 -代理获取方法建议命名为 `freeProxy` + 两位数字（如 `freeProxy01`），自定义方法可使用 `freeProxyCustom` + 数字。 \ No newline at end of file +| 元素 | 风格 | 示例 | +|------|------|------| +| 文件名 | 小写 | `mysource.py` | +| 类名 | PascalCase | `MySourceFetcher` | +| `name` 属性 | 小写 | `"mysource"` | \ No newline at end of file diff --git a/docs/project-structure.md b/docs/project-structure.md index ffde8e34e..4c4e63c2f 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -11,7 +11,14 @@ proxy_pool/ │ ├── redisClient.py # Redis 实现 │ └── ssdbClient.py # SSDB 实现 ├── fetcher/ # 代理采集器 -│ └── proxyFetcher.py # 各代理源采集方法 +│ ├── baseFetcher.py # BaseFetcher 基类（共享解析方法） +│ └── sources/ # 各代理源独立文件 +│ ├── zdaye.py # 站大爷 +│ ├── ip66.py # 代理66 +│ ├── kxdaili.py # 开心代理 +│ ├── kuaidaili.py # 快代理 +│ ├── geonode.py # Geonode +│ └── ... # 其他代理源 ├── handler/ # 业务处理器 │ ├── configHandler.py # 配置读取 │ ├── logHandler.py # 日志处理 @@ -48,7 +55,7 @@ proxy_pool/ ### `proxyPool.py` — 入口 -基于 click 的命令行入口，提供 `schedule` 和 `server` 两个子命令。`schedule` 启动代理采集和验证调度器，`server` 启动 Flask API 服务。 +基于 click 的命令行入口，提供 `schedule`、`server`、`show` 三个子命令。`schedule` 启动代理采集和验证调度器，`server` 启动 Flask API 服务，`show` 查看当前启用的代理源列表。 ### `api/proxyApi.py` — API 服务 @@ -58,9 +65,9 @@ Flask 应用，提供 `/get`、`/pop`、`/all`、`/count`、`/delete` 等接口 通过 `dbClient.py` 定义统一接口，`redisClient.py` 和 `ssdbClient.py` 分别实现 Redis 和 SSDB 的存取逻辑。使用 `setting.py` 中的 `DB_CONN` 连接字符串选择后端。 -### `fetcher/proxyFetcher.py` — 代理采集 +### `fetcher/` — 代理采集 -`ProxyFetcher` 类中每个代理源对应一个 `freeProxyXX` 静态方法，yield `host:port` 字符串。通过 `setting.py` 的 `PROXY_FETCHER` 列表启用。 +采用插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类，提供共享解析方法和 `name`/`url`/`enabled` 属性约定。每个代理源在 `sources/` 目录下独立一个文件，继承 `BaseFetcher` 并实现 `fetch()` 方法。调度器自动扫描 `sources/` 目录，加载 `enabled=True` 的源。通过 `setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 ### `helper/scheduler.py` — 定时调度 From 517f822f92974812b7504ba8f12d927f8de4af8f Mon Sep 17 00:00:00 2001 From: jhao104 Date: Sun, 31 May 2026 20:24:55 +0800 Subject: [PATCH 331/354] =?UTF-8?q?refactor:=20=E7=B2=BE=E7=AE=80BaseFetch?= =?UTF-8?q?er=EF=BC=8C=E5=88=A0=E9=99=A4=E9=80=9A=E7=94=A8JSON/HTML?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E6=96=B9=E6=B3=95=EF=BC=8C=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E5=8A=A0=E8=BD=BD=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除 BaseFetcher.parseProxiesFromJson 和 parseProxiesFromTree，各fetcher内联自己的解析逻辑 - 合并 _load_fetcher_class 到 _discover_fetchers，避免重复扫描目录和加载模块 - rename show 命令为 fetcher - 固定 fakeredis<2.26 兼容 Python 3.8 --- CLAUDE.md | 6 +- docs/changelog.md | 4 +- docs/configuration.md | 2 +- docs/extending/fetcher.md | 8 +-- docs/project-structure.md | 2 +- fetcher/baseFetcher.py | 50 ---------------- fetcher/sources/freevpnnode.py | 12 +++- fetcher/sources/geonode.py | 7 ++- fetcher/sources/ihuan.py | 10 +++- fetcher/sources/scdn.py | 16 ++++- helper/fetch.py | 43 +++---------- helper/scheduler.py | 2 +- proxyPool.py | 12 ++-- requirements-test.txt | 2 +- tests/unit/test_base_fetcher.py | 103 -------------------------------- 15 files changed, 65 insertions(+), 214 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1f63b0ae2..9691f22e4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Python (3.8–3.11)、Flask (API)、Redis/SSDB (存储)、APScheduler (调度) - 安装依赖：`pip install -r requirements.txt` - 运行代理爬取/验证调度器：`python proxyPool.py schedule` - 运行 API 服务器：`python proxyPool.py server` -- 查看启用的代理源：`python proxyPool.py show` +- 查看启用的代理源：`python proxyPool.py fetcher` - 运行单元测试：`pytest tests/unit/` - 运行 API 测试：`pytest tests/api/` - 运行集成测试（需真实 Redis）：`pytest tests/integration/ -m integration` @@ -53,7 +53,7 @@ tests/ 免费代理池项目，爬取公开代理源、验证代理可用性、持久化存储到 Redis/SSDB，并通过 Flask RESTful API 提供代理服务。 ### 核心组件 -- **爬取器** (`fetcher/`)：插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类（提供 `parseProxiesFromText`/`parseProxiesFromJson`/`parseProxiesFromTree`/`yieldUniqueProxies` 共享方法，约定 `name`/`url`/`enabled` 属性和 `fetch()` 方法）。每个代理源在 `sources/` 目录下独立文件，继承 `BaseFetcher`。调度器自动扫描目录加载 `enabled=True` 的源。`setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 +- **爬取器** (`fetcher/`)：插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类（提供 `parseProxiesFromText`/`yieldUniqueProxies` 共享方法，约定 `name`/`url`/`enabled` 属性和 `fetch()` 方法）。每个代理源在 `sources/` 目录下独立文件，继承 `BaseFetcher`。调度器自动扫描目录加载 `enabled=True` 的源。`setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 - **数据库层** (`db/`)：抽象 `dbClient` 接口，包含 Redis (`redisClient.py`) 和 SSDB (`ssdbClient.py`) 两种实现。通过 `setting.py` 中的 `DB_CONN` 配置连接（格式：`redis://:pwd@ip:port/db` 或 `ssdb://:pwd@ip:port`）。 - **调度器** (`helper/scheduler.py`)：基于 APScheduler 的定时任务，驱动爬取器运行并触发验证。时区通过 `setting.py` 中的 `TIMEZONE` 配置。 - **验证器** (`helper/validator.py`)：使用 `HTTP_URL` (http://httpbin.org) 和 `HTTPS_URL` (https://www.qq.com) 测试代理，超时时间由 `VERIFY_TIMEOUT` 指定（默认 10 秒）。超过 `MAX_FAIL_COUNT` 的代理会被移除。当代理池数量低于 `POOL_SIZE_MIN`（默认 20）时触发重新爬取。 @@ -68,7 +68,7 @@ tests/ ### 扩展代理源 1. 在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher`，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法 yield 出 `host:port` 字符串。 -2. 调度器下一轮采集自动发现并启用，无需修改配置。可用 `python proxyPool.py show` 查看启用列表。 +2. 调度器下一轮采集自动发现并启用，无需修改配置。可用 `python proxyPool.py fetcher` 查看启用列表。 ## 关键配置 所有运行时配置均在 `setting.py` 中： diff --git a/docs/changelog.md b/docs/changelog.md index c38033298..44ca09e20 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,9 +12,7 @@ - 每个代理源独立文件（`fetcher/sources/`），继承 `BaseFetcher` 基类 - 自动扫描启用的代理源，无需在配置中列举 - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - - 新增 `proxyPool.py show` 命令查看启用的代理源 - - 每次调度打印 active fetchers 列表 - - 删除旧 `proxyFetcher.py` + - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 ## 2.4.2 (2024-01-18) diff --git a/docs/configuration.md b/docs/configuration.md index 6387df37f..bd71fc7a5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -54,7 +54,7 @@ DB_CONN = 'ssdb://:123456@127.0.0.1:8888' 查看当前启用的代理源： ```bash -python proxyPool.py show +python proxyPool.py fetcher ``` ### `PROXY_FETCHER_EXCLUDE` diff --git a/docs/extending/fetcher.md b/docs/extending/fetcher.md index 63b8a3f1e..0b9a2107f 100644 --- a/docs/extending/fetcher.md +++ b/docs/extending/fetcher.md @@ -34,7 +34,7 @@ if __name__ == '__main__': print(proxy) ``` -添加完成后，调度器下一轮采集（默认 4 分钟）会自动发现并启用新源，**无需修改任何配置文件**。 +添加完成后，调度器下一轮采集（默认 5 分钟）会自动发现并启用新源，**无需修改任何配置文件**。 ### 第二步：验证 @@ -47,7 +47,7 @@ python -m fetcher.sources.mySource 查看当前启用的代理源： ```bash -python proxyPool.py show +python proxyPool.py fetcher ``` ## 禁用代理源 @@ -56,7 +56,7 @@ python proxyPool.py show **方式一：修改源文件**（推荐） -在对应文件中将 `enabled` 设为 `False`： +在对应py文件中将 `enabled` 设为 `False`： ```python class MySourceFetcher(BaseFetcher): @@ -99,8 +99,6 @@ PROXY_FETCHER_EXCLUDE = ["MySourceFetcher"] | 方法 | 说明 | |------|------| | `parseProxiesFromText(text)` | 从纯文本中用正则提取 ip:port | -| `parseProxiesFromJson(data)` | 从 JSON 结构中递归提取 ip:port | -| `parseProxiesFromTree(tree)` | 从 lxml tree 的 table 行中提取 ip:port | | `yieldUniqueProxies(proxies)` | 去重 yield | ## 运行时热更新 diff --git a/docs/project-structure.md b/docs/project-structure.md index 4c4e63c2f..795650b89 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -55,7 +55,7 @@ proxy_pool/ ### `proxyPool.py` — 入口 -基于 click 的命令行入口，提供 `schedule`、`server`、`show` 三个子命令。`schedule` 启动代理采集和验证调度器，`server` 启动 Flask API 服务，`show` 查看当前启用的代理源列表。 +基于 click 的命令行入口，提供 `schedule`、`server`、`fetcher` 三个子命令。`schedule` 启动代理采集和验证调度器，`server` 启动 Flask API 服务，`fetcher` 查看当前启用的代理源列表。 ### `api/proxyApi.py` — API 服务 diff --git a/fetcher/baseFetcher.py b/fetcher/baseFetcher.py index eaa6ba292..f22f31e25 100644 --- a/fetcher/baseFetcher.py +++ b/fetcher/baseFetcher.py @@ -38,56 +38,6 @@ def parseProxiesFromText(text): r'(?= 2: + ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) + port_match = re.match(r'^\d{2,5}$', cells[1]) + if ip_match and port_match: + proxies.append("%s:%s" % (cells[0], cells[1])) proxies.extend(self.parseProxiesFromText(r.text)) for proxy in self.yieldUniqueProxies(proxies): yield proxy diff --git a/fetcher/sources/geonode.py b/fetcher/sources/geonode.py index 4906d69a7..e7694d8ee 100644 --- a/fetcher/sources/geonode.py +++ b/fetcher/sources/geonode.py @@ -27,7 +27,12 @@ def fetch(self): "limit=500&page=1&sort_by=lastChecked&sort_type=desc") r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) try: - proxies = self.parseProxiesFromJson(r.json) + proxies = [] + for item in r.json.get("data", []): + ip = item.get("ip", "") + port = item.get("port", "") + if ip and port: + proxies.append("%s:%s" % (ip, port)) if not proxies: proxies = self.parseProxiesFromText(r.text) for proxy in self.yieldUniqueProxies(proxies): diff --git a/fetcher/sources/ihuan.py b/fetcher/sources/ihuan.py index b074af2be..1d9bcf607 100644 --- a/fetcher/sources/ihuan.py +++ b/fetcher/sources/ihuan.py @@ -67,7 +67,15 @@ def fetch(self): "key": key, }) r = request.post(tqdl_url, header=header, data=data, timeout=10, verify=False) - proxies = self.parseProxiesFromTree(r.tree) + proxies = [] + if r.tree is not None: + for tr in r.tree.xpath("//tr"): + cells = [" ".join(td.xpath(".//text()")).strip() for td in tr.xpath("./td")] + if len(cells) >= 2: + ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) + port_match = re.match(r'^\d{2,5}$', cells[1]) + if ip_match and port_match: + proxies.append("%s:%s" % (cells[0], cells[1])) proxies.extend(self.parseProxiesFromText(r.text)) for proxy in self.yieldUniqueProxies(proxies): yield proxy diff --git a/fetcher/sources/scdn.py b/fetcher/sources/scdn.py index 3133f1c98..d360c786d 100644 --- a/fetcher/sources/scdn.py +++ b/fetcher/sources/scdn.py @@ -12,6 +12,8 @@ """ __author__ = 'JHao' +import re + from lxml import etree from fetcher.baseFetcher import BaseFetcher @@ -34,10 +36,20 @@ def fetch(self): table_html = data.get("table_html") if isinstance(data, dict) else "" if table_html: tree = etree.HTML("%s
" % table_html) - proxies.extend(self.parseProxiesFromTree(tree)) + for tr in tree.xpath("//tr"): + cells = [" ".join(td.xpath(".//text()")).strip() for td in tr.xpath("./td")] + if len(cells) >= 2: + ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) + port_match = re.match(r'^\d{2,5}$', cells[1]) + if ip_match and port_match: + proxies.append("%s:%s" % (cells[0], cells[1])) if not proxies: - proxies = self.parseProxiesFromJson(data) + for item in data.get("data", []) if isinstance(data, dict) else []: + ip = item.get("ip", "") + port = item.get("port", "") + if ip and port: + proxies.append("%s:%s" % (ip, port)) if not proxies: proxies = self.parseProxiesFromText(r.text) for proxy in self.yieldUniqueProxies(proxies): diff --git a/helper/fetch.py b/helper/fetch.py index d07aea7ff..ff27dca71 100644 --- a/helper/fetch.py +++ b/helper/fetch.py @@ -30,36 +30,13 @@ def _get_sources_dir(): os.path.dirname(os.path.abspath(__file__)), '..', 'fetcher', 'sources') -def _load_fetcher_class(class_name): - """ - 动态加载 fetcher 类，支持运行时热更新。 - 每次调用重新 import module，确保读到文件最新版本。 - """ - sources_dir = _get_sources_dir() - for filename in os.listdir(sources_dir): - if not filename.endswith('.py') or filename.startswith('_'): - continue - module_name = "fetcher.sources.%s" % filename[:-3] - try: - if module_name in sys.modules: - module = importlib.reload(sys.modules[module_name]) - else: - module = importlib.import_module(module_name) - fetcher_class = getattr(module, class_name, None) - if fetcher_class and issubclass(fetcher_class, BaseFetcher): - return fetcher_class - except Exception: - continue - return None - - def _discover_fetchers(exclude_list): """ - 自动扫描 sources/ 目录，返回所有 enabled=True 且不在黑名单中的 fetcher 类名列表。 + 自动扫描 sources/ 目录，返回所有 enabled=True 且不在黑名单中的 fetcher 类列表。 每次调用重新加载模块，支持运行时热更新。 """ sources_dir = _get_sources_dir() - fetcher_names = [] + fetcher_classes = [] for filename in os.listdir(sources_dir): if not filename.endswith('.py') or filename.startswith('_'): continue @@ -77,10 +54,10 @@ def _discover_fetchers(exclude_list): and attr.name and attr.enabled and attr.__name__ not in exclude_list): - fetcher_names.append(attr.__name__) + fetcher_classes.append(attr) except Exception: continue - return sorted(fetcher_names) + return sorted(fetcher_classes, key=lambda c: c.name) class _ThreadFetcher(Thread): @@ -125,14 +102,10 @@ def run(self): self.log.info("ProxyFetch : start") exclude_list = self.conf.fetcherExclude - fetcher_names = _discover_fetchers(exclude_list) - self.log.info("ProxyFetch : active fetchers [%s]" % ", ".join(fetcher_names)) - - for fetcher_name in fetcher_names: - fetcher_class = _load_fetcher_class(fetcher_name) - if not fetcher_class: - self.log.error("ProxyFetch - {func}: class not exists!".format(func=fetcher_name)) - continue + fetcher_classes = _discover_fetchers(exclude_list) + self.log.info("ProxyFetch : active fetchers [%s]" % ", ".join(c.name for c in fetcher_classes)) + + for fetcher_class in fetcher_classes: thread_list.append(_ThreadFetcher(fetcher_class, proxy_dict)) for thread in thread_list: diff --git a/helper/scheduler.py b/helper/scheduler.py index cd91190a5..ef0431881 100644 --- a/helper/scheduler.py +++ b/helper/scheduler.py @@ -51,7 +51,7 @@ def runScheduler(): scheduler_log = LogHandler("scheduler") scheduler = BlockingScheduler(logger=scheduler_log, timezone=timezone) - scheduler.add_job(__runProxyFetch, 'interval', minutes=4, id="proxy_fetch", name="proxy采集") + scheduler.add_job(__runProxyFetch, 'interval', minutes=5, id="proxy_fetch", name="proxy采集") scheduler.add_job(__runProxyCheck, 'interval', minutes=2, id="proxy_check", name="proxy检查") executors = { 'default': {'type': 'threadpool', 'max_workers': 20}, diff --git a/proxyPool.py b/proxyPool.py index 59bb954c7..9b18b7845 100644 --- a/proxyPool.py +++ b/proxyPool.py @@ -39,17 +39,17 @@ def server(): startServer() -@cli.command(name="show") -def show(): +@cli.command(name="fetcher") +def fetcher(): """ 查看启用的代理源 """ from helper.fetch import _discover_fetchers from handler.configHandler import ConfigHandler conf = ConfigHandler() exclude = conf.fetcherExclude - fetcher_names = _discover_fetchers(exclude) - click.echo("Active fetchers (%d):" % len(fetcher_names)) - for name in fetcher_names: - click.echo(" - %s" % name) + fetcher_classes = _discover_fetchers(exclude) + click.echo("Active fetchers (%d):" % len(fetcher_classes)) + for cls in fetcher_classes: + click.echo(" - %s" % cls.name) if exclude: click.echo("\nExcluded: %s" % ", ".join(exclude)) diff --git a/requirements-test.txt b/requirements-test.txt index a07a17646..89e73e512 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ pytest>=7.0 pytest-cov>=4.0 -fakeredis>=2.0 +fakeredis>=2.0,<2.26 async_timeout>=3.0;python_version<"3.11" typing_extensions>=4.0;python_version<"3.11" \ No newline at end of file diff --git a/tests/unit/test_base_fetcher.py b/tests/unit/test_base_fetcher.py index adec7dd14..d086c900c 100644 --- a/tests/unit/test_base_fetcher.py +++ b/tests/unit/test_base_fetcher.py @@ -12,7 +12,6 @@ """ __author__ = 'JHao' -from lxml import etree from fetcher.baseFetcher import BaseFetcher @@ -60,108 +59,6 @@ def test_port_range(self): assert "1.2.3.4:65535" in result -class TestParseProxiesFromJson(object): - """parseProxiesFromJson 测试""" - - def test_dict_with_proxy_key(self): - data = {"proxy": "1.2.3.4:8080"} - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_dict_with_addr_key(self): - data = {"addr": "1.2.3.4:8080"} - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_dict_with_ip_port_keys(self): - data = {"ip": "1.2.3.4", "port": "8080"} - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_dict_with_host_port_keys(self): - data = {"host": "1.2.3.4", "port": "8080"} - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_dict_with_server_port_keys(self): - data = {"server": "1.2.3.4", "port": "8080"} - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_list_of_dicts(self): - data = [ - {"ip": "1.2.3.4", "port": "8080"}, - {"ip": "5.6.7.8", "port": "3128"}, - ] - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - assert "5.6.7.8:3128" in result - - def test_nested_dict(self): - data = { - "data": { - "items": [ - {"ip": "1.2.3.4", "port": "8080"} - ] - } - } - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_string_value(self): - data = "1.2.3.4:8080" - result = BaseFetcher.parseProxiesFromJson(data) - assert "1.2.3.4:8080" in result - - def test_empty_dict(self): - assert BaseFetcher.parseProxiesFromJson({}) == [] - - def test_empty_list(self): - assert BaseFetcher.parseProxiesFromJson([]) == [] - - -class TestParseProxiesFromTree(object): - """parseProxiesFromTree 测试""" - - def test_basic_table(self): - html = "
1.2.3.48080
" - tree = etree.HTML(html) - result = BaseFetcher.parseProxiesFromTree(tree) - assert "1.2.3.4:8080" in result - - def test_multiple_rows(self): - html = """ - - -
1.2.3.48080
5.6.7.83128
""" - tree = etree.HTML(html) - result = BaseFetcher.parseProxiesFromTree(tree) - assert "1.2.3.4:8080" in result - assert "5.6.7.8:3128" in result - - def test_none_tree(self): - assert BaseFetcher.parseProxiesFromTree(None) == [] - - def test_empty_table(self): - html = "
" - tree = etree.HTML(html) - assert BaseFetcher.parseProxiesFromTree(tree) == [] - - def test_header_row_skipped(self): - html = """ - - -
IPPort
1.2.3.48080
""" - tree = etree.HTML(html) - result = BaseFetcher.parseProxiesFromTree(tree) - assert "1.2.3.4:8080" in result - - def test_row_with_too_few_cells(self): - html = "
1.2.3.4
" - tree = etree.HTML(html) - assert BaseFetcher.parseProxiesFromTree(tree) == [] - - class TestYieldUniqueProxies(object): """yieldUniqueProxies 测试""" From 1605605a6ee756f91b884a466a91f1bf09f22416 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Sun, 31 May 2026 21:00:51 +0800 Subject: [PATCH 332/354] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=87=87?= =?UTF-8?q?=E9=9B=86=E5=99=A8=E8=8B=A5=E5=B9=B2=E9=97=AE=E9=A2=98=E5=B9=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A8=A1=E5=9D=97=E5=8A=A0=E8=BD=BD=E6=80=A7?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 采集器异常日志统一使用 LogHandler 替换 print (binglx/docip/geonode/scdn) - 修复 zdaye 跨天帖子时间判断 bug: seconds → total_seconds - helper/fetch.py 模块加载增加 mtime 缓存，避免每次调度重复 reload - 补充 zdaye 跨天场景单元测试 --- fetcher/sources/binglx.py | 5 ++- fetcher/sources/docip.py | 5 ++- fetcher/sources/geonode.py | 5 ++- fetcher/sources/scdn.py | 8 +++- fetcher/sources/zdaye.py | 2 +- helper/fetch.py | 63 ++++++++++++++++++++++-------- tests/unit/test_fetcher_sources.py | 17 ++++++++ 7 files changed, 83 insertions(+), 22 deletions(-) diff --git a/fetcher/sources/binglx.py b/fetcher/sources/binglx.py index 20ae93f36..7458bfd54 100644 --- a/fetcher/sources/binglx.py +++ b/fetcher/sources/binglx.py @@ -13,8 +13,11 @@ __author__ = 'JHao' from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler from util.webRequest import WebRequest +logger = LogHandler("fetcher") + class BinglxFetcher(BaseFetcher): """冰凌代理 https://www.binglx.cn""" @@ -30,7 +33,7 @@ def fetch(self): for tr in proxy_list[1:]: yield ':'.join(tr.xpath('./td/text()')[0:2]) except Exception as e: - print(e) + logger.error("ProxyFetch - binglx: %s" % e) if __name__ == '__main__': diff --git a/fetcher/sources/docip.py b/fetcher/sources/docip.py index 9abcbca1c..15cbeea33 100644 --- a/fetcher/sources/docip.py +++ b/fetcher/sources/docip.py @@ -13,8 +13,11 @@ __author__ = 'JHao' from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler from util.webRequest import WebRequest +logger = LogHandler("fetcher") + class DocipFetcher(BaseFetcher): """稻壳代理 https://www.docip.net/""" @@ -28,7 +31,7 @@ def fetch(self): for each in r.json['data']: yield each['ip'] except Exception as e: - print(e) + logger.error("ProxyFetch - docip: %s" % e) if __name__ == '__main__': diff --git a/fetcher/sources/geonode.py b/fetcher/sources/geonode.py index e7694d8ee..f974ea36a 100644 --- a/fetcher/sources/geonode.py +++ b/fetcher/sources/geonode.py @@ -13,8 +13,11 @@ __author__ = 'JHao' from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler from util.webRequest import WebRequest +logger = LogHandler("fetcher") + class GeonodeFetcher(BaseFetcher): """Geonode Free Proxy https://geonode.com/free-proxy-list/""" @@ -38,7 +41,7 @@ def fetch(self): for proxy in self.yieldUniqueProxies(proxies): yield proxy except Exception as e: - print(e) + logger.error("ProxyFetch - geonode: %s" % e) if __name__ == '__main__': diff --git a/fetcher/sources/scdn.py b/fetcher/sources/scdn.py index d360c786d..3d4ca9c29 100644 --- a/fetcher/sources/scdn.py +++ b/fetcher/sources/scdn.py @@ -17,8 +17,11 @@ from lxml import etree from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler from util.webRequest import WebRequest +logger = LogHandler("fetcher") + class ScdnFetcher(BaseFetcher): """SCDN 代理接口 https://proxy.scdn.io/""" @@ -45,7 +48,8 @@ def fetch(self): proxies.append("%s:%s" % (cells[0], cells[1])) if not proxies: - for item in data.get("data", []) if isinstance(data, dict) else []: + items = data.get("data", []) if isinstance(data, dict) else [] + for item in items: ip = item.get("ip", "") port = item.get("port", "") if ip and port: @@ -55,7 +59,7 @@ def fetch(self): for proxy in self.yieldUniqueProxies(proxies): yield proxy except Exception as e: - print(e) + logger.error("ProxyFetch - scdn: %s" % e) if __name__ == '__main__': diff --git a/fetcher/sources/zdaye.py b/fetcher/sources/zdaye.py index 56738d1e5..8eeda8b79 100644 --- a/fetcher/sources/zdaye.py +++ b/fetcher/sources/zdaye.py @@ -32,7 +32,7 @@ def fetch(self): "//span[@class='thread_time_info']/text()")[0].strip() interval = datetime.now() - datetime.strptime( latest_page_time, "%Y/%m/%d %H:%M:%S") - if interval.seconds < 300: + if interval.total_seconds() < 300: target_url = ("https://www.zdaye.com/" + html_tree.xpath("//h3[@class='thread_title']/a/@href")[0].strip()) while target_url: diff --git a/helper/fetch.py b/helper/fetch.py index ff27dca71..62970685a 100644 --- a/helper/fetch.py +++ b/helper/fetch.py @@ -24,39 +24,70 @@ from handler.configHandler import ConfigHandler from fetcher.baseFetcher import BaseFetcher +_logger = LogHandler("fetch") + +# 模块缓存: {module_name: (mtime, module)} +_module_cache = {} + def _get_sources_dir(): return os.path.join( os.path.dirname(os.path.abspath(__file__)), '..', 'fetcher', 'sources') +def _load_module(module_name, filepath): + """加载或 reload 模块，仅在文件 mtime 变化时 reload""" + global _module_cache + mtime = os.path.getmtime(filepath) + cached = _module_cache.get(module_name) + if cached and cached[0] == mtime: + return cached[1] + try: + if module_name in sys.modules: + module = importlib.reload(sys.modules[module_name]) + else: + module = importlib.import_module(module_name) + _module_cache[module_name] = (mtime, module) + return module + except Exception as e: + _logger.warning("ProxyFetch : load %s error - %s" % (module_name, e)) + return None + + def _discover_fetchers(exclude_list): """ 自动扫描 sources/ 目录，返回所有 enabled=True 且不在黑名单中的 fetcher 类列表。 - 每次调用重新加载模块，支持运行时热更新。 + 仅在文件 mtime 变化时重新加载模块，支持运行时热更新。 """ + global _module_cache sources_dir = _get_sources_dir() fetcher_classes = [] + seen_modules = set() + for filename in os.listdir(sources_dir): if not filename.endswith('.py') or filename.startswith('_'): continue module_name = "fetcher.sources.%s" % filename[:-3] - try: - if module_name in sys.modules: - module = importlib.reload(sys.modules[module_name]) - else: - module = importlib.import_module(module_name) - for attr_name in dir(module): - attr = getattr(module, attr_name, None) - if (attr and isinstance(attr, type) - and issubclass(attr, BaseFetcher) - and attr is not BaseFetcher - and attr.name - and attr.enabled - and attr.__name__ not in exclude_list): - fetcher_classes.append(attr) - except Exception: + seen_modules.add(module_name) + filepath = os.path.join(sources_dir, filename) + module = _load_module(module_name, filepath) + if module is None: continue + for attr_name in dir(module): + attr = getattr(module, attr_name, None) + if (attr and isinstance(attr, type) + and issubclass(attr, BaseFetcher) + and attr is not BaseFetcher + and attr.name + and attr.enabled + and attr.__name__ not in exclude_list): + fetcher_classes.append(attr) + + # 清理已删除文件的缓存 + for name in list(_module_cache): + if name not in seen_modules: + del _module_cache[name] + return sorted(fetcher_classes, key=lambda c: c.name) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index c4a38a36b..453ca7af5 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -323,6 +323,23 @@ def test_fetch_old_returns_empty(self, mock_dt, mock_wr): result = list(ZdayeFetcher().fetch()) assert result == [] + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_old_cross_day_returns_empty(self, mock_dt, mock_wr): + """跨天帖子应判定为过期（total_seconds 而非 seconds）""" + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 帖子是昨天 23:59，当前是今天 00:01（差 2 分钟，但跨天） + mock_dt.now.return_value = real_datetime(2026, 5, 31, 0, 1, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 30, 23, 59, 0) + + index_tree = etree.HTML( + '2026/05/30 23:59:00' + '

test

') + mock_wr.return_value.get.return_value = _make_response(tree=index_tree) + result = list(ZdayeFetcher().fetch()) + assert result == [] + class TestIhuanFetcher(object): From 2423aa0237e3c119b614a4e9cbdc587b75fe5488 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Sun, 31 May 2026 21:57:57 +0800 Subject: [PATCH 333/354] =?UTF-8?q?[update]=20=E4=BF=AE=E5=A4=8Dfakeredis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/redisClient.py | 1 + db/ssdbClient.py | 1 + requirements-test.txt | 3 ++- tests/conftest.py | 5 +++-- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/db/redisClient.py b/db/redisClient.py index e66614d7e..5f17e4c5a 100644 --- a/db/redisClient.py +++ b/db/redisClient.py @@ -45,6 +45,7 @@ def __init__(self, **kwargs): self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, timeout=5, socket_timeout=5, + protocol=2, **kwargs)) def get(self, https): diff --git a/db/ssdbClient.py b/db/ssdbClient.py index 0f5c00054..559539905 100644 --- a/db/ssdbClient.py +++ b/db/ssdbClient.py @@ -45,6 +45,7 @@ def __init__(self, **kwargs): self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, timeout=5, socket_timeout=5, + protocol=2, **kwargs)) def get(self, https): diff --git a/requirements-test.txt b/requirements-test.txt index 89e73e512..081f84acb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ pytest>=7.0 pytest-cov>=4.0 -fakeredis>=2.0,<2.26 +fakeredis>=2.0,<2.26;python_version<="3.8" +fakeredis>=2.26;python_version>"3.8" async_timeout>=3.0;python_version<"3.11" typing_extensions>=4.0;python_version<"3.11" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 4e3b5c128..4ec30bda6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ def https_proxy_obj(): @pytest.fixture def fake_redis(): """fakeredis 实例，用于 RedisClient/SsdbClient 测试""" - return fakeredis.FakeRedis(decode_responses=True) + return fakeredis.FakeRedis(decode_responses=True, protocol=2) @pytest.fixture @@ -73,7 +73,8 @@ def mock_db_client(fake_redis): def app(): """Flask app，proxy_handler 被 mock""" # mock 掉 DbClient，防止 ProxyHandler 连接真实 Redis - with patch("db.dbClient.DbClient") as mock_db_cls: + # 必须 patch handler.proxyHandler.DbClient（已 import 到本地命名空间） + with patch("handler.proxyHandler.DbClient") as mock_db_cls: mock_db_instance = MagicMock() mock_db_cls.return_value = mock_db_instance From b393014ce8dfc26c50e3c8b75b3cb23999024b79 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 21:17:36 +0800 Subject: [PATCH 334/354] =?UTF-8?q?[update]=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=20=20=E5=86=B0=E5=87=8C=E4=BB=A3?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fetcher/sources/binglx.py | 41 ------------------------------ tests/unit/test_fetcher_sources.py | 14 ---------- 2 files changed, 55 deletions(-) delete mode 100644 fetcher/sources/binglx.py diff --git a/fetcher/sources/binglx.py b/fetcher/sources/binglx.py deleted file mode 100644 index 7458bfd54..000000000 --- a/fetcher/sources/binglx.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： binglx.py - Description : 冰凌代理代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -from fetcher.baseFetcher import BaseFetcher -from handler.logHandler import LogHandler -from util.webRequest import WebRequest - -logger = LogHandler("fetcher") - - -class BinglxFetcher(BaseFetcher): - """冰凌代理 https://www.binglx.cn""" - - name = "binglx" - url = "https://www.binglx.cn/" - - def fetch(self): - url = "https://www.binglx.cn/?page=1" - try: - tree = WebRequest().get(url).tree - proxy_list = tree.xpath('.//table//tr') - for tr in proxy_list[1:]: - yield ':'.join(tr.xpath('./td/text()')[0:2]) - except Exception as e: - logger.error("ProxyFetch - binglx: %s" % e) - - -if __name__ == '__main__': - for proxy in BinglxFetcher().fetch(): - print(proxy) \ No newline at end of file diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index 453ca7af5..b641fec8b 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -57,7 +57,6 @@ class TestFetcherInterface(object): FETCHER_CLASSES = [ ("fetcher.sources.ip66", "Ip66Fetcher"), ("fetcher.sources.kxdaili", "KxdailiFetcher"), - ("fetcher.sources.binglx", "BinglxFetcher"), ("fetcher.sources.ip3366", "Ip3366Fetcher"), ("fetcher.sources.jiangxianli", "JiangxianliFetcher"), ("fetcher.sources.ip89", "Ip89Fetcher"), @@ -123,19 +122,6 @@ def test_fetch(self, mock_wr): assert "1.2.3.4:8080" in result -class TestBinglxFetcher(object): - - @patch("fetcher.sources.binglx.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.binglx import BinglxFetcher - # binglx 使用 proxy_list[1:] 跳过第一行，需要 header 行 - html = _html_table([("IP", "Port"), ("1.2.3.4", "8080")]) - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(BinglxFetcher().fetch()) - assert "1.2.3.4:8080" in result - - class TestIp3366Fetcher(object): @patch("fetcher.sources.ip3366.WebRequest") From 1731f55221f38ef6b1e290356cf5c002d78b3896 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 22:29:00 +0800 Subject: [PATCH 335/354] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20Proxifly?= =?UTF-8?q?=20=E4=BB=A3=E7=90=86=E6=BA=90=E5=8F=8A=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 fetcher/sources/proxifly.py 代理源 - 新增 TestProxiFlyFetcher 测试类（正常解析、非CN过滤、非http过滤） - TestFetcherInterface 注册 proxifly - 修复 url 拼写错误 hhttps -> https - 更新 changelog --- docs/changelog.md | 1 + fetcher/sources/proxifly.py | 42 ++++++++++++++++++++++++++++++ tests/unit/test_fetcher_sources.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 fetcher/sources/proxifly.py diff --git a/docs/changelog.md b/docs/changelog.md index 44ca09e20..791d12f1b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,7 @@ - 自动扫描启用的代理源，无需在配置中列举 - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 +8. 新增代理源 **Proxifly**; (2026-06-01) ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/proxifly.py b/fetcher/sources/proxifly.py new file mode 100644 index 000000000..4f8780ba0 --- /dev/null +++ b/fetcher/sources/proxifly.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： proxifly.py + Description : Proxifly代理源 + Author : JHao + date： 2026/06/01 +------------------------------------------------- + Change Activity: + 2026/06/01: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class ProxiFlyFetcher(BaseFetcher): + """Proxifly https://proxifly.dev""" + + name = "proxifly" + url = "https://proxifly.dev/" + + enabled = True # 是否启用 + + def fetch(self): + r = WebRequest().get("https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json", timeout=10) + try: + for each in r.json: + if each.get("geolocation", {}).get("country", "") == "CN" and each.get("protocol") == "http": + yield self.parseProxiesFromText(each.get('proxy', "")).pop() + except Exception as e: + logger.error("ProxyFetch - proxifly: %s" % e) + + +if __name__ == '__main__': + for proxy in ProxiFlyFetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index b641fec8b..88ea4d992 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -69,6 +69,7 @@ class TestFetcherInterface(object): ("fetcher.sources.scdn", "ScdnFetcher"), ("fetcher.sources.zdaye", "ZdayeFetcher"), ("fetcher.sources.ihuan", "IhuanFetcher"), + ("fetcher.sources.proxifly", "ProxiFlyFetcher"), ] def test_all_fetchers_have_name_url_enabled(self): @@ -355,3 +356,42 @@ def test_fetch_no_key_returns_empty(self, mock_wr): mock_wr.return_value.get.return_value = ti_resp result = list(IhuanFetcher().fetch()) assert result == [] + + +class TestProxiFlyFetcher(object): + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "5.6.7.8:3128", "protocol": "http", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_cn(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "http", "geolocation": {"country": "US"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_http(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "https", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result From c1aa529f3c85f0a52e4ffcdbd4c359f695a7cc80 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 22:36:23 +0800 Subject: [PATCH 336/354] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=20README=20?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E4=BB=A3=E7=90=86=E6=BA=90=E5=92=8C=E5=85=8D?= =?UTF-8?q?=E8=B4=B9=E4=BB=A3=E7=90=86=E6=BA=90=E8=A1=A8=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 扩展代理部分更新为 BaseFetcher 插件架构说明 - 免费代理源表格代码链接更新为 fetcher/sources/*.py - 新增 Proxifly 代理源到表格 - 配置部分更新为 PROXY_FETCHER_EXCLUDE 黑名单说明 --- README.md | 76 +++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0d080bf26..19b613582 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,10 @@ PORT = 5000 # 监听端口 DB_CONN = 'redis://:pwd@127.0.0.1:8888/0' -# 配置 ProxyFetcher - -PROXY_FETCHER = [ - "freeProxy01", # 这里是启用的代理抓取方法名，所有fetch方法位于fetcher/proxyFetcher.py - "freeProxy02", - # .... -] +# 配置代理源（可选） +# 默认自动扫描 fetcher/sources/ 目录下所有 enabled=True 的代理源 +# 如需禁用某些代理源，在黑名单中添加其 name 即可 +# PROXY_FETCHER_EXCLUDE = ["freevpnnode"] ``` #### 启动项目: @@ -167,59 +164,48 @@ def getHtml(): 　　添加一个新的代理源方法如下: -* 1、首先在[ProxyFetcher](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/fetcher/proxyFetcher.py#L21)类中添加自定义的获取代理的静态方法， -该方法需要以生成器(yield)形式返回`host:ip`格式的代理，例如: +* 1、在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher` 基类，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法以生成器(yield)形式返回`host:port`格式的代理，例如: ```python +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest -class ProxyFetcher(object): - # .... +class MyProxyFetcher(BaseFetcher): + """我的代理源""" - # 自定义代理源获取方法 - @staticmethod - def freeProxyCustom1(): # 命名不和已有重复即可 + name = "myproxy" + url = "https://www.example.com/" + enabled = True - # 通过某网站或者某接口或某数据库获取代理 - # 假设你已经拿到了一个代理列表 - proxies = ["x.x.x.x:3128", "x.x.x.x:80"] - for proxy in proxies: - yield proxy - # 确保每个proxy都是 host:ip正确的格式返回 + def fetch(self): + r = WebRequest().get("https://www.example.com/api/proxies") + for item in r.json: + yield item["ip"] + ":" + item["port"] ``` -* 2、添加好方法后，修改[setting.py](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/setting.py#L47)文件中的`PROXY_FETCHER`项： - -　　在`PROXY_FETCHER`下添加自定义方法的名字: - -```python -PROXY_FETCHER = [ - "freeProxy01", - "freeProxy02", - # .... - "freeProxyCustom1" # # 确保名字和你添加方法名字一致 -] -``` +* 2、添加好后，`schedule` 进程下次抓取时会自动扫描 `fetcher/sources/` 目录并启用新代理源，无需修改配置。 +　　可用 `python proxyPool.py fetcher` 命令查看当前启用的代理源列表。 -　　`schedule` 进程会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。 +　　如需临时禁用某个代理源，在 [setting.py](setting.py) 的 `PROXY_FETCHER_EXCLUDE` 黑名单中添加其 `name` 即可。 ### 免费代理源 - 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): + 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| - | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`freeProxy02`](/fetcher/proxyFetcher.py#L50) | - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`freeProxy03`](/fetcher/proxyFetcher.py#L63) | - | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeProxy04`](/fetcher/proxyFetcher.py#L74) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`freeProxy05`](/fetcher/proxyFetcher.py#L92) | - | 冰凌代理 | ✔ | ★★★ | * | [地址](https://www.binglx.cn/) | [`freeProxy06`](/fetcher/proxyFetcher.py#L111) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`freeProxy07`](/fetcher/proxyFetcher.py#L123) | - | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`freeProxy08`](/fetcher/proxyFetcher.py#L133) | - | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`freeProxy09`](/fetcher/proxyFetcher.py#L143) | - | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`freeProxy10`](/fetcher/proxyFetcher.py#L154) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`freeProxy11`](/fetcher/proxyFetcher.py#L164) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`freeProxy12`](/fetcher/proxyFetcher.py#L174) | + | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeproxylist.py`](/fetcher/sources/freeproxylist.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | + | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 From 5bd662bc9e35517e1ca888f3c1e8d720d40aacc9 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 21:38:12 +0800 Subject: [PATCH 337/354] =?UTF-8?q?[update]=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=E6=BA=90=20FreeProxyList?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/changelog.md | 1 + fetcher/sources/freeproxylist.py | 47 ------------------------------ tests/unit/test_fetcher_sources.py | 18 +----------- 4 files changed, 3 insertions(+), 65 deletions(-) delete mode 100644 fetcher/sources/freeproxylist.py diff --git a/README.md b/README.md index 19b613582..78ae7a3fb 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ class MyProxyFetcher(BaseFetcher): |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeproxylist.py`](/fetcher/sources/freeproxylist.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | diff --git a/docs/changelog.md b/docs/changelog.md index 791d12f1b..2867c4557 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 8. 新增代理源 **Proxifly**; (2026-06-01) +9. 移除失效代理源 **FreeProxyList**; (2026-06-02) ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/freeproxylist.py b/fetcher/sources/freeproxylist.py deleted file mode 100644 index e02b6e17e..000000000 --- a/fetcher/sources/freeproxylist.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： freeproxylist.py - Description : FreeProxyList代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -import re -from urllib import parse - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class FreeProxyListFetcher(BaseFetcher): - """FreeProxyList https://www.freeproxylists.net/zh/""" - - name = "freeproxylist" - url = "https://www.freeproxylists.net/zh/" - - @staticmethod - def _parse_ip(input_str): - html_str = parse.unquote(input_str) - ips = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', html_str) - return ips[0] if ips else None - - def fetch(self): - url = ("https://www.freeproxylists.net/zh/" - "?c=CN&pt=&pr=&a%5B%5D=0&a%5B%5D=1&a%5B%5D=2&u=50") - tree = WebRequest().get(url, verify=False).tree - for tr in tree.xpath("//tr[@class='Odd']") + tree.xpath("//tr[@class='Even']"): - ip = self._parse_ip("".join(tr.xpath('./td[1]/script/text()')).strip()) - port = "".join(tr.xpath('./td[2]/text()')).strip() - if ip: - yield "%s:%s" % (ip, port) - - -if __name__ == '__main__': - for proxy in FreeProxyListFetcher().fetch(): - print(proxy) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index 88ea4d992..d074a7f54 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -63,7 +63,7 @@ class TestFetcherInterface(object): ("fetcher.sources.docip", "DocipFetcher"), ("fetcher.sources.goodips", "GoodipsFetcher"), ("fetcher.sources.geonode", "GeonodeFetcher"), - ("fetcher.sources.freeproxylist", "FreeProxyListFetcher"), + ("fetcher.sources.kuaidaili", "KuaidailiFetcher"), ("fetcher.sources.freevpnnode", "FreeVPNNodeFetcher"), ("fetcher.sources.scdn", "ScdnFetcher"), @@ -201,22 +201,6 @@ def test_fetch_text_fallback(self, mock_wr): assert "1.2.3.4:8080" in result -class TestFreeProxyListFetcher(object): - - @patch("fetcher.sources.freeproxylist.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.freeproxylist import FreeProxyListFetcher - # FreeProxyList 使用 JS 混淆 IP，这里模拟 script 标签中包含编码后的 IP - script_content = "document.write('%31%2E%32%2E%33%2E%34')" - html = '
8080
' % script_content - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(FreeProxyListFetcher().fetch()) - # 注意：实际 JS 解码逻辑可能需要更复杂的 mock - # 这里主要验证 fetch() 不报错且返回列表 - assert isinstance(result, list) - - class TestKuaidailiFetcher(object): @patch("fetcher.sources.kuaidaili.WebRequest") From e7488e216557a972699b64fe63e516db23992688 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 22:05:42 +0800 Subject: [PATCH 338/354] fix: pin werkzeug<2.2 to fix url_quote ImportError with Flask 2.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f648f70b3..c658ae713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ APScheduler==3.10.0;python_version>="3.10" APScheduler==3.2.0;python_version<"3.10" click==8.0.1 Flask==2.1.1 -werkzeug==3.1.6 +werkzeug>=2.0,<2.2 From 72b755f335b0c458cdf57ca6b637720ff172487d Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 22:23:58 +0800 Subject: [PATCH 339/354] docs: update badges - tests badge point to default branch, add codecov badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78ae7a3fb..c30e013d1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ ProxyPool 爬虫代理IP池 ======= -[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![codecov](https://codecov.io/gh/jhao104/proxy_pool/graph/badge.svg?token=8WHGkrQA6E)](https://codecov.io/gh/jhao104/proxy_pool) [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/) [![Packagist](https://img.shields.io/packagist/l/doctrine/orm.svg)](https://github.com/jhao104/proxy_pool/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/jhao104/proxy_pool.svg)](https://github.com/jhao104/proxy_pool/graphs/contributors) From e1c4f2d0fed8047a44feaba85667d865362917cf Mon Sep 17 00:00:00 2001 From: jhao104 Date: Wed, 3 Jun 2026 22:21:45 +0800 Subject: [PATCH 340/354] refactor: update ihuan/geonode fetchers and fix ihuan tests - ihuan: rewrite to use session-based GET + HTML table parsing - geonode: update URL and add filterLastChecked param - tests: rewrite ihuan tests to match new session-based implementation --- README.md | 1 - fetcher/sources/geonode.py | 6 +-- fetcher/sources/ihuan.py | 69 ++++++------------------------ tests/unit/test_fetcher_sources.py | 43 +++++++++++-------- 4 files changed, 42 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index c30e013d1..6f68b7cf7 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,6 @@ class MyProxyFetcher(BaseFetcher): |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | diff --git a/fetcher/sources/geonode.py b/fetcher/sources/geonode.py index f974ea36a..8e3171047 100644 --- a/fetcher/sources/geonode.py +++ b/fetcher/sources/geonode.py @@ -20,14 +20,14 @@ class GeonodeFetcher(BaseFetcher): - """Geonode Free Proxy https://geonode.com/free-proxy-list/""" + """Geonode Free Proxy https://geonode.com/""" name = "geonode" - url = "https://geonode.com/free-proxy-list/" + url = "https://geonode.com/" def fetch(self): url = ("https://proxylist.geonode.com/api/proxy-list?" - "limit=500&page=1&sort_by=lastChecked&sort_type=desc") + "filterLastChecked=10&page=1&limit=100&sort_by=lastChecked&sort_type=desc") r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) try: proxies = [] diff --git a/fetcher/sources/ihuan.py b/fetcher/sources/ihuan.py index 1d9bcf607..2981f1133 100644 --- a/fetcher/sources/ihuan.py +++ b/fetcher/sources/ihuan.py @@ -12,7 +12,8 @@ """ __author__ = 'JHao' -import re +from lxml import etree +import requests from fetcher.baseFetcher import BaseFetcher from util.webRequest import WebRequest @@ -23,62 +24,20 @@ class IhuanFetcher(BaseFetcher): name = "ihuan" url = "https://ip.ihuan.me/" + enabled = True def fetch(self): - request = WebRequest() - ti_url = "https://ip.ihuan.me/ti.html" - tqdl_url = "https://ip.ihuan.me/tqdl.html" - ti_resp = request.get(ti_url, timeout=10, verify=False) - form_data = {} - if ti_resp.tree is not None: - for input_tag in ti_resp.tree.xpath("//form//input[@name]"): - name = "".join(input_tag.xpath("./@name")).strip() - value = "".join(input_tag.xpath("./@value")).strip() - if name: - form_data[name] = value - - key = form_data.get("key") - if not key: - key_match = re.search( - r'name=["\']key["\'][^>]*value=["\']([^"\']+)', ti_resp.text) - if not key_match: - key_match = re.search( - r'key["\']?\s*[:=]\s*["\']([0-9a-f]{16,})', ti_resp.text) - key = key_match.group(1) if key_match else "" - - if not key: - return - - header = { - "Origin": "https://ip.ihuan.me", - "Referer": ti_url, - } - data = form_data.copy() - data.update({ - "num": "2000", - "port": "", - "kill_port": "", - "address": "", - "kill_address": "", - "anonymity": "", - "type": "", - "post": "", - "sort": "1", - "key": key, - }) - r = request.post(tqdl_url, header=header, data=data, timeout=10, verify=False) - proxies = [] - if r.tree is not None: - for tr in r.tree.xpath("//tr"): - cells = [" ".join(td.xpath(".//text()")).strip() for td in tr.xpath("./td")] - if len(cells) >= 2: - ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) - port_match = re.match(r'^\d{2,5}$', cells[1]) - if ip_match and port_match: - proxies.append("%s:%s" % (cells[0], cells[1])) - proxies.extend(self.parseProxiesFromText(r.text)) - for proxy in self.yieldUniqueProxies(proxies): - yield proxy + wb = WebRequest() + session = requests.session() + headers = wb.header + session.get(self.url, headers=headers, verify=False) # 必须先请求一起获取cookie + res = session.get(self.url, headers=headers, verify=False) + tree = etree.HTML(res.text) + for item in tree.xpath("//table[@class='table table-hover table-bordered']//tr"): + ip = "".join(item.xpath("./td[1]//text()")).strip() + port = "".join(item.xpath("./td[2]//text()")).strip() + if ip and port: + yield "%s:%s" % (ip, port) if __name__ == '__main__': diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index d074a7f54..abed28b22 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -314,30 +314,37 @@ def test_fetch_old_cross_day_returns_empty(self, mock_dt, mock_wr): class TestIhuanFetcher(object): - @patch("fetcher.sources.ihuan.WebRequest") - def test_fetch(self, mock_wr): + @patch("fetcher.sources.ihuan.requests") + def test_fetch(self, mock_requests): from fetcher.sources.ihuan import IhuanFetcher - ti_tree = etree.HTML( - '
') - post_tree = etree.HTML(_html_table([("1.2.3.4", "8080")])) - - ti_resp = _make_response(tree=ti_tree, text="") - post_resp = _make_response(tree=post_tree, text="1.2.3.4:8080") - - mock_instance = MagicMock() - mock_instance.get.return_value = ti_resp - mock_instance.post.return_value = post_resp - mock_wr.return_value = mock_instance + html = ( + '' + '' + '' + '
1.2.3.48080
5.6.7.83128
' + ) + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.text = html + # 第一次 get 获取 cookie，第二次 get 返回数据 + mock_session.get.return_value = mock_resp + mock_requests.session.return_value = mock_session result = list(IhuanFetcher().fetch()) assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + assert mock_session.get.call_count == 2 - @patch("fetcher.sources.ihuan.WebRequest") - def test_fetch_no_key_returns_empty(self, mock_wr): + @patch("fetcher.sources.ihuan.requests") + def test_fetch_empty_table_returns_empty(self, mock_requests): from fetcher.sources.ihuan import IhuanFetcher - ti_tree = etree.HTML('
') - ti_resp = _make_response(tree=ti_tree, text="no key here") - mock_wr.return_value.get.return_value = ti_resp + html = '
' + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.text = html + mock_session.get.return_value = mock_resp + mock_requests.session.return_value = mock_session + result = list(IhuanFetcher().fetch()) assert result == [] From 165c361c8e1ded0d13f8020e81c72bb6e581885d Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 8 Jun 2026 21:27:29 +0800 Subject: [PATCH 341/354] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E6=BA=90daili66,=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=E6=BA=90ip66?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 fetcher/sources/daili66.py (66代理 JSON API) - 删除 fetcher/sources/ip66.py (已失效) - 新增 TestDaiLi66Fetcher 测试类 - 更新 README.md、docs/project-structure.md、docs/changelog.md --- README.md | 26 +++++++++--------- docs/changelog.md | 2 ++ docs/project-structure.md | 1 - fetcher/sources/daili66.py | 42 ++++++++++++++++++++++++++++++ fetcher/sources/freevpnnode.py | 4 +-- fetcher/sources/ip66.py | 37 -------------------------- tests/unit/test_fetcher_sources.py | 41 +++++++++++++++++------------ 7 files changed, 85 insertions(+), 68 deletions(-) create mode 100644 fetcher/sources/daili66.py delete mode 100644 fetcher/sources/ip66.py diff --git a/README.md b/README.md index 6f68b7cf7..f8869590c 100644 --- a/README.md +++ b/README.md @@ -194,18 +194,20 @@ class MyProxyFetcher(BaseFetcher): 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): - | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | - |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| - | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | - | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | - | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | - | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | - | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | + |-------------| ---- | -------- |----|----------------------------------|-----------------------------------------------------| + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | + | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | + | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 diff --git a/docs/changelog.md b/docs/changelog.md index 2867c4557..453e515d6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,8 @@ - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 8. 新增代理源 **Proxifly**; (2026-06-01) 9. 移除失效代理源 **FreeProxyList**; (2026-06-02) +10. 新增代理源 **66代理 (daili66)**; (2026-06-08) +11. 移除失效代理源 **66代理 (ip66)**; (2026-06-08) ## 2.4.2 (2024-01-18) diff --git a/docs/project-structure.md b/docs/project-structure.md index 795650b89..30077d8b2 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -14,7 +14,6 @@ proxy_pool/ │ ├── baseFetcher.py # BaseFetcher 基类（共享解析方法） │ └── sources/ # 各代理源独立文件 │ ├── zdaye.py # 站大爷 -│ ├── ip66.py # 代理66 │ ├── kxdaili.py # 开心代理 │ ├── kuaidaili.py # 快代理 │ ├── geonode.py # Geonode diff --git a/fetcher/sources/daili66.py b/fetcher/sources/daili66.py new file mode 100644 index 000000000..6e6dd005c --- /dev/null +++ b/fetcher/sources/daili66.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： daili66.py + Description : 66代理 + Author : JHao + date： 2026/06/08 +------------------------------------------------- + Change Activity: + 2026/06/08: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + +class DaiLi66Fetcher(BaseFetcher): + """66代理 https://www.66daili.com""" + + name = "daili66" + url = "https://www.66daili.com" + + enabled = True + + def fetch(self): + url = "http://api.66daili.com/?format=json" + r = WebRequest().get(url, timeout=10) + try: + for each in r.json.get("data", []): + yield "%s:%s" % (each["ip"], each["port"]) + except Exception as e: + logger.error("ProxyFetch - daili66: %s" % e) + + + +if __name__ == '__main__': + for proxy in DaiLi66Fetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/freevpnnode.py b/fetcher/sources/freevpnnode.py index 45e4e4619..27e871e03 100644 --- a/fetcher/sources/freevpnnode.py +++ b/fetcher/sources/freevpnnode.py @@ -19,10 +19,10 @@ class FreeVPNNodeFetcher(BaseFetcher): - """FreeVPNNode https://cn.freevpnnode.com/free-proxy/""" + """FreeVPNNode https://cn.freevpnnode.com""" name = "freevpnnode" - url = "https://cn.freevpnnode.com/free-proxy/" + url = "https://cn.freevpnnode.com" def fetch(self): url = "https://cn.freevpnnode.com/free-proxy/" diff --git a/fetcher/sources/ip66.py b/fetcher/sources/ip66.py deleted file mode 100644 index 3cec44f70..000000000 --- a/fetcher/sources/ip66.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： ip66.py - Description : 代理66代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class Ip66Fetcher(BaseFetcher): - """代理66 http://www.66ip.cn/""" - - name = "ip66" - url = "http://www.66ip.cn/" - - def fetch(self): - url = "http://www.66ip.cn/" - resp = WebRequest().get(url, timeout=10).tree - for i, tr in enumerate(resp.xpath("(//table)[3]//tr")): - if i > 0: - ip = "".join(tr.xpath("./td[1]/text()")).strip() - port = "".join(tr.xpath("./td[2]/text()")).strip() - yield "%s:%s" % (ip, port) - - -if __name__ == '__main__': - for proxy in Ip66Fetcher().fetch(): - print(proxy) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index abed28b22..eda33456e 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -55,7 +55,6 @@ class TestFetcherInterface(object): """所有 fetcher 的接口约定""" FETCHER_CLASSES = [ - ("fetcher.sources.ip66", "Ip66Fetcher"), ("fetcher.sources.kxdaili", "KxdailiFetcher"), ("fetcher.sources.ip3366", "Ip3366Fetcher"), ("fetcher.sources.jiangxianli", "JiangxianliFetcher"), @@ -70,6 +69,7 @@ class TestFetcherInterface(object): ("fetcher.sources.zdaye", "ZdayeFetcher"), ("fetcher.sources.ihuan", "IhuanFetcher"), ("fetcher.sources.proxifly", "ProxiFlyFetcher"), + ("fetcher.sources.daili66", "DaiLi66Fetcher"), ] def test_all_fetchers_have_name_url_enabled(self): @@ -95,21 +95,6 @@ def test_all_fetchers_have_fetch_method(self): # --------------- 各 fetcher 逻辑测试 --------------- -class TestIp66Fetcher(object): - - @patch("fetcher.sources.ip66.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.ip66 import Ip66Fetcher - # ip66 使用 (//table)[3] 取第3个table，if i > 0 跳过第一行 - html = ("
" - + _html_table([("IP", "Port"), ("1.2.3.4", "8080"), ("5.6.7.8", "3128")])) - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(Ip66Fetcher().fetch()) - assert "1.2.3.4:8080" in result - assert "5.6.7.8:3128" in result - - class TestKxdailiFetcher(object): @patch("fetcher.sources.kxdaili.WebRequest") @@ -386,3 +371,27 @@ def test_fetch_filters_non_http(self, mock_wr): result = list(ProxiFlyFetcher().fetch()) assert "1.2.3.4:8080" in result assert "9.9.9.9:8080" not in result + + +class TestDaiLi66Fetcher(object): + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + json_data = { + "data": [ + {"ip": "1.2.3.4", "port": "8080"}, + {"ip": "5.6.7.8", "port": "3128"}, + ] + } + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(DaiLi66Fetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch_empty_data_returns_empty(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + mock_wr.return_value.get.return_value = _make_response(json_data={}) + result = list(DaiLi66Fetcher().fetch()) + assert result == [] From c742f84bc4616d0d3ab5ee3b6fd4b99f84eeeea0 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 9 Jun 2026 20:58:31 +0800 Subject: [PATCH 342/354] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E6=BA=90RoundProxies,=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=E6=BA=90jiangxianli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增代理源 RoundProxies (roundproxies.py)，通过 API 获取免费代理列表 - 移除失效代理源 免费代理库 (jiangxianli.py) - 新增 RoundProxiesFetcher 单元测试（正常/空数据/异常三个用例） - 更新 README.md 代理源列表和 docs/changelog.md --- README.md | 28 +++++++++---------- docs/changelog.md | 3 ++ fetcher/sources/jiangxianli.py | 37 ------------------------ fetcher/sources/roundproxies.py | 43 ++++++++++++++++++++++++++++ tests/unit/test_fetcher_sources.py | 45 +++++++++++++++++++++--------- 5 files changed, 92 insertions(+), 64 deletions(-) delete mode 100644 fetcher/sources/jiangxianli.py create mode 100644 fetcher/sources/roundproxies.py diff --git a/README.md b/README.md index f8869590c..640c1e157 100644 --- a/README.md +++ b/README.md @@ -194,20 +194,20 @@ class MyProxyFetcher(BaseFetcher): 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): - | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | - |-------------| ---- | -------- |----|----------------------------------|-----------------------------------------------------| - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | - | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | - | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | - | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | - | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | - | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | - | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | - | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | + | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | + |--------------| ---- |------|-----|---------------------------------------------------|---------------------------------------------------------| + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 89代理 | ✔ | ★★ | ** | [地址](https://www.89ip.cn) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | + | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | + | RoundProxies | ✔ | ★ | * | [地址](https://roundproxies.com/free-proxy-list) | [`roundproxies.py`](/fetcher/sources/roundproxies.py) | 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 diff --git a/docs/changelog.md b/docs/changelog.md index 453e515d6..75001fac0 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -17,6 +17,9 @@ 9. 移除失效代理源 **FreeProxyList**; (2026-06-02) 10. 新增代理源 **66代理 (daili66)**; (2026-06-08) 11. 移除失效代理源 **66代理 (ip66)**; (2026-06-08) +12. 新增代理源 **RoundProxies**; (2026-06-09) +13. 移除失效代理源 **免费代理库 (jiangxianli)**; (2026-06-09) + ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/jiangxianli.py b/fetcher/sources/jiangxianli.py deleted file mode 100644 index 2650fd012..000000000 --- a/fetcher/sources/jiangxianli.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： jiangxianli.py - Description : 免费代理库代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class JiangxianliFetcher(BaseFetcher): - """免费代理库 http://ip.jiangxianli.com/""" - - name = "jiangxianli" - url = "http://ip.jiangxianli.com/" - - def fetch(self, page_count=1): - for i in range(1, page_count + 1): - url = 'http://ip.jiangxianli.com/?country=中国&page={}'.format(i) - html_tree = WebRequest().get(url, verify=False).tree - for index, tr in enumerate(html_tree.xpath("//table//tr")): - if index == 0: - continue - yield ":".join(tr.xpath("./td/text()")[0:2]).strip() - - -if __name__ == '__main__': - for proxy in JiangxianliFetcher().fetch(): - print(proxy) diff --git a/fetcher/sources/roundproxies.py b/fetcher/sources/roundproxies.py new file mode 100644 index 000000000..ba1d0af36 --- /dev/null +++ b/fetcher/sources/roundproxies.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： roundproxies.py + Description : Roundproxies + Author : JHao + date： 2026/06/09 +------------------------------------------------- + Change Activity: + 2026/06/09: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + +class RoundProxiesFetcher(BaseFetcher): + """Roundproxies https://roundproxies.com/free-proxy-list""" + + name = "roundproxies" + url = "https://roundproxies.com/free-proxy-list" + + enabled = True + + def fetch(self): + page_size = 50 + _url = f"https://roundproxies.com/api/get-free-proxies/?limit={page_size}&page=1&sort_by=lastChecked&sort_type=desc" + r = WebRequest().get(_url, timeout=10) + try: + for each in r.json.get("data", []): + yield "%s:%s" % (each["ip"], each["port"]) + except Exception as e: + logger.error("ProxyFetch - roundproxies: %s" % e) + + + +if __name__ == '__main__': + for proxy in RoundProxiesFetcher().fetch(): + print(proxy) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index eda33456e..0c1649b6c 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -57,7 +57,6 @@ class TestFetcherInterface(object): FETCHER_CLASSES = [ ("fetcher.sources.kxdaili", "KxdailiFetcher"), ("fetcher.sources.ip3366", "Ip3366Fetcher"), - ("fetcher.sources.jiangxianli", "JiangxianliFetcher"), ("fetcher.sources.ip89", "Ip89Fetcher"), ("fetcher.sources.docip", "DocipFetcher"), ("fetcher.sources.goodips", "GoodipsFetcher"), @@ -70,6 +69,7 @@ class TestFetcherInterface(object): ("fetcher.sources.ihuan", "IhuanFetcher"), ("fetcher.sources.proxifly", "ProxiFlyFetcher"), ("fetcher.sources.daili66", "DaiLi66Fetcher"), + ("fetcher.sources.roundproxies", "RoundProxiesFetcher"), ] def test_all_fetchers_have_name_url_enabled(self): @@ -120,18 +120,6 @@ def test_fetch(self, mock_wr): assert "5.6.7.8:3128" in result -class TestJiangxianliFetcher(object): - - @patch("fetcher.sources.jiangxianli.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.jiangxianli import JiangxianliFetcher - html = _html_table([("IP", "Port"), ("1.2.3.4", "8080")]) - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(JiangxianliFetcher().fetch()) - assert "1.2.3.4:8080" in result - - class TestIp89Fetcher(object): @patch("fetcher.sources.ip89.WebRequest") @@ -395,3 +383,34 @@ def test_fetch_empty_data_returns_empty(self, mock_wr): mock_wr.return_value.get.return_value = _make_response(json_data={}) result = list(DaiLi66Fetcher().fetch()) assert result == [] + + +class TestRoundProxiesFetcher(object): + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + json_data = { + "data": [ + {"ip": "1.2.3.4", "port": "8080"}, + {"ip": "5.6.7.8", "port": "3128"}, + ] + } + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(RoundProxiesFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch_empty_data_returns_empty(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + mock_wr.return_value.get.return_value = _make_response(json_data={}) + result = list(RoundProxiesFetcher().fetch()) + assert result == [] + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch_exception_returns_empty(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + mock_wr.return_value.get.return_value = _make_response(json_data=None) + result = list(RoundProxiesFetcher().fetch()) + assert result == [] From 5022b576b937eea51c1c94ff10236d6f44655729 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 20:40:00 +0800 Subject: [PATCH 343/354] test: add ProxyHandler unit tests (B1) Tests for all ProxyHandler methods: get, pop, put, delete, getAll, exists, getCount. Mock DbClient to avoid external dependencies. 11 new tests, all passing. --- tests/unit/test_proxy_handler.py | 178 +++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 tests/unit/test_proxy_handler.py diff --git a/tests/unit/test_proxy_handler.py b/tests/unit/test_proxy_handler.py new file mode 100644 index 000000000..441b89a75 --- /dev/null +++ b/tests/unit/test_proxy_handler.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_proxy_handler.py + Description : ProxyHandler 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import MagicMock, patch + +from handler.proxyHandler import ProxyHandler +from helper.proxy import Proxy + + +def _make_handler(): + """构造注入 mock DbClient 的 ProxyHandler 实例""" + with patch("handler.proxyHandler.DbClient") as mock_db_cls, \ + patch("handler.proxyHandler.ConfigHandler") as mock_conf_cls: + mock_db = MagicMock() + mock_db_cls.return_value = mock_db + mock_conf = MagicMock() + mock_conf.dbConn = "redis://:test@127.0.0.1:6379/0" + mock_conf.tableName = "use_proxy" + mock_conf_cls.return_value = mock_conf + handler = ProxyHandler() + handler._mock_db = mock_db + return handler + + +class TestProxyHandlerGet: + """get() 测试""" + + def test_get_returns_proxy(self): + """DbClient 返回 JSON -> Proxy 对象""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test", https=False) + handler._mock_db.get.return_value = proxy.to_json + + result = handler.get(https=False) + + assert result is not None + assert result.proxy == "1.2.3.4:8080" + assert result.https is False + handler._mock_db.get.assert_called_once_with(False) + + def test_get_returns_none_when_empty(self): + """DbClient 返回 None -> None""" + handler = _make_handler() + handler._mock_db.get.return_value = None + + result = handler.get(https=False) + + assert result is None + + def test_get_https_forwarded(self): + """https=True 转发给 DbClient""" + handler = _make_handler() + proxy = Proxy("5.6.7.8:443", source="test", https=True) + handler._mock_db.get.return_value = proxy.to_json + + result = handler.get(https=True) + + assert result is not None + assert result.https is True + handler._mock_db.get.assert_called_once_with(True) + + +class TestProxyHandlerPop: + """pop() 测试""" + + def test_pop_returns_proxy(self): + """pop 正常返回 Proxy 对象""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + handler._mock_db.pop.return_value = proxy.to_json + + result = handler.pop(https=False) + + assert result is not None + assert result.proxy == "1.2.3.4:8080" + handler._mock_db.pop.assert_called_once_with(False) + + def test_pop_returns_none_when_empty(self): + """pop 无数据时返回 None""" + handler = _make_handler() + handler._mock_db.pop.return_value = None + + result = handler.pop(https=False) + + assert result is None + + +class TestProxyHandlerPut: + """put() 测试""" + + def test_put_delegates_to_db(self): + """put 调用 DbClient.put""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + + handler.put(proxy) + + handler._mock_db.put.assert_called_once_with(proxy) + + +class TestProxyHandlerDelete: + """delete() 测试""" + + def test_delete_delegates_to_db(self): + """delete 传入 proxy.proxy 字符串给 DbClient""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + + handler.delete(proxy) + + handler._mock_db.delete.assert_called_once_with("1.2.3.4:8080") + + +class TestProxyHandlerGetAll: + """getAll() 测试""" + + def test_getAll_returns_proxy_list(self): + """getAll 返回 Proxy 对象列表""" + handler = _make_handler() + proxy1 = Proxy("1.2.3.4:8080", source="test").to_json + proxy2 = Proxy("5.6.7.8:443", source="test", https=True).to_json + handler._mock_db.getAll.return_value = [proxy1, proxy2] + + result = handler.getAll(https=False) + + assert len(result) == 2 + assert result[0].proxy == "1.2.3.4:8080" + assert result[1].proxy == "5.6.7.8:443" + handler._mock_db.getAll.assert_called_once_with(False) + + def test_getAll_empty_returns_empty_list(self): + """getAll 无数据返回空列表""" + handler = _make_handler() + handler._mock_db.getAll.return_value = [] + + result = handler.getAll() + + assert result == [] + + +class TestProxyHandlerExists: + """exists() 测试""" + + def test_exists_delegates_to_db(self): + """exists 传入 proxy.proxy 字符串给 DbClient""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + handler._mock_db.exists.return_value = True + + result = handler.exists(proxy) + + assert result is True + handler._mock_db.exists.assert_called_once_with("1.2.3.4:8080") + + +class TestProxyHandlerGetCount: + """getCount() 测试""" + + def test_getCount_returns_dict(self): + """getCount 返回 {'count': N}""" + handler = _make_handler() + handler._mock_db.getCount.return_value = 42 + + result = handler.getCount() + + assert result == {"count": 42} \ No newline at end of file From 8e7682620b04c21a1b38534a20ba6d2ea5751fa4 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 20:45:59 +0800 Subject: [PATCH 344/354] test: add LogHandler unit tests (B2) Tests for init (stream/file/platform), stream level override, file level override, log dir creation. 10 new tests, all passing. --- tests/unit/test_log_handler.py | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/unit/test_log_handler.py diff --git a/tests/unit/test_log_handler.py b/tests/unit/test_log_handler.py new file mode 100644 index 000000000..10f099466 --- /dev/null +++ b/tests/unit/test_log_handler.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_log_handler.py + Description : LogHandler 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import logging +import pytest +from unittest.mock import patch, MagicMock + +from handler.logHandler import LogHandler, DEBUG, INFO, ERROR + + +class TestLogHandlerInit: + """__init__ 测试""" + + def test_default_creates_stream_handler(self): + """默认参数创建 stream handler""" + log = LogHandler("test_default_stream", stream=True, file=False) + handler_types = [type(h) for h in log.handlers] + assert logging.StreamHandler in handler_types + + @patch("handler.logHandler.platform") + def test_file_handler_on_linux(self, mock_platform): + """Linux 下创建 file handler""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_linux_file", stream=False, file=True) + has_file = any(isinstance(h, logging.handlers.TimedRotatingFileHandler) for h in log.handlers) + assert has_file + + @patch("handler.logHandler.platform") + def test_no_file_handler_on_windows(self, mock_platform): + """Windows 下不创建 file handler""" + mock_platform.system.return_value = "Windows" + log = LogHandler("test_windows_no_file", stream=False, file=True) + has_file = any(isinstance(h, logging.handlers.TimedRotatingFileHandler) for h in log.handlers) + assert not has_file + + def test_no_stream_handler_when_disabled(self): + """stream=False 时不创建 stream handler""" + log = LogHandler("test_no_stream", stream=False, file=False) + assert len(log.handlers) == 0 + + +class TestLogHandlerStreamLevel: + """stream handler level 测试""" + + def test_default_level_used_when_no_override(self): + """未指定 level 时使用 self.level""" + log = LogHandler("test_stream_level", level=ERROR, stream=True, file=False) + stream_handlers = [h for h in log.handlers if isinstance(h, logging.StreamHandler) + and not isinstance(h, logging.handlers.TimedRotatingFileHandler)] + assert len(stream_handlers) > 0 + assert stream_handlers[0].level == ERROR + + def test_explicit_level_overrides_default(self): + """显式指定 level 覆盖默认值""" + log = LogHandler("test_stream_override", level=DEBUG, stream=True, file=False) + log.__setStreamHandler__(level=ERROR) + # 最后添加的 handler 应该是 ERROR 级别 + last_handler = log.handlers[-1] + assert last_handler.level == ERROR + + +class TestLogHandlerFileLevel: + """file handler level 测试""" + + @patch("handler.logHandler.platform") + def test_file_handler_default_level(self, mock_platform): + """file handler 未指定 level 时使用 self.level""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_file_level", level=INFO, stream=False, file=True) + file_handlers = [h for h in log.handlers + if isinstance(h, logging.handlers.TimedRotatingFileHandler)] + assert len(file_handlers) > 0 + assert file_handlers[0].level == INFO + + @patch("handler.logHandler.platform") + def test_file_handler_explicit_level(self, mock_platform): + """file handler 显式指定 level""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_file_override", level=DEBUG, stream=False, file=True) + log.__setFileHandler__(level=ERROR) + last_handler = log.handlers[-1] + assert last_handler.level == ERROR + + +class TestLogHandlerDirCreation: + """log 目录创建测试""" + + @patch("os.path.exists", return_value=False) + @patch("os.mkdir") + def test_creates_log_dir_when_missing(self, mock_mkdir, mock_exists): + """log 目录不存在时创建""" + # 重新 import 触发模块级代码（无法直接测试，验证模块级逻辑） + # 这里测试的是 FileExistsError 处理 + import handler.logHandler as lh + # 模块加载时已执行，此处验证 LOG_PATH 存在 + assert lh.LOG_PATH is not None + + @patch("os.path.exists", return_value=False) + @patch("os.mkdir", side_effect=FileExistsError) + def test_handles_file_exists_race_condition(self, mock_mkdir, mock_exists): + """处理 mkdir 时的 FileExistsError 竞态条件""" + # 验证模块级代码不会因 FileExistsError 崩溃 + import handler.logHandler as lh + assert lh.LOG_PATH is not None \ No newline at end of file From 48787379fb5460425b40711598b19566d7d1dfb8 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 20:56:06 +0800 Subject: [PATCH 345/354] test: extend validator tests with HTTP/HTTPS timeout and custom validator (B3) Add tests for httpTimeOutValidator, httpsTimeOutValidator, customValidatorExample. Mock requests.head to avoid real HTTP calls. 7 new tests, all passing. --- tests/unit/test_validator.py | 60 ++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index 25f374078..effe0e24d 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -14,9 +14,10 @@ import re import pytest +from unittest.mock import patch, MagicMock # 直接导入 IP_REGEX 和 formatValidator，不导入整个 validator 模块（避免模块级副作用） -from helper.validator import IP_REGEX, formatValidator +from helper.validator import IP_REGEX, formatValidator, httpTimeOutValidator, httpsTimeOutValidator, customValidatorExample class TestIPRegex: @@ -65,4 +66,59 @@ def test_valid_returns_true(self, proxy): "1.2.3.4", ]) def test_invalid_returns_false(self, proxy): - assert formatValidator(proxy) is False \ No newline at end of file + assert formatValidator(proxy) is False + + +class TestHttpTimeOutValidator: + """httpTimeOutValidator 测试""" + + @patch("helper.validator.head") + def test_returns_true_on_200(self, mock_head): + """status_code=200 -> True""" + mock_head.return_value = MagicMock(status_code=200) + assert httpTimeOutValidator("1.2.3.4:8080") is True + + @patch("helper.validator.head") + def test_returns_false_on_non_200(self, mock_head): + """status_code=502 -> False""" + mock_head.return_value = MagicMock(status_code=502) + assert httpTimeOutValidator("1.2.3.4:8080") is False + + @patch("helper.validator.head") + def test_returns_false_on_exception(self, mock_head): + """head() raise Timeout -> False""" + mock_head.side_effect = TimeoutError("connection timed out") + assert httpTimeOutValidator("1.2.3.4:8080") is False + + +class TestHttpsTimeOutValidator: + """httpsTimeOutValidator 测试""" + + @patch("helper.validator.head") + def test_returns_true_on_200(self, mock_head): + """status_code=200 -> True""" + mock_head.return_value = MagicMock(status_code=200) + assert httpsTimeOutValidator("1.2.3.4:8080") is True + # 验证 verify=False 被传递 + call_kwargs = mock_head.call_args + assert call_kwargs[1]["verify"] is False + + @patch("helper.validator.head") + def test_returns_false_on_non_200(self, mock_head): + """status_code=502 -> False""" + mock_head.return_value = MagicMock(status_code=502) + assert httpsTimeOutValidator("1.2.3.4:8080") is False + + @patch("helper.validator.head") + def test_returns_false_on_exception(self, mock_head): + """head() raise Timeout -> False""" + mock_head.side_effect = TimeoutError("connection timed out") + assert httpsTimeOutValidator("1.2.3.4:8080") is False + + +class TestCustomValidatorExample: + """customValidatorExample 测试""" + + def test_always_returns_true(self): + """customValidatorExample 始终返回 True""" + assert customValidatorExample("1.2.3.4:8080") is True \ No newline at end of file From 7bfc919183e814c43c6be3bd9ce5d4af5e95ec0a Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:10:25 +0800 Subject: [PATCH 346/354] test: extend proxy API tests with refresh, JsonResponse, runFlask (B4) Add tests for /refresh/ endpoint, JsonResponse.force_type with dict/list, runFlask on Windows. 3 new tests, all passing. --- tests/api/test_proxy_api.py | 39 ++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/api/test_proxy_api.py b/tests/api/test_proxy_api.py index de6728149..012a11e88 100644 --- a/tests/api/test_proxy_api.py +++ b/tests/api/test_proxy_api.py @@ -13,7 +13,9 @@ __author__ = 'JHao' import pytest +from unittest.mock import patch, MagicMock from helper.proxy import Proxy +from api.proxyApi import JsonResponse @pytest.fixture @@ -151,4 +153,39 @@ def test_count_empty(self, client, mocks): data = resp.get_json() assert data["count"] == 0 assert data["http_type"] == {} - assert data["source"] == {} \ No newline at end of file + assert data["source"] == {} + + +class TestRefresh: + + def test_refresh_returns_success(self, client): + resp = client.get("/refresh/") + assert resp.status_code == 200 + assert b"success" in resp.data + + +class TestJsonResponse: + + def test_force_type_with_dict(self, app): + """dict -> JSON Response""" + with app.app_context(): + resp = JsonResponse.force_type({"key": "val"}) + assert resp.content_type == "application/json" + + def test_force_type_with_list(self, app): + """list -> JSON Response""" + with app.app_context(): + resp = JsonResponse.force_type([1, 2, 3]) + assert resp.content_type == "application/json" + + +class TestRunFlask: + + @patch("api.proxyApi.platform") + @patch("api.proxyApi.app") + def test_runflask_windows_path(self, mock_app, mock_platform): + """Windows 下调用 app.run()""" + mock_platform.system.return_value = "Windows" + from api.proxyApi import runFlask + runFlask() + mock_app.run.assert_called_once() \ No newline at end of file From 1a38deb7eb62d384a4213209756f65599b43b5e4 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:13:54 +0800 Subject: [PATCH 347/354] test: extend DbClient tests with init and delegation (B5) Add tests for DbClient init (redis/ssdb) and all 11 delegation methods. 13 new tests, all passing. --- tests/unit/test_db_client.py | 102 ++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_db_client.py b/tests/unit/test_db_client.py index 31becb5cf..49ae429c4 100644 --- a/tests/unit/test_db_client.py +++ b/tests/unit/test_db_client.py @@ -13,6 +13,8 @@ __author__ = 'JHao' import pytest +from unittest.mock import MagicMock, patch + from db.dbClient import DbClient @@ -59,4 +61,102 @@ def test_parse_returns_cls(self, uri, expected_type): """parseDbConn 返回 cls 以支持链式调用""" result = DbClient.parseDbConn(uri) assert result is DbClient - assert DbClient.db_type == expected_type \ No newline at end of file + assert DbClient.db_type == expected_type + + +class TestDbClientInit: + + @patch("db.dbClient.DbClient.parseDbConn") + def test_redis_init(self, mock_parse): + """Redis URI -> RedisClient 实例""" + with patch.object(DbClient, "_DbClient__initDbClient") as mock_init: + db = DbClient.__new__(DbClient) + DbClient.__init__(db, "redis://:pwd@127.0.0.1:6379/0") + mock_parse.assert_called_once_with("redis://:pwd@127.0.0.1:6379/0") + mock_init.assert_called_once() + + @patch("db.dbClient.DbClient.parseDbConn") + def test_ssdb_init(self, mock_parse): + """SSDB URI -> SsdbClient 实例""" + with patch.object(DbClient, "_DbClient__initDbClient") as mock_init: + db = DbClient.__new__(DbClient) + DbClient.__init__(db, "ssdb://:pwd@127.0.0.1:8888") + mock_parse.assert_called_once_with("ssdb://:pwd@127.0.0.1:8888") + mock_init.assert_called_once() + + +class TestDbClientDelegation: + """所有委托方法测试""" + + def _make_client(self): + """构造注入 mock client 的 DbClient""" + db = DbClient.__new__(DbClient) + db.client = MagicMock() + return db + + def test_get(self): + db = self._make_client() + db.client.get.return_value = '{"proxy": "1.2.3.4:8080"}' + result = db.get(True) + db.client.get.assert_called_once_with(True) + assert result == '{"proxy": "1.2.3.4:8080"}' + + def test_put(self): + db = self._make_client() + db.put("1.2.3.4:8080") + db.client.put.assert_called_once_with("1.2.3.4:8080") + + def test_update(self): + db = self._make_client() + db.update("key", "value") + db.client.update.assert_called_once_with("key", "value") + + def test_delete(self): + db = self._make_client() + db.delete("1.2.3.4:8080") + db.client.delete.assert_called_once_with("1.2.3.4:8080") + + def test_exists(self): + db = self._make_client() + db.client.exists.return_value = True + result = db.exists("1.2.3.4:8080") + db.client.exists.assert_called_once_with("1.2.3.4:8080") + assert result is True + + def test_pop(self): + db = self._make_client() + db.client.pop.return_value = '{"proxy": "1.2.3.4:8080"}' + result = db.pop(True) + db.client.pop.assert_called_once_with(True) + assert result == '{"proxy": "1.2.3.4:8080"}' + + def test_getAll(self): + db = self._make_client() + db.client.getAll.return_value = [] + result = db.getAll(False) + db.client.getAll.assert_called_once_with(False) + assert result == [] + + def test_clear(self): + db = self._make_client() + db.clear() + db.client.clear.assert_called_once() + + def test_changeTable(self): + db = self._make_client() + db.changeTable("use_proxy") + db.client.changeTable.assert_called_once_with("use_proxy") + + def test_getCount(self): + db = self._make_client() + db.client.getCount.return_value = 42 + result = db.getCount() + db.client.getCount.assert_called_once() + assert result == 42 + + def test_test(self): + db = self._make_client() + db.client.test.return_value = True + result = db.test() + db.client.test.assert_called_once() + assert result is True \ No newline at end of file From e8109825940538bcd1e1b508b80a01303a7a9354 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:23:11 +0800 Subject: [PATCH 348/354] test: add WebRequest unit tests (C1) Tests for get/post (success, header merge, retry exhaustion), tree, text, json, user_agent/header. Note: get() retry exhaustion doesn't assign self.response (inconsistent with post()). 13 new tests, all passing. --- tests/unit/test_web_request.py | 182 +++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/unit/test_web_request.py diff --git a/tests/unit/test_web_request.py b/tests/unit/test_web_request.py new file mode 100644 index 000000000..a62aad79d --- /dev/null +++ b/tests/unit/test_web_request.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_web_request.py + Description : WebRequest 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock +from requests.models import Response + +from util.webRequest import WebRequest + + +def _mock_response(status_code=200, text=None, content=None, json_data=None): + """构造 mock Response""" + resp = Response() + resp.status_code = status_code + if json_data is not None: + import json + resp._content = json.dumps(json_data).encode("utf-8") + resp.json = lambda: json_data + elif content is not None: + resp._content = content + elif text is not None: + resp._content = text.encode("utf-8") + else: + resp._content = b"ok" + return resp + + +class TestWebRequestGet: + """get() 测试""" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_success_path(self, mock_get, mock_sleep): + """正常返回 -> self.response 被设置""" + mock_get.return_value = _mock_response(200, "hello") + wr = WebRequest() + result = wr.get("http://example.com", retry_time=1, retry_interval=0, timeout=1) + + assert result is wr + assert wr.response.status_code == 200 + assert wr.text == "hello" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_custom_header_merge(self, mock_get, mock_sleep): + """自定义 header 合并到默认 header""" + mock_get.return_value = _mock_response(200) + wr = WebRequest() + wr.get("http://example.com", header={"X-Custom": "v"}, retry_time=1, retry_interval=0, timeout=1) + + call_kwargs = mock_get.call_args[1] + assert call_kwargs["headers"]["X-Custom"] == "v" + assert "User-Agent" in call_kwargs["headers"] + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_retry_exhaustion(self, mock_get, mock_sleep): + """全部失败 -> 返回 fallback（注意 get() 的 bug: 未赋值 self.response）""" + mock_get.side_effect = TimeoutError("timeout") + wr = WebRequest() + result = wr.get("http://example.com", retry_time=2, retry_interval=0, timeout=1) + + assert result is wr + # 注意：get() 在 retry 耗尽时创建了 resp 但未赋值给 self.response + # 所以 self.response 仍为初始的空 Response + assert mock_get.call_count == 2 + + +class TestWebRequestPost: + """post() 测试""" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_success_path(self, mock_post, mock_sleep): + """正常返回 -> self.response 被设置""" + mock_post.return_value = _mock_response(200, "posted") + wr = WebRequest() + result = wr.post("http://example.com", retry_time=1, retry_interval=0, timeout=1) + + assert result is wr + assert wr.response.status_code == 200 + assert wr.text == "posted" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_custom_header_merge(self, mock_post, mock_sleep): + """自定义 header 合并到默认 header""" + mock_post.return_value = _mock_response(200) + wr = WebRequest() + wr.post("http://example.com", header={"X-Custom": "v"}, retry_time=1, retry_interval=0, timeout=1) + + call_kwargs = mock_post.call_args[1] + assert call_kwargs["headers"]["X-Custom"] == "v" + assert "User-Agent" in call_kwargs["headers"] + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_retry_exhaustion(self, mock_post, mock_sleep): + """全部失败 -> self.response 被正确赋值（与 get() 不同）""" + mock_post.side_effect = TimeoutError("timeout") + wr = WebRequest() + result = wr.post("http://example.com", retry_time=2, retry_interval=0, timeout=1) + + assert result is wr + # post() 正确赋值 self.response = resp + assert wr.response.status_code == 200 + assert mock_post.call_count == 2 + + +class TestWebRequestTree: + """tree 属性测试""" + + def test_empty_content_returns_none(self): + """空 content -> None""" + wr = WebRequest() + wr.response = _mock_response(200, content=b"") + assert wr.tree is None + + def test_valid_html_returns_element(self): + """有效 HTML -> lxml element""" + wr = WebRequest() + html = b"

hello

" + wr.response = _mock_response(200, content=html) + tree = wr.tree + assert tree is not None + assert tree.xpath("//p/text()") == ["hello"] + + +class TestWebRequestText: + """text 属性测试""" + + def test_returns_response_text(self): + """返回 response.text""" + wr = WebRequest() + wr.response = _mock_response(200, text="hello world") + assert wr.text == "hello world" + + +class TestWebRequestJson: + """json 属性测试""" + + def test_valid_json_returns_dict(self): + """有效 JSON -> dict""" + wr = WebRequest() + wr.response = _mock_response(200, json_data={"key": "val"}) + assert wr.json == {"key": "val"} + + def test_invalid_json_returns_empty_dict(self): + """无效 JSON -> {}""" + wr = WebRequest() + resp = _mock_response(200, content=b"not json") + resp.json = lambda: (_ for _ in ()).throw(ValueError("Invalid JSON")) + wr.response = resp + assert wr.json == {} + + +class TestWebRequestProperties: + """header/user_agent 属性测试""" + + def test_user_agent_returns_string(self): + wr = WebRequest() + ua = wr.user_agent + assert isinstance(ua, str) + assert len(ua) > 0 + + def test_header_contains_user_agent(self): + wr = WebRequest() + h = wr.header + assert "User-Agent" in h + assert "Accept" in h + assert "Connection" in h From 1251b1c0036fc5fefc597c47ed67351303c377b3 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:37:30 +0800 Subject: [PATCH 349/354] test: add CLI unit tests (A1) Tests for --version, schedule, server, fetcher commands using click.testing.CliRunner. 4 new tests, all passing. --- tests/unit/test_cli.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/test_cli.py diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 000000000..8c8fe662a --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_cli.py + Description : proxyPool CLI 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock +from click.testing import CliRunner + +from proxyPool import cli +from setting import VERSION + + +@pytest.fixture +def runner(): + return CliRunner() + + +class TestCli: + + def test_version_flag(self, runner): + """--version 显示版本号""" + result = runner.invoke(cli, ["--version"]) + assert result.exit_code == 0 + assert VERSION in result.output + + @patch("proxyPool.startScheduler") + def test_schedule_command(self, mock_scheduler, runner): + """schedule 命令调用 startScheduler""" + result = runner.invoke(cli, ["schedule"]) + assert result.exit_code == 0 + mock_scheduler.assert_called_once() + + @patch("proxyPool.startServer") + def test_server_command(self, mock_server, runner): + """server 命令调用 startServer""" + result = runner.invoke(cli, ["server"]) + assert result.exit_code == 0 + mock_server.assert_called_once() + + @patch("handler.configHandler.ConfigHandler") + @patch("helper.fetch._discover_fetchers") + def test_fetcher_command(self, mock_discover, mock_conf_cls, runner): + """fetcher 命令输出启用的代理源列表""" + mock_cls1 = MagicMock() + mock_cls1.name = "freeProxy01" + mock_cls2 = MagicMock() + mock_cls2.name = "freeProxy02" + mock_discover.return_value = [mock_cls1, mock_cls2] + mock_conf = MagicMock() + mock_conf.fetcherExclude = [] + mock_conf_cls.return_value = mock_conf + + result = runner.invoke(cli, ["fetcher"]) + assert result.exit_code == 0 + assert "freeProxy01" in result.output + assert "freeProxy02" in result.output \ No newline at end of file From cc839de8e17f5e6662a2542bab8b788747dbab03 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:41:27 +0800 Subject: [PATCH 350/354] test: add check.py unit tests (A2) Tests for DoValidator.validator (http/https pass/fail, fail_count, region), regionGetter (success/exception), _ThreadChecker.__ifRaw/__ifUse logic. 14 new tests, all passing. --- tests/unit/test_check.py | 276 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 tests/unit/test_check.py diff --git a/tests/unit/test_check.py b/tests/unit/test_check.py new file mode 100644 index 000000000..eb1daf8f1 --- /dev/null +++ b/tests/unit/test_check.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_check.py + Description : helper/check.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock, PropertyMock +from datetime import datetime + +from helper.proxy import Proxy +from helper.check import DoValidator, _ThreadChecker + + +class TestDoValidator: + """DoValidator.validator 测试""" + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_pass_https_pass(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + HTTPS 通过 -> https=True, fail_count 不变""" + mock_pv = MagicMock() + mock_pv.http_validator = [MagicMock(return_value=True)] + mock_pv.https_validator = [MagicMock(return_value=True)] + mock_pv_cls.http_validator = mock_pv.http_validator + mock_pv_cls.https_validator = mock_pv.https_validator + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + mock_conf_cls.return_value = mock_conf + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 0 + + # Patch DoValidator.conf at class level + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.https is True + assert result.last_status is True + assert result.check_count == 1 + assert result.fail_count == 0 + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_pass_https_fail(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + HTTPS 失败 -> https=False""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=False)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.https is False + assert result.last_status is True + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_fail(self, mock_pv_cls, mock_conf_cls): + """HTTP 失败 -> fail_count += 1, last_status=False""" + mock_pv_cls.http_validator = [MagicMock(return_value=False)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 0 + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.last_status is False + assert result.fail_count == 1 + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_fail_count_decrement(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + fail_count > 0 -> fail_count -= 1""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 3 + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.fail_count == 2 + + @patch("helper.check.DoValidator.regionGetter", return_value="US") + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_raw_sets_region(self, mock_pv_cls, mock_conf_cls, mock_region): + """work_type='raw' + proxyRegion=True -> regionGetter 被调用""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = True + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "raw") + + assert result.region == "US" + mock_region.assert_called_once_with(proxy) + + @patch("helper.check.DoValidator.regionGetter") + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_use_skips_region(self, mock_pv_cls, mock_conf_cls, mock_region): + """work_type='use' -> 不调用 regionGetter""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = True + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + DoValidator.validator(proxy, "use") + + mock_region.assert_not_called() + + +class TestRegionGetter: + """DoValidator.regionGetter 测试""" + + @patch("helper.check.WebRequest") + def test_success_returns_country_code(self, mock_wr_cls): + """正常返回 -> country_code""" + mock_wr = MagicMock() + mock_wr.get.return_value.json = {"country_code": "CN"} + mock_wr_cls.return_value = mock_wr + + proxy = Proxy("1.2.3.4:8080") + result = DoValidator.regionGetter(proxy) + assert result == "CN" + + @patch("helper.check.WebRequest") + def test_exception_returns_error(self, mock_wr_cls): + """异常 -> 'error'""" + mock_wr = MagicMock() + mock_wr.get.side_effect = Exception("timeout") + mock_wr_cls.return_value = mock_wr + + proxy = Proxy("1.2.3.4:8080") + result = DoValidator.regionGetter(proxy) + assert result == "error" + + +def _make_checker(work_type, proxy_handler, conf=None): + """构造手动注入依赖的 _ThreadChecker（绕过 Thread.__init__）""" + checker = _ThreadChecker.__new__(_ThreadChecker) + # 手动初始化 Thread 所需的状态 + checker._initialized = True + checker._name = "test_thread" + checker._target = None + checker._args = () + checker._kwargs = {} + checker._daemonic = False + checker._ident = None + checker._tstate_lock = None + checker._started = MagicMock() + checker._is_stopped = False + checker._block = MagicMock() + checker._waiters = [] + checker._stderr = None + # 注入依赖 + checker.proxy_handler = proxy_handler + checker.log = MagicMock() + checker.work_type = work_type + checker.conf = conf or MagicMock() + return checker + + +class TestThreadCheckerIfRaw: + """_ThreadChecker.__ifRaw 测试""" + + def test_ifraw_new_proxy_gets_put(self): + """last_status=True, exists=False -> put""" + mock_ph = MagicMock() + mock_ph.exists.return_value = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_called_once_with(proxy) + + def test_ifraw_existing_proxy_skipped(self): + """last_status=True, exists=True -> 不 put""" + mock_ph = MagicMock() + mock_ph.exists.return_value = True + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_not_called() + + def test_ifraw_failed_proxy_not_put(self): + """last_status=False -> 不 put""" + mock_ph = MagicMock() + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_not_called() + + +class TestThreadCheckerIfUse: + """_ThreadChecker.__ifUse 测试""" + + def test_ifuse_pass_gets_put(self): + """last_status=True -> put""" + mock_ph = MagicMock() + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("use", mock_ph) + checker._ThreadChecker__ifUse(proxy) + mock_ph.put.assert_called_once_with(proxy) + + def test_ifuse_fail_exceeds_max_deleted(self): + """fail_count > maxFailCount -> delete""" + mock_ph = MagicMock() + mock_conf = MagicMock() + mock_conf.maxFailCount = 3 + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + proxy.fail_count = 5 + + checker = _make_checker("use", mock_ph, mock_conf) + checker._ThreadChecker__ifUse(proxy) + mock_ph.delete.assert_called_once_with(proxy) + mock_ph.put.assert_not_called() + + def test_ifuse_fail_below_max_kept(self): + """fail_count <= maxFailCount -> put""" + mock_ph = MagicMock() + mock_conf = MagicMock() + mock_conf.maxFailCount = 3 + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + proxy.fail_count = 2 + + checker = _make_checker("use", mock_ph, mock_conf) + checker._ThreadChecker__ifUse(proxy) + mock_ph.put.assert_called_once_with(proxy) + mock_ph.delete.assert_not_called() From df2b7b892818a6b58acb44190a669d8e24ab55a7 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:43:15 +0800 Subject: [PATCH 351/354] test: add fetch.py unit tests (A3) Tests for _get_sources_dir, _load_module (fresh/cache/reload/error), _discover_fetchers (enabled/exclude/sorted/prune), _ThreadFetcher (collect/merge). 11 new tests, all passing. --- tests/unit/test_fetch.py | 136 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 tests/unit/test_fetch.py diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py new file mode 100644 index 000000000..be4b17c9b --- /dev/null +++ b/tests/unit/test_fetch.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_fetch.py + Description : helper/fetch.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import sys +import pytest +from unittest.mock import patch, MagicMock + +import helper.fetch as fetch_mod +from helper.fetch import _get_sources_dir, _load_module, _discover_fetchers, _ThreadFetcher +from helper.proxy import Proxy +from fetcher.baseFetcher import BaseFetcher + + +class TestGetSourcesDir: + + def test_returns_correct_path(self): + """返回 fetcher/sources/ 目录路径""" + path = _get_sources_dir() + assert path.endswith(os.path.join("fetcher", "sources")) + assert os.path.isdir(path) + + +class TestLoadModule: + + def setup_method(self): + """每个测试前清空缓存""" + fetch_mod._module_cache.clear() + + def test_fresh_load(self): + """缓存为空 -> importlib.import_module""" + # 使用一个已知存在的模块 + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + result = _load_module("fetcher.sources.kuaidaili", filepath) + assert result is not None + assert "fetcher.sources.kuaidaili" in fetch_mod._module_cache + + def test_cache_hit(self): + """mtime 不变 -> 返回缓存""" + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + first = _load_module("fetcher.sources.kuaidaili", filepath) + second = _load_module("fetcher.sources.kuaidaili", filepath) + assert first is second + + def test_cache_miss_reload(self): + """mtime 变化 -> importlib.reload""" + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + first = _load_module("fetcher.sources.kuaidaili", filepath) + # 模拟 mtime 变化 + fetch_mod._module_cache["fetcher.sources.kuaidaili"] = (0, first) + second = _load_module("fetcher.sources.kuaidaili", filepath) + assert second is not None + + @patch("helper.fetch.os.path.getmtime", return_value=0) + @patch("helper.fetch.importlib.import_module", side_effect=ImportError("not found")) + def test_import_exception_returns_none(self, mock_import, mock_mtime): + """import 失败 -> 返回 None""" + result = _load_module("fetcher.sources.nonexistent", "/fake/path.py") + assert result is None + + +class TestDiscoverFetchers: + + def setup_method(self): + fetch_mod._module_cache.clear() + + def test_filters_enabled_only(self): + """enabled=False 的 fetcher 被排除""" + # 使用真实扫描，检查结果中所有 fetcher 都是 enabled=True + fetchers = _discover_fetchers([]) + for f in fetchers: + assert f.enabled is True + + def test_filters_exclude_list(self): + """exclude_list 中的被排除""" + all_fetchers = _discover_fetchers([]) + if not all_fetchers: + pytest.skip("No fetchers available") + first_name = all_fetchers[0].__name__ + filtered = _discover_fetchers([first_name]) + filtered_names = [f.__name__ for f in filtered] + assert first_name not in filtered_names + + def test_returns_sorted_by_name(self): + """返回结果按 name 排序""" + fetchers = _discover_fetchers([]) + names = [f.name for f in fetchers] + assert names == sorted(names) + + def test_prunes_stale_cache(self): + """已删除文件的缓存被清理""" + fetch_mod._module_cache["fetcher.sources.deleted_module"] = (0, MagicMock()) + _discover_fetchers([]) + assert "fetcher.sources.deleted_module" not in fetch_mod._module_cache + + +class TestThreadFetcher: + + def test_collects_proxies(self): + """fetcher.fetch() yield 代理 -> proxy_dict 有值""" + mock_cls = MagicMock() + mock_cls.name = "test_fetcher" + mock_cls.return_value.fetch.return_value = ["1.2.3.4:8080", "5.6.7.8:443"] + + proxy_dict = {} + thread = _ThreadFetcher(mock_cls, proxy_dict) + thread.run() + + assert "1.2.3.4:8080" in proxy_dict + assert "5.6.7.8:443" in proxy_dict + assert isinstance(proxy_dict["1.2.3.4:8080"], Proxy) + + def test_merges_duplicate_sources(self): + """同一代理出现两次 -> add_source 被调用""" + mock_cls = MagicMock() + mock_cls.name = "test_fetcher" + mock_cls.return_value.fetch.return_value = ["1.2.3.4:8080", "1.2.3.4:8080"] + + proxy_dict = {} + thread = _ThreadFetcher(mock_cls, proxy_dict) + thread.run() + + assert "1.2.3.4:8080" in proxy_dict + # source 应该包含两次 "test_fetcher"（add_source 去重，但只出现一次） + assert "test_fetcher" in proxy_dict["1.2.3.4:8080"].source From c3fb01b2f2faad99f695604ba0b9bac8e7032961 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:51:47 +0800 Subject: [PATCH 352/354] test: add launcher.py unit tests (A4) Tests for startServer/startScheduler (call order), __beforeStart (exit/continue), __checkDBConfig (returns db.test() result). 5 new tests, all passing. --- tests/unit/test_launcher.py | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/unit/test_launcher.py diff --git a/tests/unit/test_launcher.py b/tests/unit/test_launcher.py new file mode 100644 index 000000000..20ceb9ea4 --- /dev/null +++ b/tests/unit/test_launcher.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_launcher.py + Description : helper/launcher.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock + +import helper.launcher as launcher_mod + + +class TestStartServer: + + def test_calls_before_start_then_flask(self): + """startServer 先调用 __beforeStart 再调用 runFlask""" + with patch.object(launcher_mod, "__beforeStart") as mock_before, \ + patch("api.proxyApi.runFlask") as mock_flask: + launcher_mod.startServer() + mock_before.assert_called_once() + + +class TestStartScheduler: + + def test_calls_before_start_then_scheduler(self): + """startScheduler 先调用 __beforeStart 再调用 runScheduler""" + with patch.object(launcher_mod, "__beforeStart") as mock_before, \ + patch("helper.scheduler.runScheduler") as mock_sched: + launcher_mod.startScheduler() + mock_before.assert_called_once() + + +class TestBeforeStart: + + def test_exits_when_db_check_fails(self): + """DB 检查失败 -> sys.exit()""" + with patch.object(launcher_mod, "__showVersion"), \ + patch.object(launcher_mod, "__showConfigure"), \ + patch.object(launcher_mod, "__checkDBConfig", return_value=True), \ + patch("helper.launcher.sys") as mock_sys: + getattr(launcher_mod, "__beforeStart")() + mock_sys.exit.assert_called_once() + + def test_continues_when_db_check_passes(self): + """DB 检查通过 -> 不调用 sys.exit""" + with patch.object(launcher_mod, "__showVersion"), \ + patch.object(launcher_mod, "__showConfigure"), \ + patch.object(launcher_mod, "__checkDBConfig", return_value=False), \ + patch("helper.launcher.sys") as mock_sys: + getattr(launcher_mod, "__beforeStart")() + mock_sys.exit.assert_not_called() + + +class TestCheckDBConfig: + + def test_returns_db_test_result(self): + """返回 db.test() 的结果""" + with patch.object(launcher_mod, "DbClient") as mock_db_cls, \ + patch.object(launcher_mod, "ConfigHandler") as mock_conf_cls: + mock_conf = MagicMock() + mock_conf.dbConn = "redis://:@127.0.0.1:6379/0" + mock_conf_cls.return_value = mock_conf + + mock_db = MagicMock() + mock_db.test.return_value = False + mock_db_cls.return_value = mock_db + + result = getattr(launcher_mod, "__checkDBConfig")() + + assert result is False + mock_db.test.assert_called_once() \ No newline at end of file From 5494e3fe807af28c273b3192fc8f59ddbc6a7d64 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 21:58:44 +0800 Subject: [PATCH 353/354] test: add scheduler.py unit tests (A5) Tests for __runProxyFetch (queue flow), __runProxyCheck (low pool triggers fetch), runScheduler (2 jobs, 5min fetch, 2min check intervals). 6 new tests, all passing. --- tests/unit/test_scheduler.py | 132 +++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 tests/unit/test_scheduler.py diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py new file mode 100644 index 000000000..ecc5b75dc --- /dev/null +++ b/tests/unit/test_scheduler.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_scheduler.py + Description : helper/scheduler.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock + +import helper.scheduler as scheduler_mod + + +def _get_attr(name): + """获取模块中双下划线开头的属性（绕过类内 name mangling）""" + return getattr(scheduler_mod, name) + + +class TestRunProxyFetch: + + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.Fetcher") + def test_fetcher_yields_go_to_queue(self, mock_fetcher_cls, mock_checker): + """Fetcher yield 的代理放入 queue，传给 Checker""" + mock_proxy = MagicMock() + mock_fetcher = MagicMock() + mock_fetcher.run.return_value = iter([mock_proxy]) + mock_fetcher_cls.return_value = mock_fetcher + + _get_attr("__runProxyFetch")() + + mock_fetcher_cls.assert_called_once() + mock_checker.assert_called_once() + call_args = mock_checker.call_args + assert call_args[0][0] == "raw" + + +class TestRunProxyCheck: + + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.ProxyHandler") + def test_triggers_fetch_when_pool_low(self, mock_ph_cls, mock_checker, mock_fetch): + """count < poolSizeMin -> 触发 __runProxyFetch""" + mock_ph = MagicMock() + mock_ph.db.getCount.return_value = {"total": 5} + mock_ph.conf.poolSizeMin = 20 + mock_ph.getAll.return_value = [] + mock_ph_cls.return_value = mock_ph + + _get_attr("__runProxyCheck")() + + mock_fetch.assert_called_once() + + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.ProxyHandler") + def test_skips_fetch_when_pool_sufficient(self, mock_ph_cls, mock_checker, mock_fetch): + """count >= poolSizeMin -> 不触发 __runProxyFetch""" + mock_ph = MagicMock() + mock_ph.db.getCount.return_value = {"total": 50} + mock_ph.conf.poolSizeMin = 20 + mock_ph.getAll.return_value = [] + mock_ph_cls.return_value = mock_ph + + _get_attr("__runProxyCheck")() + + mock_fetch.assert_not_called() + + +class TestRunScheduler: + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_adds_two_jobs(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """runScheduler 添加两个定时任务""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + assert mock_sched.add_job.call_count == 2 + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_fetch_job_interval_5min(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """采集任务间隔 5 分钟""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + calls = mock_sched.add_job.call_args_list + first_call = calls[0] + assert first_call[0][1] == "interval" + assert first_call[1]["minutes"] == 5 + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_check_job_interval_2min(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """检查任务间隔 2 分钟""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + calls = mock_sched.add_job.call_args_list + second_call = calls[1] + assert second_call[0][1] == "interval" + assert second_call[1]["minutes"] == 2 From a18e011c97cbeb408c4a88a928bcf3e2edddc6c0 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 15 Jun 2026 22:33:09 +0800 Subject: [PATCH 354/354] fix: mock apscheduler import for tox/uv compatibility (py39+) - test_scheduler.py: pre-mock apscheduler modules before import - test_fetch.py: fix test_import_exception_returns_none for py311 reload path --- tests/unit/test_fetch.py | 16 ++++++++++++---- tests/unit/test_scheduler.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index be4b17c9b..73e3b29e5 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -63,11 +63,19 @@ def test_cache_miss_reload(self): assert second is not None @patch("helper.fetch.os.path.getmtime", return_value=0) - @patch("helper.fetch.importlib.import_module", side_effect=ImportError("not found")) - def test_import_exception_returns_none(self, mock_import, mock_mtime): + @patch("helper.fetch.importlib") + def test_import_exception_returns_none(self, mock_importlib, mock_mtime): """import 失败 -> 返回 None""" - result = _load_module("fetcher.sources.nonexistent", "/fake/path.py") - assert result is None + mock_importlib.import_module.side_effect = ImportError("not found") + # 确保模块不在 sys.modules 中，避免走 reload 分支 + mock_importlib.reload.side_effect = ImportError("not found") + saved = sys.modules.pop("fetcher.sources.nonexistent", None) + try: + result = _load_module("fetcher.sources.nonexistent", "/fake/path.py") + assert result is None + finally: + if saved is not None: + sys.modules["fetcher.sources.nonexistent"] = saved class TestDiscoverFetchers: diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index ecc5b75dc..81c6e4e3d 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -12,9 +12,20 @@ """ __author__ = 'JHao' +import sys import pytest from unittest.mock import patch, MagicMock + +# apscheduler 依赖 pkg_resources，在 tox/uv 环境中可能缺失 +# 在 import 前 mock 掉，避免 collection 阶段报错 +_apscheduler_mock = MagicMock() +sys.modules.setdefault("apscheduler", _apscheduler_mock) +sys.modules.setdefault("apscheduler.schedulers", _apscheduler_mock.schedulers) +sys.modules.setdefault("apscheduler.schedulers.blocking", _apscheduler_mock.schedulers.blocking) +sys.modules.setdefault("apscheduler.executors", _apscheduler_mock.executors) +sys.modules.setdefault("apscheduler.executors.pool", _apscheduler_mock.executors.pool) + import helper.scheduler as scheduler_mod @@ -129,4 +140,4 @@ def test_check_job_interval_2min(self, mock_log, mock_conf_cls, mock_fetch, mock calls = mock_sched.add_job.call_args_list second_call = calls[1] assert second_call[0][1] == "interval" - assert second_call[1]["minutes"] == 2 + assert second_call[1]["minutes"] == 2 \ No newline at end of file