diff --git a/.github/workflows/docker-image-latest.yml b/.github/workflows/docker-image-latest.yml new file mode 100644 index 000000000..6c7e00ac6 --- /dev/null +++ b/.github/workflows/docker-image-latest.yml @@ -0,0 +1,35 @@ +name: Publish Docker image latest + +on: + push: + branches: + - 'master' + +jobs: + + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: jhao104/proxy_pool + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: jhao104/proxy_pool:latest diff --git a/.github/workflows/docker-image-tags.yml b/.github/workflows/docker-image-tags.yml new file mode 100644 index 000000000..9a59645ad --- /dev/null +++ b/.github/workflows/docker-image-tags.yml @@ -0,0 +1,36 @@ +name: Publish Docker image tags + +on: + push: + tags: + - '*' + +jobs: + + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: jhao104/proxy_pool + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..96369c998 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,22 @@ +name: Deploy Docs + +on: + push: + branches: + - master + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..99dfcff2a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,42 @@ +name: Tests + +on: + push: + branches: [master] + pull_request: + branches: [master, develop] + +jobs: + test: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-test.txt + + - name: Run tests + run: pytest --cov=. --cov-report=term-missing --cov-report=xml:coverage.xml + + - name: Upload coverage to Codecov + if: matrix.python-version == '3.11' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + fail_ci_if_error: false \ No newline at end of file diff --git a/.gitignore b/.gitignore index f09264408..cc91995ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,13 @@ .idea/ +site/ *.pyc +*.pyc.* +__pycache__/ *.log +.tox +.claude/ +docs/ideas/ +docs/specs/ +.coverage +.pytest_cache/ +htmlcov/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a775459a0..000000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -python: - - 2.7 - # - nightly -os: - - linux -install: - - pip install -r requirements.txt - -script: python test.py \ No newline at end of file diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py deleted file mode 100644 index 45db4843a..000000000 --- a/Api/ProxyApi.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： ProxyApi.py - Description : - Author : JHao - date： 2016/12/4 -------------------------------------------------- - Change Activity: - 2016/12/4: -------------------------------------------------- -""" -__author__ = 'JHao' - -import sys - -sys.path.append('../') - -from flask import Flask, jsonify, request -from Util.GetConfig import GetConfig - -from Manager.ProxyManager import ProxyManager - -app = Flask(__name__) - -api_list = { - 'get': u'get an usable proxy', - # 'refresh': u'refresh proxy pool', - 'get_all': u'get all proxy from proxy pool', - 'delete?proxy=127.0.0.1:8080': u'delete an unable proxy', - 'get_status': u'proxy statistics' -} - - -@app.route('/') -def index(): - return jsonify(api_list) - - -@app.route('/get/') -def get(): - proxy = ProxyManager().get() - return proxy if proxy else 'no proxy!' - - -@app.route('/refresh/') -def refresh(): - # TODO refresh会有守护程序定时执行，由api直接调用性能较差，暂不使用 - # ProxyManager().refresh() - pass - return 'success' - - -@app.route('/get_all/') -def getAll(): - proxies = ProxyManager().getAll() - return jsonify(proxies) - - -@app.route('/delete/', methods=['GET']) -def delete(): - proxy = request.args.get('proxy') - ProxyManager().delete(proxy) - return 'success' - - -@app.route('/get_status/') -def getStatus(): - status = ProxyManager().getNumber() - return jsonify(status) - - -def run(): - config = GetConfig() - app.run(host=config.host_ip, port=config.host_port) - - -if __name__ == '__main__': - run() diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..9691f22e4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,114 @@ +# CLAUDE.md + +本文件为 Claude Code (claude.ai/code) 在本仓库中工作时提供指导。 + +## 技术栈 +Python (3.8–3.11)、Flask (API)、Redis/SSDB (存储)、APScheduler (调度)。依赖版本固定记录在 `requirements.txt` 中。 + +## 常用命令 +- 安装依赖：`pip install -r requirements.txt` +- 运行代理爬取/验证调度器：`python proxyPool.py schedule` +- 运行 API 服务器：`python proxyPool.py server` +- 查看启用的代理源：`python proxyPool.py fetcher` +- 运行单元测试：`pytest tests/unit/` +- 运行 API 测试：`pytest tests/api/` +- 运行集成测试（需真实 Redis）：`pytest tests/integration/ -m integration` +- 运行全部测试：`pytest` +- 查看覆盖率：`pytest --cov=. --cov-report=term-missing` + +## 测试 + +### 目录结构 +``` +tests/ +├── conftest.py # 共享 fixtures（app、client、fake_redis、proxy_obj、reset_singleton） +├── unit/ # 纯逻辑，零外部依赖 +│ ├── test_proxy.py # Proxy 类：构造、序列化、setter、add_source +│ ├── test_db_client.py # DbClient.parseDbConn URI 解析 +│ ├── test_config.py # ConfigHandler 环境变量覆盖 +│ ├── test_validator.py # formatValidator 正则匹配 +│ ├── test_base_fetcher.py # BaseFetcher 基类解析方法 +│ └── test_fetcher_sources.py # 各代理源 fetcher yield 逻辑 +├── api/ # Flask 测试客户端，mock ProxyHandler +│ └── test_proxy_api.py # /get /pop /all /count /delete 全路由 +└── integration/ # 需要真实 Redis，标记 @pytest.mark.integration + ├── test_redis_client.py # RedisClient 完整 CRUD + └── test_ssdb_client.py # SsdbClient 完整 CRUD +``` + +### 测试分层 +- **unit/**：不依赖外部服务，用 `unittest.mock` 或 `fakeredis` 模拟，CI 必跑 +- **api/**：使用 Flask `app.test_client()`，mock 掉 `ProxyHandler`，不依赖数据库 +- **integration/**：需要真实 Redis，通过 `@pytest.mark.integration` 标记，按需执行 + +### 测试依赖 +`pytest`、`pytest-cov`、`fakeredis`（纯 Python Redis 模拟，无需真实服务） + +### 关键约定 +- 测试函数命名：`test_` 前缀 + 下划线命名（`test_get_with_https`） +- 每个测试前自动重置 `Singleton._inst`，避免单例泄漏 +- 集成测试与单元测试共存：单元测试用 fakeredis 跑，集成测试标记后按需执行 + +## 高层架构 +免费代理池项目，爬取公开代理源、验证代理可用性、持久化存储到 Redis/SSDB，并通过 Flask RESTful API 提供代理服务。 + +### 核心组件 +- **爬取器** (`fetcher/`)：插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类（提供 `parseProxiesFromText`/`yieldUniqueProxies` 共享方法，约定 `name`/`url`/`enabled` 属性和 `fetch()` 方法）。每个代理源在 `sources/` 目录下独立文件，继承 `BaseFetcher`。调度器自动扫描目录加载 `enabled=True` 的源。`setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 +- **数据库层** (`db/`)：抽象 `dbClient` 接口，包含 Redis (`redisClient.py`) 和 SSDB (`ssdbClient.py`) 两种实现。通过 `setting.py` 中的 `DB_CONN` 配置连接（格式：`redis://:pwd@ip:port/db` 或 `ssdb://:pwd@ip:port`）。 +- **调度器** (`helper/scheduler.py`)：基于 APScheduler 的定时任务，驱动爬取器运行并触发验证。时区通过 `setting.py` 中的 `TIMEZONE` 配置。 +- **验证器** (`helper/validator.py`)：使用 `HTTP_URL` (http://httpbin.org) 和 `HTTPS_URL` (https://www.qq.com) 测试代理，超时时间由 `VERIFY_TIMEOUT` 指定（默认 10 秒）。超过 `MAX_FAIL_COUNT` 的代理会被移除。当代理池数量低于 `POOL_SIZE_MIN`（默认 20）时触发重新爬取。 +- **API** (`api/proxyApi.py`)：Flask 接口，包含以下端点： + - `/get`：随机获取一个代理（`?type=https` 可筛选 HTTPS 代理） + - `/pop`：获取并删除一个代理 + - `/all`：列出所有代理 + - `/count`：代理数量统计 + - `/delete`：通过 `?proxy=host:port` 删除指定代理 + - 服务运行在 `HOST:PORT`（默认 `0.0.0.0:5010`），配置来自 `setting.py`。 +- **命令行入口** (`proxyPool.py`)：基于 click 的命令行工具，包含 `schedule` 和 `server` 两个子命令。 + +### 扩展代理源 +1. 在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher`，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法 yield 出 `host:port` 字符串。 +2. 调度器下一轮采集自动发现并启用，无需修改配置。可用 `python proxyPool.py fetcher` 查看启用列表。 + +## 关键配置 +所有运行时配置均在 `setting.py` 中： +- `HOST`/`PORT`：API 绑定的地址和端口 +- `DB_CONN`：数据库连接字符串 +- `PROXY_FETCHER_EXCLUDE`：爬取器黑名单（自动扫描 `enabled=True` 的源，排除黑名单中的） +- `HTTP_URL`/`HTTPS_URL`：验证目标 URL +- `VERIFY_TIMEOUT`：验证超时时间（默认 10 秒） +- `MAX_FAIL_COUNT`：代理被移除前允许的最大失败次数 +- `POOL_SIZE_MIN`：触发重新爬取的最小代理池数量阈值 +- `PROXY_REGION`：是否启用代理地区属性（默认 `True`） +- `TIMEZONE`：调度器时区（默认 `Asia/Shanghai`） + +## 代码风格与命名规范 +- **文件头**：每个 `.py` 文件必须包含以下标准头部： + ```python + # -*- coding: utf-8 -*- + """ + ------------------------------------------------- + File Name： fileName.py + Description : 文件功能描述 + Author : JHao + date： yyyy/mm/dd + ------------------------------------------------- + Change Activity: + yyyy/mm/dd: 修改内容简述 (修改时添加此行) + ------------------------------------------------- + """ + __author__ = 'JHao' + ``` +- **缩进**：4 个空格（Python 标准） +- **文件命名**：驼峰命名，如 `proxyFetcher.py`、`dbClient.py`、`redisClient.py`、`webRequest.py` +- **类命名**：帕斯卡命名，如 `ProxyFetcher`、`RedisClient`、`SsdbClient`、`ProxyValidator` +- **方法命名**：混合风格——数据库/爬取器方法使用驼峰命名（`getAll`、`getCount`、`changeTable`、`parseProxiesFromText`），属性和辅助方法使用下划线命名（`user_agent`、`fail_count`、`check_count`） +- **爬取器文件**：小写命名（如 `zdaye.py`、`kuaidaili.py`），类名 PascalCase（如 `ZdayeFetcher`、`KuaidailiFetcher`） +- **常量**（在 `setting.py` 中）：大写下划线命名（`DB_CONN`、`PROXY_FETCHER_EXCLUDE`、`HTTP_URL`、`MAX_FAIL_COUNT`） +- **变量**：下划线命名（`proxy_obj`、`proxy_str`、`https`） +- **注释/文档字符串**：源文件头部和行内注释通常使用中文（普通话） +- **单例模式**：使用自定义 `Singleton` 元类（`util/singleton.py`）结合 `six.withMetaclass` 实现 + +## 注意事项 +- 运行测试前需先安装测试依赖：`pip install pytest pytest-cov fakeredis` +- 单元测试和 API 测试不依赖外部服务，可直接运行；集成测试需启动 Redis diff --git a/Config.ini b/Config.ini deleted file mode 100644 index 14e785427..000000000 --- a/Config.ini +++ /dev/null @@ -1,27 +0,0 @@ -[DB] -;Configure the database information -;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB -type = SSDB -host = 127.0.0.1 -port = 6379 -;port = 8888 -name = proxy - -[ProxyGetter] -;register the proxy getter function -freeProxyFirst = 1 -freeProxySecond = 1 -;freeProxyThird = 1 -freeProxyFourth = 1 -freeProxyFifth = 1 -freeProxySixth = 1 -freeProxySeventh = 1 -freeProxyEight = 1 -;foreign website, outside the wall -;freeProxyWallFirst = 1 -;freeProxyWallSecond = 1 - -[HOST] -; API接口配置 http://127.0.0.1:5010 -ip = 0.0.0.0 -port = 5010 diff --git a/DB/DbClient.py b/DB/DbClient.py deleted file mode 100644 index 68c5db7a7..000000000 --- a/DB/DbClient.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： DbClient.py - Description : DB工厂类 - Author : JHao - date： 2016/12/2 -------------------------------------------------- - Change Activity: - 2016/12/2: -------------------------------------------------- -""" -__author__ = 'JHao' - -import os -import sys - -from Util.GetConfig import GetConfig -from Util.utilClass import Singleton - -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - - -class DbClient(object): - """ - DbClient DB工厂类 提供get/put/pop/delete/getAll/changeTable方法 - - 目前存放代理的table/collection/hash有两种： - raw_proxy： 存放原始的代理； - useful_proxy_queue： 存放检验后的代理； - - 抽象方法定义： - get(proxy): 返回proxy的信息； - put(proxy): 存入一个代理； - pop(): 弹出一个代理 - exists(proxy)： 判断代理是否存在 - getNumber(raw_proxy): 返回代理总数（一个计数器）； - update(proxy, num): 修改代理属性计数器的值; - delete(proxy): 删除指定代理； - getAll(): 返回所有代理； - changeTable(name): 切换 table or collection or hash; - - - 所有方法需要相应类去具体实现： - SSDB：SsdbClient.py - REDIS:RedisClient.py - - """ - - __metaclass__ = Singleton - - def __init__(self): - """ - init - :return: - """ - self.config = GetConfig() - self.__initDbClient() - - def __initDbClient(self): - """ - init DB Client - :return: - """ - __type = None - if "SSDB" == self.config.db_type: - __type = "SsdbClient" - elif "REDIS" == self.config.db_type: - __type = "RedisClient" - elif "MONGODB" == self.config.db_type: - __type = "MongodbClient" - else: - pass - assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type) - self.client = getattr(__import__(__type), __type)(name=self.config.db_name, - host=self.config.db_host, - port=self.config.db_port) - - def get(self, key, **kwargs): - return self.client.get(key, **kwargs) - - def put(self, key, **kwargs): - return self.client.put(key, **kwargs) - - def update(self, key, value, **kwargs): - return self.client.update(key, value, **kwargs) - - def delete(self, key, **kwargs): - return self.client.delete(key, **kwargs) - - def exists(self, key, **kwargs): - return self.client.exists(key, **kwargs) - - def pop(self, **kwargs): - return self.client.pop(**kwargs) - - def getAll(self): - return self.client.getAll() - - def changeTable(self, name): - self.client.changeTable(name) - - def getNumber(self): - return self.client.getNumber() - - -if __name__ == "__main__": - account = DbClient() - print(account.get()) - account.changeTable('use') - account.put('ac') - print(account.get()) diff --git a/DB/MongodbClient.py b/DB/MongodbClient.py deleted file mode 100644 index bd0647f51..000000000 --- a/DB/MongodbClient.py +++ /dev/null @@ -1,74 +0,0 @@ -# coding: utf-8 -""" -------------------------------------------------- - File Name： MongodbClient.py - Description : 封装mongodb操作 - Author : JHao netAir - date： 2017/3/3 -------------------------------------------------- - Change Activity: - 2017/3/3: - 2017/9/26:完成对mongodb的支持 -------------------------------------------------- -""" -__author__ = 'Maps netAir' - -from pymongo import MongoClient - - -class MongodbClient(object): - def __init__(self, name, host, port): - self.name = name - self.client = MongoClient(host, port) - self.db = self.client.proxy - - def changeTable(self, name): - self.name = name - - def get(self, proxy): - data = self.db[self.name].find_one({'proxy': proxy}) - return data['num'] if data != None else None - - def put(self, proxy, num=1): - if self.db[self.name].find_one({'proxy': proxy}): - return None - else: - self.db[self.name].insert({'proxy': proxy, 'num': num}) - - def pop(self): - data = list(self.db[self.name].aggregate([{'$sample': {'size': 1}}])) - if data: - data = data[0] - value = data['proxy'] - self.delete(value) - return {'proxy': value, 'value': data['num']} - return None - - def delete(self, value): - self.db[self.name].remove({'proxy': value}) - - def getAll(self): - return {p['proxy']: p['num'] for p in self.db[self.name].find()} - - def clean(self): - self.client.drop_database('proxy') - - def delete_all(self): - self.db[self.name].remove() - - def update(self, key, value): - self.db[self.name].update({'proxy': key}, {'$inc': {'num': value}}) - - def exists(self, key): - return True if self.db[self.name].find_one({'proxy': key}) != None else False - - def getNumber(self): - return self.db[self.name].count() - - -if __name__ == "__main__": - db = MongodbClient('first', 'localhost', 27017) - # db.put('127.0.0.1:1') - # db2 = MongodbClient('second', 'localhost', 27017) - # db2.put('127.0.0.1:2') - print(db.pop()) diff --git a/DB/RedisClient.py b/DB/RedisClient.py deleted file mode 100644 index 7d9af4386..000000000 --- a/DB/RedisClient.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python - -''' -self.name为Redis中的一个key -2017/4/17 修改pop -''' - -# ############################ -# 已弃用， -# SsdbClient.py 支持redis -############################## - -import json -import random -import redis -import sys - - -class RedisClient(object): - """ - Reids client - """ - - def __init__(self, name, host, port): - """ - init - :param name: - :param host: - :param port: - :return: - """ - self.name = name - self.__conn = redis.Redis(host=host, port=port, db=0) - - def get(self): - """ - get random result - :return: - """ - key = self.__conn.hgetall(name=self.name) - # return random.choice(key.keys()) if key else None - # key.keys()在python3中返回dict_keys，不支持index，不能直接使用random.choice - # 另：python3中，redis返回为bytes,需要解码 - rkey = random.choice(list(key.keys())) if key else None - if isinstance(rkey, bytes): - return rkey.decode('utf-8') - else: - return rkey - # return self.__conn.srandmember(name=self.name) - - def put(self, key): - """ - put an item - :param value: - :return: - """ - key = json.dumps(key) if isinstance(key, (dict, list)) else key - return self.__conn.hincrby(self.name, key, 1) - # return self.__conn.sadd(self.name, value) - - def getvalue(self, key): - value = self.__conn.hget(self.name, key) - return value if value else None - - def pop(self): - """ - pop an item - :return: - """ - key = self.get() - if key: - self.__conn.hdel(self.name, key) - return key - # return self.__conn.spop(self.name) - - def delete(self, key): - """ - delete an item - :param key: - :return: - """ - self.__conn.hdel(self.name, key) - # self.__conn.srem(self.name, value) - - def inckey(self, key, value): - self.__conn.hincrby(self.name, key, value) - - def getAll(self): - # return self.__conn.hgetall(self.name).keys() - # python3 redis返回bytes类型,需要解码 - if sys.version_info.major == 3: - return [key.decode('utf-8') for key in self.__conn.hgetall(self.name).keys()] - else: - return self.__conn.hgetall(self.name).keys() - # return self.__conn.smembers(self.name) - - def get_status(self): - return self.__conn.hlen(self.name) - # return self.__conn.scard(self.name) - - def changeTable(self, name): - self.name = name - - -if __name__ == '__main__': - redis_con = RedisClient('proxy', 'localhost', 6379) - # redis_con.put('abc') - # redis_con.put('123') - # redis_con.put('123.115.235.221:8800') - # redis_con.put(['123', '115', '235.221:8800']) - # print(redis_con.getAll()) - # redis_con.delete('abc') - # print(redis_con.getAll()) - - # print(redis_con.getAll()) - redis_con.changeTable('raw_proxy') - redis_con.pop() - - # redis_con.put('132.112.43.221:8888') - # redis_con.changeTable('proxy') - print(redis_con.get_status()) - print(redis_con.getAll()) diff --git a/DB/SsdbClient.py b/DB/SsdbClient.py deleted file mode 100644 index d9a4030f4..000000000 --- a/DB/SsdbClient.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： SsdbClient.py - Description : 封装SSDB操作 - Author : JHao - date： 2016/12/2 -------------------------------------------------- - Change Activity: - 2016/12/2: - 2017/09/22: PY3中 redis-py返回的数据是bytes型 - 2017/09/27: 修改pop()方法 返回{proxy:value}字典 -------------------------------------------------- -""" -__author__ = 'JHao' - -from Util import EnvUtil - -from redis.connection import BlockingConnectionPool -from redis import Redis -import random - - -class SsdbClient(object): - """ - SSDB client - - SSDB中代理存放的容器为hash： - 原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为为None,以后扩展可能会加入代理属性； - 验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1； - - """ - - def __init__(self, name, host, port): - """ - init - :param name: hash name - :param host: ssdb host - :param port: ssdb port - :return: - """ - self.name = name - self.__conn = Redis(connection_pool=BlockingConnectionPool(host=host, port=port)) - - def get(self, proxy): - """ - get an item - 从hash中获取对应的proxy, 使用前需要调用changeTable() - :param proxy: - :return: - """ - data = self.__conn.hget(name=self.name, key=proxy) - if data: - return data.decode('utf-8') if EnvUtil.PY3 else data - else: - return None - - def put(self, proxy, num=1): - """ - 将代理放入hash, 使用changeTable指定hash name - :param proxy: - :param num: - :return: - """ - data = self.__conn.hset(self.name, proxy, num) - return data - - def delete(self, key): - """ - Remove the ``key`` from hash ``name`` - :param key: - :return: - """ - self.__conn.hdel(self.name, key) - - def update(self, key, value): - self.__conn.hincrby(self.name, key, value) - - def pop(self): - """ - 弹出一个代理 - :return: dict {proxy: value} - """ - proxies = self.__conn.hkeys(self.name) - if proxies: - proxy = random.choice(proxies) - value = self.__conn.hget(self.name, proxy) - self.delete(proxy) - return {'proxy': proxy.decode('utf-8') if EnvUtil.PY3 else proxy, - 'value': value.decode('utf-8') if EnvUtil.PY3 and value else value} - return None - - def exists(self, key): - return self.__conn.hexists(self.name, key) - - def getAll(self): - item_dict = self.__conn.hgetall(self.name) - if EnvUtil.PY3: - return {key.decode('utf8'): value.decode('utf8') for key, value in item_dict.items()} - else: - return item_dict - - def getNumber(self): - """ - Return the number of elements in hash ``name`` - :return: - """ - return self.__conn.hlen(self.name) - - def changeTable(self, name): - self.name = name diff --git a/Dockerfile b/Dockerfile index 7c815a4e7..3e1ddade2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,23 @@ -FROM python:3.6 -WORKDIR /usr/src/app +FROM python:3.10-alpine + +LABEL maintainer="jhao104 " + +WORKDIR /app + +COPY ./requirements.txt . + +# timezone and init process +RUN apk add -U tzdata tini && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + apk del tzdata + +# runtime environment +RUN apk add musl-dev gcc libxml2-dev libxslt-dev && \ + pip install --no-cache-dir -r requirements.txt && \ + apk del gcc musl-dev + COPY . . -ENV DEBIAN_FRONTEND noninteractive -ENV TZ Asia/Shanghai -RUN pip install --no-cache-dir -r requirements.txt && \ - apt-get update && \ - apt-get install -y --force-yes git make gcc g++ autoconf && apt-get clean && \ - git clone --depth 1 https://github.com/ideawu/ssdb.git ssdb && \ - cd ssdb && make && make install && cp ssdb-server /usr/bin && \ - apt-get remove -y --force-yes git make gcc g++ autoconf && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - cp ssdb.conf /etc && cd .. && yes | rm -r ssdb && \ - mkdir -p /var/lib/ssdb && \ - sed \ - -e 's@home.*@home /var/lib@' \ - -e 's/loglevel.*/loglevel info/' \ - -e 's@work_dir = .*@work_dir = /var/lib/ssdb@' \ - -e 's@pidfile = .*@pidfile = /run/ssdb.pid@' \ - -e 's@level:.*@level: info@' \ - -e 's@ip:.*@ip: 0.0.0.0@' \ - -i /etc/ssdb.conf && \ - echo "# ! /bin/sh " > /usr/src/app/run.sh && \ - echo "cd Run" >> /usr/src/app/run.sh && \ - echo "/usr/bin/ssdb-server /etc/ssdb.conf &" >> /usr/src/app/run.sh && \ - echo "python main.py" >> /usr/src/app/run.sh && \ - chmod 777 run.sh + EXPOSE 5010 -CMD [ "sh", "run.sh" ] + +ENTRYPOINT ["tini", "--", "bash", "proxy_pool.sh", "start", "--fg"] diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py deleted file mode 100644 index 6131c089a..000000000 --- a/Manager/ProxyManager.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： ProxyManager.py - Description : - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: -------------------------------------------------- -""" -__author__ = 'JHao' - -import random - -from Util import EnvUtil -from DB.DbClient import DbClient -from Util.GetConfig import GetConfig -from Util.LogHandler import LogHandler -from Util.utilFunction import verifyProxyFormat -from ProxyGetter.getFreeProxy import GetFreeProxy - - -class ProxyManager(object): - """ - ProxyManager - """ - - def __init__(self): - self.db = DbClient() - self.config = GetConfig() - self.raw_proxy_queue = 'raw_proxy' - self.log = LogHandler('proxy_manager') - self.useful_proxy_queue = 'useful_proxy' - - def refresh(self): - """ - fetch proxy into Db by ProxyGetter - :return: - """ - for proxyGetter in self.config.proxy_getter_functions: - # fetch - proxy_set = set() - try: - self.log.info("{func}: fetch proxy start".format(func=proxyGetter)) - proxy_iter = [_ for _ in getattr(GetFreeProxy, proxyGetter.strip())()] - except Exception as e: - self.log.error("{func}: fetch proxy fail".format(func=proxyGetter)) - continue - for proxy in proxy_iter: - proxy = proxy.strip() - if proxy and verifyProxyFormat(proxy): - self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) - proxy_set.add(proxy) - else: - self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy)) - - # store - for proxy in proxy_set: - self.db.changeTable(self.useful_proxy_queue) - if self.db.exists(proxy): - continue - self.db.changeTable(self.raw_proxy_queue) - self.db.put(proxy) - - def get(self): - """ - return a useful proxy - :return: - """ - self.db.changeTable(self.useful_proxy_queue) - item_dict = self.db.getAll() - if item_dict: - if EnvUtil.PY3: - return random.choice(list(item_dict.keys())) - else: - return random.choice(item_dict.keys()) - return None - # return self.db.pop() - - def delete(self, proxy): - """ - delete proxy from pool - :param proxy: - :return: - """ - self.db.changeTable(self.useful_proxy_queue) - self.db.delete(proxy) - - def getAll(self): - """ - get all proxy from pool as list - :return: - """ - self.db.changeTable(self.useful_proxy_queue) - item_dict = self.db.getAll() - if EnvUtil.PY3: - return list(item_dict.keys()) if item_dict else list() - return item_dict.keys() if item_dict else list() - - def getNumber(self): - self.db.changeTable(self.raw_proxy_queue) - total_raw_proxy = self.db.getNumber() - self.db.changeTable(self.useful_proxy_queue) - total_useful_queue = self.db.getNumber() - return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue} - - -if __name__ == '__main__': - pp = ProxyManager() - pp.refresh() diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py deleted file mode 100644 index ff9ee8197..000000000 --- a/ProxyGetter/getFreeProxy.py +++ /dev/null @@ -1,248 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： GetFreeProxy.py - Description : 抓取免费代理 - Author : JHao - date： 2016/11/25 -------------------------------------------------- - Change Activity: - 2016/11/25: -------------------------------------------------- -""" -import re -import sys -import requests - -try: - from importlib import reload # py3 实际不会实用，只是为了不显示语法错误 -except: - reload(sys) - sys.setdefaultencoding('utf-8') - -sys.path.append('../') - -from Util.utilFunction import robustCrawl, getHtmlTree -from Util.WebRequest import WebRequest - -# for debug to disable insecureWarning -requests.packages.urllib3.disable_warnings() - -""" - 66ip.cn - data5u.com - ip181.com - xicidaili.com - goubanjia.com - xdaili.cn - kuaidaili.com - cn-proxy.com - proxy-list.org - www.mimiip.com -""" - - -class GetFreeProxy(object): - """ - proxy getter - """ - - def __init__(self): - pass - - @staticmethod - def freeProxyFirst(page=10): - """ - 抓取无忧代理 http://www.data5u.com/ - :param page: 页数 - :return: - """ - url_list = ['http://www.data5u.com/', - 'http://www.data5u.com/free/', - 'http://www.data5u.com/free/gngn/index.shtml', - 'http://www.data5u.com/free/gnpt/index.shtml'] - for url in url_list: - html_tree = getHtmlTree(url) - ul_list = html_tree.xpath('//ul[@class="l2"]') - for ul in ul_list: - try: - yield ':'.join(ul.xpath('.//li/text()')[0:2]) - except Exception as e: - pass - - @staticmethod - def freeProxySecond(proxy_number=100): - """ - 抓取代理66 http://www.66ip.cn/ - :param proxy_number: 代理数量 - :return: - """ - url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format( - proxy_number) - request = WebRequest() - html = request.get(url).text - for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html): - yield proxy - - @staticmethod - def freeProxyThird(days=1): - """ - 抓取ip181 http://www.ip181.com/ - :param days: - :return: - """ - url = 'http://www.ip181.com/' - html_tree = getHtmlTree(url) - try: - tr_list = html_tree.xpath('//tr')[1:] - for tr in tr_list: - yield ':'.join(tr.xpath('./td/text()')[0:2]) - except Exception as e: - pass - - @staticmethod - def freeProxyFourth(): - """ - 抓取西刺代理 http://api.xicidaili.com/free2016.txt - :return: - """ - url_list = ['http://www.xicidaili.com/nn', # 高匿 - 'http://www.xicidaili.com/nt', # 透明 - ] - for each_url in url_list: - tree = getHtmlTree(each_url) - proxy_list = tree.xpath('.//table[@id="ip_list"]//tr') - for proxy in proxy_list: - try: - yield ':'.join(proxy.xpath('./td/text()')[0:2]) - except Exception as e: - pass - - @staticmethod - def freeProxyFifth(): - """ - 抓取guobanjia http://www.goubanjia.com/ - :return: - """ - url = "http://www.goubanjia.com/" - tree = getHtmlTree(url) - proxy_list = tree.xpath('//td[@class="ip"]') - # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号 - # 需要过滤掉

的内容 - xpath_str = """.//*[not(contains(@style, 'display: none')) - and not(contains(@style, 'display:none')) - and not(contains(@class, 'port')) - ]/text() - """ - for each_proxy in proxy_list: - try: - # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port - ip_addr = ''.join(each_proxy.xpath(xpath_str)) - port = each_proxy.xpath(".//span[contains(@class, 'port')]/text()")[0] - yield '{}:{}'.format(ip_addr, port) - except Exception as e: - pass - - @staticmethod - def freeProxySixth(): - """ - 抓取讯代理免费proxy http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10 - :return: - """ - url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10' - request = WebRequest() - try: - res = request.get(url).json() - for row in res['RESULT']['rows']: - yield '{}:{}'.format(row['ip'], row['port']) - except Exception as e: - pass - - @staticmethod - def freeProxySeventh(): - """ - 快代理免费https://www.kuaidaili.com/free/inha/1/ - """ - url = 'https://www.kuaidaili.com/free/inha/{page}/' - for page in range(1, 10): - page_url = url.format(page=page) - tree = getHtmlTree(page_url) - proxy_list = tree.xpath('.//table//tr') - for tr in proxy_list[1:]: - yield ':'.join(tr.xpath('./td/text()')[0:2]) - - @staticmethod - def freeProxyEight(): - """ - 秘密代理IP网站http://www.mimiip.com - """ - url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)] # 国内高匿 - url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 10)] # 国内普匿 - url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 10)] # 国内透明 - url_list = url_gngao + url_gnpu + url_gntou - - request = WebRequest() - for url in url_list: - r = request.get(url) - proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W].*(\d+)', r.text) - for proxy in proxies: - yield ':'.join(proxy) - - @staticmethod - def freeProxyWallFirst(): - """ - 墙外网站 cn-proxy - :return: - """ - urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] - request = WebRequest() - for url in urls: - r = request.get(url) - proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\w\W](\d+)', r.text) - for proxy in proxies: - yield ':'.join(proxy) - - @staticmethod - def freeProxyWallSecond(): - urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)] - request = WebRequest() - import base64 - for url in urls: - r = request.get(url) - proxies = re.findall(r"Proxy\('(.*?)'\)", r.text) - for proxy in proxies: - yield base64.b64decode(proxy).decode() - - -if __name__ == '__main__': - gg = GetFreeProxy() - # for e in gg.freeProxyFirst(): - # print(e) - # - # for e in gg.freeProxySecond(): - # print(e) - # - # for e in gg.freeProxyThird(): - # print(e) - # - # for e in gg.freeProxyFourth(): - # print(e) - # - # for e in gg.freeProxyFifth(): - # print(e) - # - # for e in gg.freeProxySixth(): - # print(e) - # - # for e in gg.freeProxySeventh(): - # print(e) - # - # for e in gg.freeProxyEight(): - # print(e) - # - # for e in gg.freeProxyWallFirst(): - # print(e) - # - # for e in gg.freeProxyWallSecond(): - # print(e) diff --git a/README.md b/README.md index 32a0ac968..640c1e157 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -爬虫IP代理池 +ProxyPool 爬虫代理IP池 ======= -[![Build Status](https://travis-ci.org/jhao104/proxy_pool.svg?branch=master)](https://travis-ci.org/jhao104/proxy_pool) +[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![codecov](https://codecov.io/gh/jhao104/proxy_pool/graph/badge.svg?token=8WHGkrQA6E)](https://codecov.io/gh/jhao104/proxy_pool) [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/) -[![Requirements Status](https://requires.io/github/jhao104/proxy_pool/requirements.svg?branch=master)](https://requires.io/github/jhao104/proxy_pool/requirements/?branch=master) [![Packagist](https://img.shields.io/packagist/l/doctrine/orm.svg)](https://github.com/jhao104/proxy_pool/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/jhao104/proxy_pool.svg)](https://github.com/jhao104/proxy_pool/graphs/contributors) [![](https://img.shields.io/badge/language-Python-green.svg)](https://github.com/jhao104/proxy_pool) @@ -17,78 +17,116 @@ __ / / /___ / -##### [介绍文档](https://github.com/jhao104/proxy_pool/blob/master/doc/introduce.md) +### ProxyPool -* 支持版本: ![](https://img.shields.io/badge/Python-2.x-green.svg) ![](https://img.shields.io/badge/Python-3.x-blue.svg) +爬虫代理IP池项目,主要功能为定时采集网上发布的免费代理验证入库，定时验证入库的代理保证代理的可用性，提供API和CLI两种使用方式。同时你也可以扩展代理源以增加代理池IP的质量和数量。 -* 测试地址: http://123.207.35.36:5010 (单机勿压。感谢) +* 文档: [document](https://jhao104.github.io/proxy_pool/) -### 下载安装 +* 支持版本: +[![](https://img.shields.io/badge/Python-3.8-blue.svg)](https://docs.python.org/3.8/) +[![](https://img.shields.io/badge/Python-3.9-blue.svg)](https://docs.python.org/3.9/) +[![](https://img.shields.io/badge/Python-3.10-blue.svg)](https://docs.python.org/3.10/) +[![](https://img.shields.io/badge/Python-3.11-blue.svg)](https://docs.python.org/3.11/) -* 下载源码: +* 测试地址: http://demo.spiderpy.cn (勿压谢谢) -```shell -git clone git@github.com:jhao104/proxy_pool.git +* 付费代理推荐: [亮数据 Bright Data](https://get.brightdata.com/github_jh)（前身 Luminati）.全球代理与网络抓取行业头部领导者。覆盖 195+ 国家的 1.5亿+ 真人住宅IP，亲测成功率极高，轻松突破反爬封锁。需要高质量代理IP的可以注册后联系中文客服。[申请免费试用](https://get.brightdata.com/github_jh) (PS:用不明白的同学可以参考这个[使用教程](https://www.cnblogs.com/jhao/p/15611785.html))。 -或者直接到https://github.com/jhao104/proxy_pool 下载zip文件 +   想自建爬虫？接入 [Bright Data MCP Server](https://get.brightdata.com/cd3yy5)，让 Claude、Cursor、Windsurf 等 AI 助手直接实时抓取网页——自动破解验证码、绕过地区限制。[Scraper Studio](https://get.brightdata.com/cd3yy5) 支持 AI 一键生成或 JS 代码定制，全托管基础设施运行，无需自购代理、无需搭服务器，分钟级上线。所有产品底层均由同一套顶级代理网络驱动。 + +   API 产品现享7折 + 免费试用额度，注册后可联系中文客服快速上手。(用不明白的同学可参考使用教程，或注册后直接使用互动 AI 智能助手) +👉 [https://get.brightdata.com/cd3yy5](https://get.brightdata.com/cd3yy5) + + +### 运行项目 + +##### 下载代码: + +* git clone + +```bash +git clone https://github.com/jhao104/proxy_pool.git +``` + +* releases + +```bash +https://github.com/jhao104/proxy_pool/releases 下载对应zip文件 ``` -* 安装依赖: +##### 安装依赖: -```shell +```bash pip install -r requirements.txt ``` -* 配置Config.ini: +##### 更新配置: + + +```python +# setting.py 为项目配置文件 + +# 配置API服务 + +HOST = "0.0.0.0" # IP +PORT = 5000 # 监听端口 -```shell -# Config.ini 为项目配置文件 -# 配置DB -type = SSDB # 如果使用SSDB或redis数据库，均配置为SSDB -host = localhost # db host -port = 8888 # db port -name = proxy # 默认配置 -# 配置 ProxyGetter -freeProxyFirst = 1 # 这里是启动的抓取函数，可在ProxyGetter/getFreeProxy.py 扩展 -freeProxySecond = 1 -.... +# 配置数据库 -# 配置 HOST (api服务) -ip = 127.0.0.1 # 监听ip,0.0.0.0开启外网访问 -port = 5010 # 监听端口 -# 上面配置启动后，代理api地址为 http://127.0.0.1:5010 +DB_CONN = 'redis://:pwd@127.0.0.1:8888/0' + +# 配置代理源（可选） +# 默认自动扫描 fetcher/sources/ 目录下所有 enabled=True 的代理源 +# 如需禁用某些代理源，在黑名单中添加其 name 即可 +# PROXY_FETCHER_EXCLUDE = ["freevpnnode"] ``` -* 启动: +#### 启动项目: + +```bash +# 如果已经具备运行条件, 可用通过proxyPool.py启动。 +# 程序分为: schedule 调度程序 和 server Api服务 -```shell -# 如果你的依赖已经安全完成并且具备运行条件,可以直接在Run下运行main.py -# 到Run目录下: ->>>python main.py +# 启动调度程序 +python proxyPool.py schedule -# 如果运行成功你应该看到有4个main.py进程 +# 启动webApi服务 +python proxyPool.py server -# 你也可以分别运行他们, -# 依次到Api下启动ProxyApi.py,Schedule下启动ProxyRefreshSchedule.py和ProxyValidSchedule.py即可. ``` -### 使用 +### Docker Image -　　启动过几分钟后就能看到抓取到的代理IP，你可以直接到数据库中查看，推荐一个[SSDB可视化工具](https://github.com/jhao104/SSDBAdmin)。 +```bash +docker pull jhao104/proxy_pool -　　也可以通过api访问http://127.0.0.1:5010 查看。 +docker run --env DB_CONN=redis://:password@ip:port/0 -p 5010:5010 jhao104/proxy_pool:latest +``` +### docker-compose + +项目目录下运行: +``` bash +docker-compose up -d +``` + +### 使用 * Api -| api | method | Description | arg| +启动web服务后, 默认配置下会开启 http://127.0.0.1:5010 的api接口服务: + +| api | method | Description | params| | ----| ---- | ---- | ----| | / | GET | api介绍 | None | -| /get | GET | 随机获取一个代理 | None| -| /get_all | GET | 获取所有代理 |None| -| /get_status | GET | 查看代理数量 |None| -| /delete | GET | 删除代理 |proxy=host:ip| +| /get | GET | 随机获取一个代理| 可选参数: `?type=https` 过滤支持https的代理| +| /pop | GET | 获取并删除一个代理| 可选参数: `?type=https` 过滤支持https的代理| +| /all | GET | 获取所有代理 |可选参数: `?type=https` 过滤支持https的代理| +| /count | GET | 查看代理数量 |None| +| /delete | GET | 删除代理 |`?proxy=host:ip`| + * 爬虫使用 @@ -98,7 +136,7 @@ port = 5010 # 监听端口 import requests def get_proxy(): - return requests.get("http://127.0.0.1:5010/get/").content + return requests.get("http://127.0.0.1:5010/get/").json() def delete_proxy(proxy): requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy)) @@ -108,65 +146,75 @@ def delete_proxy(proxy): def getHtml(): # .... retry_count = 5 - proxy = get_proxy() + proxy = get_proxy().get("proxy") while retry_count > 0: try: - html = requests.get('https://www.example.com', proxies={"http": "http://{}".format(proxy)}) + html = requests.get('http://www.example.com', proxies={"http": "http://{}".format(proxy)}) # 使用代理访问 return html except Exception: retry_count -= 1 - # 出错5次, 删除代理池中代理 + # 删除代理池中代理 delete_proxy(proxy) return None ``` ### 扩展代理 -　　项目默认包含几个免费的代理获取方法，但是免费的毕竟质量不好，所以如果直接运行可能拿到的代理质量不理想。所以，提供了代理获取的扩展方法。 +　　项目默认包含几个免费的代理获取源，但是免费的毕竟质量有限，所以如果直接运行可能拿到的代理质量不理想。所以，提供了代理获取的扩展方法。 -　　添加一个新的代理获取方法如下: +　　添加一个新的代理源方法如下: -* 1、首先在[GetFreeProxy](https://github.com/jhao104/proxy_pool/blob/b9ccdfaada51b57cfb1bbd0c01d4258971bc8352/ProxyGetter/getFreeProxy.py#L32)类中添加你的获取代理的静态方法， -该方法需要以生成器(yield)形式返回`host:ip`格式的代理，例如: +* 1、在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher` 基类，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法以生成器(yield)形式返回`host:port`格式的代理，例如: ```python +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest -class GetFreeProxy(object): - # .... +class MyProxyFetcher(BaseFetcher): + """我的代理源""" - # 你自己的方法 - @staticmethod - def freeProxyCustom(): # 命名不和已有重复即可 + name = "myproxy" + url = "https://www.example.com/" + enabled = True - # 通过某网站或者某接口或某数据库获取代理 任意你喜欢的姿势都行 - # 假设你拿到了一个代理列表 - proxies = ["139.129.166.68:3128", "139.129.166.61:3128", ...] - for proxy in proxies: - yield proxy - # 确保每个proxy都是 host:ip正确的格式就行 + def fetch(self): + r = WebRequest().get("https://www.example.com/api/proxies") + for item in r.json: + yield item["ip"] + ":" + item["port"] ``` -* 2、添加好方法后，修改Config.ini文件中的`[ProxyGetter]`项： - -　　在`Config.ini`的`[ProxyGetter]`下添加自定义的方法的名字: +* 2、添加好后，`schedule` 进程下次抓取时会自动扫描 `fetcher/sources/` 目录并启用新代理源，无需修改配置。 -```shell +　　可用 `python proxyPool.py fetcher` 命令查看当前启用的代理源列表。 -[ProxyGetter] -;register the proxy getter function -freeProxyFirst = 0 # 如果要取消某个方法，将其删除或赋为0即可 -.... -freeProxyCustom = 1 # 确保名字和你添加方法名字一致 +　　如需临时禁用某个代理源，在 [setting.py](setting.py) 的 `PROXY_FETCHER_EXCLUDE` 黑名单中添加其 `name` 即可。 -``` +### 免费代理源 + 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): + + | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | + |--------------| ---- |------|-----|---------------------------------------------------|---------------------------------------------------------| + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 89代理 | ✔ | ★★ | ** | [地址](https://www.89ip.cn) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | + | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | + | RoundProxies | ✔ | ★ | * | [地址](https://roundproxies.com/free-proxy-list) | [`roundproxies.py`](/fetcher/sources/roundproxies.py) | -　　`ProxyRefreshSchedule`会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。 + + 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 ### 问题反馈 -　　任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈，如果没有账号可以去 我的[博客](http://www.spiderpy.cn/blog/message)中留言。 +　　任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈，同时也可以到我的[博客](http://www.spiderpy.cn/blog/message)中留言。 　　你的反馈会让此项目变得更加完美。 @@ -174,14 +222,15 @@ freeProxyCustom = 1 # 确保名字和你添加方法名字一致 　　本项目仅作为基本的通用的代理池架构，不接收特有功能(当然,不限于特别好的idea)。 -　　本项目依然不够完善，如果发现bug或有新的功能添加，请在[Issues](https://github.com/jhao104/proxy_pool/issues)中提交bug(或新功能)描述，在确认后提交你的代码。 +　　本项目依然不够完善，如果发现bug或有新的功能添加，请在[Issues](https://github.com/jhao104/proxy_pool/issues)中提交bug(或新功能)描述，我会尽力改进，使她更加完美。 　　这里感谢以下contributor的无私奉献： -　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)| [@highroom](https://github.com/highroom) +　　[@kangnwh](https://github.com/kangnwh) | [@bobobo80](https://github.com/bobobo80) | [@halleywj](https://github.com/halleywj) | [@newlyedward](https://github.com/newlyedward) | [@wang-ye](https://github.com/wang-ye) | [@gladmo](https://github.com/gladmo) | [@bernieyangmh](https://github.com/bernieyangmh) | [@PythonYXY](https://github.com/PythonYXY) | [@zuijiawoniu](https://github.com/zuijiawoniu) | [@netAir](https://github.com/netAir) | [@scil](https://github.com/scil) | [@tangrela](https://github.com/tangrela) | [@highroom](https://github.com/highroom) | [@luocaodan](https://github.com/luocaodan) | [@vc5](https://github.com/vc5) | [@1again](https://github.com/1again) | [@obaiyan](https://github.com/obaiyan) | [@zsbh](https://github.com/zsbh) | [@jiannanya](https://github.com/jiannanya) | [@Jerry12228](https://github.com/Jerry12228) | [@zeyudada](https://github.com/zeyudada) ### Release Notes - [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md) + [changelog](https://jhao104.github.io/proxy_pool/changelog/) +Featured｜HelloGitHub diff --git a/Run/main.py b/Run/main.py deleted file mode 100644 index 6b07654ee..000000000 --- a/Run/main.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： main.py - Description : 运行主函数 - Author : JHao - date： 2017/4/1 -------------------------------------------------- - Change Activity: - 2017/4/1: -------------------------------------------------- -""" -__author__ = 'JHao' - -import sys -from multiprocessing import Process - -sys.path.append('../') - -from Api.ProxyApi import run as ProxyApiRun -from Schedule.ProxyValidSchedule import run as ValidRun -from Schedule.ProxyRefreshSchedule import run as RefreshRun - - -def run(): - p_list = list() - p1 = Process(target=ProxyApiRun, name='ProxyApiRun') - p_list.append(p1) - p2 = Process(target=ValidRun, name='ValidRun') - p_list.append(p2) - p3 = Process(target=RefreshRun, name='RefreshRun') - p_list.append(p3) - - for p in p_list: - p.daemon = True - p.start() - for p in p_list: - p.join() - - -if __name__ == '__main__': - run() diff --git a/Schedule/ProxyCheck.py b/Schedule/ProxyCheck.py deleted file mode 100644 index 4300f7bf7..000000000 --- a/Schedule/ProxyCheck.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： ProxyCheck - Description : 多线程验证useful_proxy - Author : J_hao - date： 2017/9/26 -------------------------------------------------- - Change Activity: - 2017/9/26: 多线程验证useful_proxy -------------------------------------------------- -""" -__author__ = 'J_hao' - -import sys -from threading import Thread - -sys.path.append('../') - -from Util.utilFunction import validUsefulProxy -from Manager.ProxyManager import ProxyManager -from Util.LogHandler import LogHandler - -FAIL_COUNT = 1 # 校验失败次数， 超过次数删除代理 - - -class ProxyCheck(ProxyManager, Thread): - def __init__(self, queue, item_dict): - ProxyManager.__init__(self) - Thread.__init__(self) - self.log = LogHandler('proxy_check', file=False) # 多线程同时写一个日志文件会有问题 - self.queue = queue - self.item_dict = item_dict - - def run(self): - self.db.changeTable(self.useful_proxy_queue) - while self.queue.qsize(): - proxy = self.queue.get() - count = self.item_dict[proxy] - if validUsefulProxy(proxy): - # 验证通过计数器减1 - if count and int(count) > 0: - self.db.put(proxy, num=int(count) - 1) - else: - pass - self.log.info('ProxyCheck: {} validation pass'.format(proxy)) - else: - self.log.info('ProxyCheck: {} validation fail'.format(proxy)) - if count and int(count) + 1 >= FAIL_COUNT: - self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy)) - self.db.delete(proxy) - else: - self.db.put(proxy, num=int(count) + 1) - self.queue.task_done() - - -if __name__ == '__main__': - # p = ProxyCheck() - # p.run() - pass diff --git a/Schedule/ProxyRefreshSchedule.py b/Schedule/ProxyRefreshSchedule.py deleted file mode 100644 index 7dac2aa34..000000000 --- a/Schedule/ProxyRefreshSchedule.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： ProxyRefreshSchedule.py - Description : 代理定时刷新 - Author : JHao - date： 2016/12/4 -------------------------------------------------- - Change Activity: - 2016/12/4: 代理定时刷新 - 2017/03/06: 使用LogHandler添加日志 - 2017/04/26: raw_proxy_queue验证通过但useful_proxy_queue中已经存在的代理不在放入 -------------------------------------------------- -""" - -import sys -import time -import logging -from threading import Thread -from apscheduler.schedulers.blocking import BlockingScheduler - -sys.path.append('../') - -from Util.utilFunction import validUsefulProxy -from Manager.ProxyManager import ProxyManager -from Util.LogHandler import LogHandler - -__author__ = 'JHao' - -logging.basicConfig() - - -class ProxyRefreshSchedule(ProxyManager): - """ - 代理定时刷新 - """ - - def __init__(self): - ProxyManager.__init__(self) - self.log = LogHandler('refresh_schedule') - - def validProxy(self): - """ - 验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue - :return: - """ - self.db.changeTable(self.raw_proxy_queue) - raw_proxy_item = self.db.pop() - self.log.info('ProxyRefreshSchedule: %s start validProxy' % time.ctime()) - # 计算剩余代理，用来减少重复计算 - remaining_proxies = self.getAll() - while raw_proxy_item: - raw_proxy = raw_proxy_item.get('proxy') - if isinstance(raw_proxy, bytes): - # 兼容Py3 - raw_proxy = raw_proxy.decode('utf8') - - if (raw_proxy not in remaining_proxies) and validUsefulProxy(raw_proxy): - self.db.changeTable(self.useful_proxy_queue) - self.db.put(raw_proxy) - self.log.info('ProxyRefreshSchedule: %s validation pass' % raw_proxy) - else: - self.log.info('ProxyRefreshSchedule: %s validation fail' % raw_proxy) - self.db.changeTable(self.raw_proxy_queue) - raw_proxy_item = self.db.pop() - remaining_proxies = self.getAll() - self.log.info('ProxyRefreshSchedule: %s validProxy complete' % time.ctime()) - - -def refreshPool(): - pp = ProxyRefreshSchedule() - pp.validProxy() - - -def main(process_num=30): - p = ProxyRefreshSchedule() - - # 获取新代理 - p.refresh() - - # 检验新代理 - pl = [] - for num in range(process_num): - proc = Thread(target=refreshPool, args=()) - pl.append(proc) - - for num in range(process_num): - pl[num].daemon = True - pl[num].start() - - for num in range(process_num): - pl[num].join() - - -def run(): - main() - sch = BlockingScheduler() - sch.add_job(main, 'interval', minutes=10) # 每10分钟抓取一次 - sch.start() - - -if __name__ == '__main__': - run() diff --git a/Schedule/ProxyValidSchedule.py b/Schedule/ProxyValidSchedule.py deleted file mode 100644 index 9b075cf90..000000000 --- a/Schedule/ProxyValidSchedule.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： ProxyValidSchedule.py - Description : 验证useful_proxy_queue中的代理,将不可用的移出 - Author : JHao - date： 2017/3/31 -------------------------------------------------- - Change Activity: - 2017/3/31: 验证useful_proxy_queue中的代理 -------------------------------------------------- -""" -__author__ = 'JHao' - -import sys -import time - -try: - from Queue import Queue # py3 -except: - from queue import Queue # py2 - -sys.path.append('../') - -from Schedule.ProxyCheck import ProxyCheck -from Manager.ProxyManager import ProxyManager - - -class ProxyValidSchedule(ProxyManager, object): - def __init__(self): - ProxyManager.__init__(self) - self.queue = Queue() - self.proxy_item = dict() - - def __validProxy(self, threads=10): - """ - 验证useful_proxy代理 - :param threads: 线程数 - :return: - """ - thread_list = list() - for index in range(threads): - thread_list.append(ProxyCheck(self.queue, self.proxy_item)) - - for thread in thread_list: - thread.daemon = True - thread.start() - - for thread in thread_list: - thread.join() - - def main(self): - self.putQueue() - while True: - if not self.queue.empty(): - self.log.info("Start valid useful proxy") - self.__validProxy() - else: - self.log.info('Valid Complete! sleep 5 minutes.') - time.sleep(60 * 5) - self.putQueue() - - def putQueue(self): - self.db.changeTable(self.useful_proxy_queue) - self.proxy_item = self.db.getAll() - for item in self.proxy_item: - self.queue.put(item) - - -def run(): - p = ProxyValidSchedule() - p.main() - - -if __name__ == '__main__': - p = ProxyValidSchedule() - p.main() diff --git a/Schedule/__init__.py b/Schedule/__init__.py deleted file mode 100644 index e94e59d11..000000000 --- a/Schedule/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： __init__.py.py - Description : - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: -------------------------------------------------- -""" -__author__ = 'JHao' \ No newline at end of file diff --git a/Test/testGetConfig.py b/Test/testGetConfig.py deleted file mode 100644 index 7f44fa6b4..000000000 --- a/Test/testGetConfig.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： testGetConfig - Description : test all function in GetConfig.py - Author : J_hao - date： 2017/7/31 -------------------------------------------------- - Change Activity: - 2017/7/31: -------------------------------------------------- -""" -__author__ = 'J_hao' - -from Util.GetConfig import GetConfig - - -# noinspection PyPep8Naming -def testGetConfig(): - """ - test class GetConfig in Util/GetConfig - :return: - """ - gg = GetConfig() - print(gg.db_type) - print(gg.db_name) - print(gg.db_host) - print(gg.db_port) - assert isinstance(gg.proxy_getter_functions, list) - print(gg.proxy_getter_functions) - -if __name__ == '__main__': - testGetConfig() diff --git a/Test/testGetFreeProxy.py b/Test/testGetFreeProxy.py deleted file mode 100644 index df99c79f3..000000000 --- a/Test/testGetFreeProxy.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： testGetFreeProxy - Description : test model ProxyGetter/getFreeProxy - Author : J_hao - date： 2017/7/31 -------------------------------------------------- - Change Activity: - 2017/7/31:function testGetFreeProxy -------------------------------------------------- -""" -__author__ = 'J_hao' - -from ProxyGetter.getFreeProxy import GetFreeProxy -from Util.GetConfig import GetConfig - - -# noinspection PyPep8Naming -def testGetFreeProxy(): - """ - test class GetFreeProxy in ProxyGetter/GetFreeProxy - :return: - """ - gc = GetConfig() - proxy_getter_functions = gc.proxy_getter_functions - for proxyGetter in proxy_getter_functions: - proxy_count = 0 - for proxy in getattr(GetFreeProxy, proxyGetter.strip())(): - if proxy: - print('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy)) - proxy_count += 1 - assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter) - - -if __name__ == '__main__': - testGetFreeProxy() diff --git a/Test/testLogHandler.py b/Test/testLogHandler.py deleted file mode 100644 index da309b707..000000000 --- a/Test/testLogHandler.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： testLogHandler - Description : - Author : J_hao - date： 2017/8/2 -------------------------------------------------- - Change Activity: - 2017/8/2: -------------------------------------------------- -""" -__author__ = 'J_hao' - -from Util.LogHandler import LogHandler - - -# noinspection PyPep8Naming -def testLogHandler(): - """ - test function LogHandler in Util/LogHandler - :return: - """ - log = LogHandler('test') - log.info('this is a log from test') - - log.resetName(name='test1') - log.info('this is a log from test1') - - log.resetName(name='test2') - log.info('this is a log from test2') - - -if __name__ == '__main__': - testLogHandler() diff --git a/Test/testWebRequest.py b/Test/testWebRequest.py deleted file mode 100644 index 07dd54762..000000000 --- a/Test/testWebRequest.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： testWebRequest - Description : test class WebRequest - Author : J_hao - date： 2017/7/31 -------------------------------------------------- - Change Activity: - 2017/7/31: function testWebRequest -------------------------------------------------- -""" -__author__ = 'J_hao' - -from Util.WebRequest import WebRequest - - -# noinspection PyPep8Naming -def testWebRequest(): - """ - test class WebRequest in Util/WebRequest.py - :return: - """ - wr = WebRequest() - request_object = wr.get('https://www.baidu.com/') - assert request_object.status_code == 200 - - -if __name__ == '__main__': - testWebRequest() diff --git a/Util/EnvUtil.py b/Util/EnvUtil.py deleted file mode 100644 index b9df83c55..000000000 --- a/Util/EnvUtil.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： EnvUtil - Description : 环境相关 - Author : J_hao - date： 2017/9/18 -------------------------------------------------- - Change Activity: - 2017/9/18: 区分Python版本 -------------------------------------------------- -""" -__author__ = 'J_hao' - -import sys - -PY3 = sys.version_info >= (3,) \ No newline at end of file diff --git a/Util/GetConfig.py b/Util/GetConfig.py deleted file mode 100644 index 24b003f28..000000000 --- a/Util/GetConfig.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： GetConfig.py - Description : fetch config from config.ini - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: get db property func -------------------------------------------------- -""" -__author__ = 'JHao' - -import os -from Util.utilClass import ConfigParse -from Util.utilClass import LazyProperty - - -class GetConfig(object): - """ - to get config from config.ini - """ - - def __init__(self): - self.pwd = os.path.split(os.path.realpath(__file__))[0] - self.config_path = os.path.join(os.path.split(self.pwd)[0], 'Config.ini') - self.config_file = ConfigParse() - self.config_file.read(self.config_path) - - @LazyProperty - def db_type(self): - return self.config_file.get('DB', 'type') - - @LazyProperty - def db_name(self): - return self.config_file.get('DB', 'name') - - @LazyProperty - def db_host(self): - return self.config_file.get('DB', 'host') - - @LazyProperty - def db_port(self): - return int(self.config_file.get('DB', 'port')) - - @LazyProperty - def proxy_getter_functions(self): - return self.config_file.options('ProxyGetter') - - @LazyProperty - def host_ip(self): - return self.config_file.get('HOST','ip') - - @LazyProperty - def host_port(self): - return int(self.config_file.get('HOST', 'port')) - -if __name__ == '__main__': - gg = GetConfig() - print(gg.db_type) - print(gg.db_name) - print(gg.db_host) - print(gg.db_port) - print(gg.proxy_getter_functions) - print(gg.host_ip) - print(gg.host_port) diff --git a/Util/WebRequest.py b/Util/WebRequest.py deleted file mode 100644 index abbdb17be..000000000 --- a/Util/WebRequest.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： WebRequest - Description : Network Requests Class - Author : J_hao - date： 2017/7/31 -------------------------------------------------- - Change Activity: - 2017/7/31: -------------------------------------------------- -""" -__author__ = 'J_hao' - -import requests -import random -import time - - -class WebRequest(object): - def __init__(self, *args, **kwargs): - pass - - @property - def user_agent(self): - """ - return an User-Agent at random - :return: - """ - ua_list = [ - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71', - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)', - 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50', - 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', - ] - return random.choice(ua_list) - - @property - def header(self): - """ - basic header - :return: - """ - return {'User-Agent': self.user_agent, - 'Accept': '*/*', - 'Connection': 'keep-alive', - 'Accept-Language': 'zh-CN,zh;q=0.8'} - - def get(self, url, header=None, retry_time=5, timeout=30, - retry_flag=list(), retry_interval=5, *args, **kwargs): - """ - get method - :param url: target url - :param header: headers - :param retry_time: retry time when network error - :param timeout: network timeout - :param retry_flag: if retry_flag in content. do retry - :param retry_interval: retry interval(second) - :param args: - :param kwargs: - :return: - """ - headers = self.header - if header and isinstance(header, dict): - headers.update(header) - while True: - try: - html = requests.get(url, headers=headers, timeout=timeout) - if any(f in html.content for f in retry_flag): - raise Exception - return html - except Exception as e: - print(e) - retry_time -= 1 - if retry_time <= 0: - # 多次请求失败时，返回百度页面 - return requests.get("https://www.baidu.com/") - time.sleep(retry_interval) diff --git a/Util/utilClass.py b/Util/utilClass.py deleted file mode 100644 index 89112ffd8..000000000 --- a/Util/utilClass.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： utilClass.py - Description : tool class - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: Class LazyProperty - 2016/12/4: rewrite ConfigParser -------------------------------------------------- -""" -__author__ = 'JHao' - - -class LazyProperty(object): - """ - LazyProperty - explain: http://www.spiderpy.cn/blog/5/ - """ - - def __init__(self, func): - self.func = func - - def __get__(self, instance, owner): - if instance is None: - return self - else: - value = self.func(instance) - setattr(instance, self.func.__name__, value) - return value - - -try: - from configparser import ConfigParser # py3 -except: - from ConfigParser import ConfigParser # py2 - - -class ConfigParse(ConfigParser): - """ - rewrite ConfigParser, for support upper option - """ - - def __init__(self): - ConfigParser.__init__(self) - - def optionxform(self, optionstr): - return optionstr - - -class Singleton(type): - """ - Singleton Metaclass - """ - - _inst = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._inst: - cls._inst[cls] = super(Singleton, cls).__call__(*args) - return cls._inst[cls] diff --git a/Util/utilFunction.py b/Util/utilFunction.py deleted file mode 100644 index fc26a59b1..000000000 --- a/Util/utilFunction.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -# !/usr/bin/env python -""" -------------------------------------------------- - File Name： utilFunction.py - Description : tool function - Author : JHao - date： 2016/11/25 -------------------------------------------------- - Change Activity: - 2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree -------------------------------------------------- -""" -import requests -import time -from lxml import etree - -from Util.LogHandler import LogHandler -from Util.WebRequest import WebRequest - -# logger = LogHandler(__name__, stream=False) - - -# noinspection PyPep8Naming -def robustCrawl(func): - def decorate(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - pass - # logger.info(u"sorry, 抓取出错。错误原因:") - # logger.info(e) - - return decorate - - -# noinspection PyPep8Naming -def verifyProxyFormat(proxy): - """ - 检查代理格式 - :param proxy: - :return: - """ - import re - verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}" - _proxy = re.findall(verify_regex, proxy) - return True if len(_proxy) == 1 and _proxy[0] == proxy else False - - -# noinspection PyPep8Naming -def getHtmlTree(url, **kwargs): - """ - 获取html树 - :param url: - :param kwargs: - :return: - """ - - header = {'Connection': 'keep-alive', - 'Cache-Control': 'max-age=0', - 'Upgrade-Insecure-Requests': '1', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate, sdch', - 'Accept-Language': 'zh-CN,zh;q=0.8', - } - # TODO 取代理服务器用代理服务器访问 - wr = WebRequest() - - # delay 2s for per request - time.sleep(2) - - html = wr.get(url=url, header=header).content - return etree.HTML(html) - - -def tcpConnect(proxy): - """ - TCP 三次握手 - :param proxy: - :return: - """ - from socket import socket, AF_INET, SOCK_STREAM - s = socket(AF_INET, SOCK_STREAM) - ip, port = proxy.split(':') - result = s.connect_ex((ip, int(port))) - return True if result == 0 else False - - -# noinspection PyPep8Naming -def validUsefulProxy(proxy): - """ - 检验代理是否可用 - :param proxy: - :return: - """ - if isinstance(proxy, bytes): - proxy = proxy.decode('utf8') - proxies = {"http": "http://{proxy}".format(proxy=proxy)} - try: - # 超过20秒的代理就不要了 - r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False) - if r.status_code == 200: - # logger.info('%s is ok' % proxy) - return True - except Exception as e: - # logger.error(str(e)) - return False diff --git a/__init__.py b/__init__.py deleted file mode 100644 index c511f3103..000000000 --- a/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： __init__.py - Description : - Author : JHao - date： 2016/12/3 -------------------------------------------------- - Change Activity: - 2016/12/3: -------------------------------------------------- -""" -__author__ = 'JHao' \ No newline at end of file diff --git a/_config.yml b/_config.yml index ddeb671b6..c4192631f 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1 @@ -theme: jekyll-theme-time-machine \ No newline at end of file +theme: jekyll-theme-cayman \ No newline at end of file diff --git a/Api/__init__.py b/api/__init__.py similarity index 100% rename from Api/__init__.py rename to api/__init__.py diff --git a/api/proxyApi.py b/api/proxyApi.py new file mode 100644 index 000000000..bd2de57e2 --- /dev/null +++ b/api/proxyApi.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python +""" +------------------------------------------------- + File Name： ProxyApi.py + Description : WebApi + Author : JHao + date： 2016/12/4 +------------------------------------------------- + Change Activity: + 2016/12/04: WebApi + 2019/08/14: 集成Gunicorn启动方式 + 2020/06/23: 新增pop接口 + 2022/07/21: 更新count接口 +------------------------------------------------- +""" +__author__ = 'JHao' + +import platform +from werkzeug.wrappers import Response +from flask import Flask, jsonify, request + +from util.six import iteritems +from helper.proxy import Proxy +from handler.proxyHandler import ProxyHandler +from handler.configHandler import ConfigHandler + +app = Flask(__name__) +conf = ConfigHandler() +proxy_handler = ProxyHandler() + + +class JsonResponse(Response): + @classmethod + def force_type(cls, response, environ=None): + if isinstance(response, (dict, list)): + response = jsonify(response) + + return super(JsonResponse, cls).force_type(response, environ) + + +app.response_class = JsonResponse + +api_list = [ + {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"}, + {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, + {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, + {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, + {"url": "/count", "params": "", "desc": "return proxy count"} + # 'refresh': 'refresh proxy pool', +] + + +@app.route('/') +def index(): + return {'url': api_list} + + +@app.route('/get/') +def get(): + https = request.args.get("type", "").lower() == 'https' + proxy = proxy_handler.get(https) + return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} + + +@app.route('/pop/') +def pop(): + https = request.args.get("type", "").lower() == 'https' + proxy = proxy_handler.pop(https) + return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} + + +@app.route('/refresh/') +def refresh(): + # TODO refresh会有守护程序定时执行，由api直接调用性能较差，暂不使用 + return 'success' + + +@app.route('/all/') +def getAll(): + https = request.args.get("type", "").lower() == 'https' + proxies = proxy_handler.getAll(https) + return jsonify([_.to_dict for _ in proxies]) + + +@app.route('/delete/', methods=['GET']) +def delete(): + proxy = request.args.get('proxy') + status = proxy_handler.delete(Proxy(proxy)) + return {"code": 0, "src": status} + + +@app.route('/count/') +def getCount(): + proxies = proxy_handler.getAll() + http_type_dict = {} + source_dict = {} + for proxy in proxies: + http_type = 'https' if proxy.https else 'http' + http_type_dict[http_type] = http_type_dict.get(http_type, 0) + 1 + for source in proxy.source.split('/'): + source_dict[source] = source_dict.get(source, 0) + 1 + return {"http_type": http_type_dict, "source": source_dict, "count": len(proxies)} + + +def runFlask(): + if platform.system() == "Windows": + app.run(host=conf.serverHost, port=conf.serverPort) + else: + import gunicorn.app.base + + class StandaloneApplication(gunicorn.app.base.BaseApplication): + + def __init__(self, app, options=None): + self.options = options or {} + self.application = app + super(StandaloneApplication, self).__init__() + + def load_config(self): + _config = dict([(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(_config): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + _options = { + 'bind': '%s:%s' % (conf.serverHost, conf.serverPort), + 'workers': 4, + 'accesslog': '-', # log to stdout + 'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"' + } + StandaloneApplication(app, _options).run() + + +if __name__ == '__main__': + runFlask() diff --git a/DB/__init__.py b/db/__init__.py similarity index 100% rename from DB/__init__.py rename to db/__init__.py diff --git a/db/dbClient.py b/db/dbClient.py new file mode 100644 index 000000000..4d9554b18 --- /dev/null +++ b/db/dbClient.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python +""" +------------------------------------------------- + File Name： DbClient.py + Description : DB工厂类 + Author : JHao + date： 2016/12/2 +------------------------------------------------- + Change Activity: + 2016/12/02: DB工厂类 + 2020/07/03: 取消raw_proxy储存 +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import sys + +from util.six import urlparse, withMetaclass +from util.singleton import Singleton + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + + +class DbClient(withMetaclass(Singleton)): + """ + DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getCount/changeTable方法 + + + 抽象方法定义： + get(): 随机返回一个proxy; + put(proxy): 存入一个proxy; + pop(): 顺序返回并删除一个proxy; + update(proxy): 更新指定proxy信息; + delete(proxy): 删除指定proxy; + exists(proxy): 判断指定proxy是否存在; + getAll(): 返回所有代理; + clean(): 清除所有proxy信息; + getCount(): 返回proxy统计信息; + changeTable(name): 切换操作对象 + + + 所有方法需要相应类去具体实现： + ssdb: ssdbClient.py + redis: redisClient.py + mongodb: mongodbClient.py + + """ + + def __init__(self, db_conn): + """ + init + :return: + """ + self.parseDbConn(db_conn) + self.__initDbClient() + + @classmethod + def parseDbConn(cls, db_conn): + db_conf = urlparse(db_conn) + cls.db_type = db_conf.scheme.upper().strip() + cls.db_host = db_conf.hostname + cls.db_port = db_conf.port + cls.db_user = db_conf.username + cls.db_pwd = db_conf.password + cls.db_name = db_conf.path[1:] + return cls + + def __initDbClient(self): + """ + init DB Client + :return: + """ + __type = None + if "SSDB" == self.db_type: + __type = "ssdbClient" + elif "REDIS" == self.db_type: + __type = "redisClient" + else: + pass + assert __type, 'type error, Not support DB type: {}'.format(self.db_type) + self.client = getattr(__import__(__type), "%sClient" % self.db_type.title())(host=self.db_host, + port=self.db_port, + username=self.db_user, + password=self.db_pwd, + db=self.db_name) + + def get(self, https, **kwargs): + return self.client.get(https, **kwargs) + + def put(self, key, **kwargs): + return self.client.put(key, **kwargs) + + def update(self, key, value, **kwargs): + return self.client.update(key, value, **kwargs) + + def delete(self, key, **kwargs): + return self.client.delete(key, **kwargs) + + def exists(self, key, **kwargs): + return self.client.exists(key, **kwargs) + + def pop(self, https, **kwargs): + return self.client.pop(https, **kwargs) + + def getAll(self, https): + return self.client.getAll(https) + + def clear(self): + return self.client.clear() + + def changeTable(self, name): + self.client.changeTable(name) + + def getCount(self): + return self.client.getCount() + + def test(self): + return self.client.test() diff --git a/db/redisClient.py b/db/redisClient.py new file mode 100644 index 000000000..5f17e4c5a --- /dev/null +++ b/db/redisClient.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +""" +----------------------------------------------------- + File Name： redisClient.py + Description : 封装Redis相关操作 + Author : JHao + date： 2019/8/9 +------------------------------------------------------ + Change Activity: + 2019/08/09: 封装Redis相关操作 + 2020/06/23: 优化pop方法, 改用hscan命令 + 2021/05/26: 区别http/https代理 +------------------------------------------------------ +""" +__author__ = 'JHao' + +from redis.exceptions import TimeoutError, ConnectionError, ResponseError +from redis.connection import BlockingConnectionPool +from handler.logHandler import LogHandler +from random import choice +from redis import Redis +import json + + +class RedisClient(object): + """ + Redis client + + Redis中代理存放的结构为hash： + key为ip:port, value为代理属性的字典; + + """ + + def __init__(self, **kwargs): + """ + init + :param host: host + :param port: port + :param password: password + :param db: db + :return: + """ + self.name = "" + kwargs.pop("username") + self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, + timeout=5, + socket_timeout=5, + protocol=2, + **kwargs)) + + def get(self, https): + """ + 返回一个代理 + :return: + """ + if https: + items = self.__conn.hvals(self.name) + proxies = list(filter(lambda x: json.loads(x).get("https"), items)) + return choice(proxies) if proxies else None + else: + proxies = self.__conn.hkeys(self.name) + proxy = choice(proxies) if proxies else None + return self.__conn.hget(self.name, proxy) if proxy else None + + def put(self, proxy_obj): + """ + 将代理放入hash, 使用changeTable指定hash name + :param proxy_obj: Proxy obj + :return: + """ + data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) + return data + + def pop(self, https): + """ + 弹出一个代理 + :return: dict {proxy: value} + """ + proxy = self.get(https) + if proxy: + self.__conn.hdel(self.name, json.loads(proxy).get("proxy", "")) + return proxy if proxy else None + + def delete(self, proxy_str): + """ + 移除指定代理, 使用changeTable指定hash name + :param proxy_str: proxy str + :return: + """ + return self.__conn.hdel(self.name, proxy_str) + + def exists(self, proxy_str): + """ + 判断指定代理是否存在, 使用changeTable指定hash name + :param proxy_str: proxy str + :return: + """ + return self.__conn.hexists(self.name, proxy_str) + + def update(self, proxy_obj): + """ + 更新 proxy 属性 + :param proxy_obj: + :return: + """ + return self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) + + def getAll(self, https): + """ + 字典形式返回所有代理, 使用changeTable指定hash name + :return: + """ + items = self.__conn.hvals(self.name) + if https: + return list(filter(lambda x: json.loads(x).get("https"), items)) + else: + return items + + def clear(self): + """ + 清空所有代理, 使用changeTable指定hash name + :return: + """ + return self.__conn.delete(self.name) + + def getCount(self): + """ + 返回代理数量 + :return: + """ + proxies = self.getAll(https=False) + return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))} + + def changeTable(self, name): + """ + 切换操作对象 + :param name: + :return: + """ + self.name = name + + def test(self): + log = LogHandler('redis_client') + try: + self.getCount() + except TimeoutError as e: + log.error('redis connection time out: %s' % str(e), exc_info=True) + return e + except ConnectionError as e: + log.error('redis connection error: %s' % str(e), exc_info=True) + return e + except ResponseError as e: + log.error('redis connection error: %s' % str(e), exc_info=True) + return e + + diff --git a/db/ssdbClient.py b/db/ssdbClient.py new file mode 100644 index 000000000..559539905 --- /dev/null +++ b/db/ssdbClient.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python +""" +------------------------------------------------- + File Name： ssdbClient.py + Description : 封装SSDB操作 + Author : JHao + date： 2016/12/2 +------------------------------------------------- + Change Activity: + 2016/12/2: + 2017/09/22: PY3中 redis-py返回的数据是bytes型 + 2017/09/27: 修改pop()方法 返回{proxy:value}字典 + 2020/07/03: 2.1.0 优化代码结构 + 2021/05/26: 区分http和https代理 +------------------------------------------------- +""" +__author__ = 'JHao' +from redis.exceptions import TimeoutError, ConnectionError, ResponseError +from redis.connection import BlockingConnectionPool +from handler.logHandler import LogHandler +from random import choice +from redis import Redis +import json + + +class SsdbClient(object): + """ + SSDB client + + SSDB中代理存放的结构为hash： + key为代理的ip:por, value为代理属性的字典; + """ + + def __init__(self, **kwargs): + """ + init + :param host: host + :param port: port + :param password: password + :return: + """ + self.name = "" + kwargs.pop("username") + self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, + timeout=5, + socket_timeout=5, + protocol=2, + **kwargs)) + + def get(self, https): + """ + 从hash中随机返回一个代理 + :return: + """ + if https: + items_dict = self.__conn.hgetall(self.name) + proxies = list(filter(lambda x: json.loads(x).get("https"), items_dict.values())) + return choice(proxies) if proxies else None + else: + proxies = self.__conn.hkeys(self.name) + proxy = choice(proxies) if proxies else None + return self.__conn.hget(self.name, proxy) if proxy else None + + def put(self, proxy_obj): + """ + 将代理放入hash + :param proxy_obj: Proxy obj + :return: + """ + result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) + return result + + def pop(self, https): + """ + 顺序弹出一个代理 + :return: proxy + """ + proxy = self.get(https) + if proxy: + self.__conn.hdel(self.name, json.loads(proxy).get("proxy", "")) + return proxy if proxy else None + + def delete(self, proxy_str): + """ + 移除指定代理, 使用changeTable指定hash name + :param proxy_str: proxy str + :return: + """ + self.__conn.hdel(self.name, proxy_str) + + def exists(self, proxy_str): + """ + 判断指定代理是否存在, 使用changeTable指定hash name + :param proxy_str: proxy str + :return: + """ + return self.__conn.hexists(self.name, proxy_str) + + def update(self, proxy_obj): + """ + 更新 proxy 属性 + :param proxy_obj: + :return: + """ + self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) + + def getAll(self, https): + """ + 字典形式返回所有代理, 使用changeTable指定hash name + :return: + """ + item_dict = self.__conn.hgetall(self.name) + if https: + return list(filter(lambda x: json.loads(x).get("https"), item_dict.values())) + else: + return item_dict.values() + + def clear(self): + """ + 清空所有代理, 使用changeTable指定hash name + :return: + """ + return self.__conn.delete(self.name) + + def getCount(self): + """ + 返回代理数量 + :return: + """ + proxies = self.getAll(https=False) + return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))} + + def changeTable(self, name): + """ + 切换操作对象 + :param name: + :return: + """ + self.name = name + + def test(self): + log = LogHandler('ssdb_client') + try: + self.getCount() + except TimeoutError as e: + log.error('ssdb connection time out: %s' % str(e), exc_info=True) + return e + except ConnectionError as e: + log.error('ssdb connection error: %s' % str(e), exc_info=True) + return e + except ResponseError as e: + log.error('ssdb connection error: %s' % str(e), exc_info=True) + return e diff --git a/doc/introduce.md b/doc/introduce.md deleted file mode 100644 index 13f45317a..000000000 --- a/doc/introduce.md +++ /dev/null @@ -1,173 +0,0 @@ - -## 代理池介绍 - -本项目通过爬虫方式持续抓取代理网站公布的免费代理IP，实时校验，维护部分可以使用的代理，并通过api的形式提供外部使用。 - -### 1、问题 - -构建一个代理IP池，可能有下面这些问题： - -* 代理IP从何而来？ - -　　许多刚接触爬虫的，都试过去西刺、快代理之类有免费代理的网站去抓些免费代理，还是有一些代理能用。 -当然，如果你有更好的代理接口也可以自己接入。 - -　　免费代理的采集也很简单，无非就是：`访问页面`` —> `正则/xpath提取` —> `保存` - -* 如何保证代理质量？ - -　　可以肯定免费的代理IP大部分都是不能用的，不然别人还提供付费接口干嘛(不过事实上很多代理商的付费IP也不稳定，也有很多是不能用)。 -所以采集回来的代理IP不能直接使用，检测的办法也很简单：可以写个程序不断的用代理访问一个稳定的网站，看是否可以正常访问即可。 -这个过程可以使用多线/进程或异步的方式，因为检测代理是个很慢的过程。 - -* 采集回来的代理如何存储？ - -　　这里不得不推荐一个国人开发的高性能支持多种数据结构的NoSQL数据库[SSDB](http://ssdb.io/docs/zh_cn/)，用于替代Redis。支持队列、hash、set、k-v对，支持T级别数据。是做分布式爬虫很好中间存储工具。 - -* 如何让爬虫更方便的用到这些代理？ - -　　答案肯定是做成服务咯，Python有这么多的web框架，随便拿一个来写个api供爬虫调用。这样代理和爬虫架构分离有很多好处， -比如：当爬虫完全不用考虑如何校验代理，如何保证拿到的代理可用，这些都由代理池来完成。这样只需要安静的码爬虫代码就行啦。 - -### 2、代理池设计 - -　　代理池由四部分组成: - -* ProxyGetter: - -　　代理获取接口，目前有5个免费代理源，每调用一次就会抓取这个5个网站的最新代理放入DB，支持自定义扩展额外的代理获取接口； - -* DB: - -　　用于存放代理IP，目前支持SSDB和Redis(推荐SSDB)。至于为什么选择SSDB，大家可以参考这篇[文章](https://www.sdk.cn/news/2684),个人觉得SSDB是个不错的Redis替代方案，如果你没有用过SSDB，安装起来也很简单，可以参考[这里](https://github.com/jhao104/memory-notes/blob/master/SSDB/SSDB%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE%E8%AE%B0%E5%BD%95.md)； - -* Schedule: - -　　计划任务，定时去检测DB中的代理可用性，删除不可用的代理。同时也会主动通过ProxyGetter去获取最新代理放入DB； - -* ProxyApi: - -　　代理池的外部接口，由[Flask](http://flask.pocoo.org/)实现，功能是给爬虫提供与代理池交互的接口。 - - -![设计](https://pic2.zhimg.com/v2-f2756da2986aa8a8cab1f9562a115b55_b.png) - -### 3、代码模块 - -　　Python中高层次的数据结构,动态类型和动态绑定,使得它非常适合于快速应用开发,也适合于作为胶水语言连接已有的软件部件。用Python来搞这个代理IP池也很简单，代码分为6个模块： - -* Api: - -　　api接口相关代码，目前api是由Flask实现，代码也非常简单。客户端请求传给Flask，Flask调用`ProxyManager`中的实现，包括`get/delete/refresh/get_all`； - -* DB: - -　　数据库相关代码，目前数据库是支持SSDB/Redis。代码用工厂模式实现，方便日后扩展其他类型数据库； - -* Manager: - -　　`get/delete/refresh/get_all`等接口的具体实现类，目前代理池只负责管理proxy，日后可能会有更多功能，比如代理和爬虫的绑定，代理和账号的绑定等等； - -* ProxyGetter: - -　　代理获取的相关代码，目前抓取了[快代理](http://www.kuaidaili.com)、[代理66](http://www.66ip.cn/)、[有代理](http://www.youdaili.net/Daili/http/)、[西刺代理](http://api.xicidaili.com/free2016.txt)、[guobanjia](http://www.goubanjia.com/free/gngn/index.shtml)这个五个网站的免费代理，经测试这个5个网站每天更新的可用代理只有六七十个，当然也支持自己扩展代理接口； - -* Schedule: - -　　定时任务相关代码，现在只是实现定时去刷新代理，并验证可用代理，采用多进程方式； - -* Util: - -　　存放一些公共的模块方法或函数，包含`GetConfig`:读取配置文件config.ini的类，`ConfigParse`: 扩展ConfigParser的类，使其对大小写敏感， `Singleton`:实现单例，`LazyProperty`:实现类属性惰性计算。等等； - -* 其他文件: - -　　配置文件:`Config.ini``,数据库配置和代理获取接口配置，可以在GetFreeProxy中添加新的代理获取方法，并在Config.ini中注册即可使用； - -### 4、安装 - -下载代码: -``` -git clone git@github.com:jhao104/proxy_pool.git - -或者直接到https://github.com/jhao104/proxy_pool 下载zip文件 -``` - -安装依赖: -``` -pip install -r requirements.txt -``` - -启动: - -``` -如果你的依赖已经安全完成并且具备运行条件,可以直接在Run下运行main.py -到Run目录下: ->>>python main.py - -如果运行成功你应该可以看到有4个main.py进程在 - - -你也可以分别运行他们,依次到Api下启动ProxyApi.py,Schedule下启动ProxyRefreshSchedule.py和ProxyValidSchedule.py即可 -``` - -docker: -``` -git clone git@github.com:jhao104/proxy_pool.git - -cd proxy_pool - -docker build -t proxy:latest -f Dockerfile . - -docker run -p 5010:5010 -d proxy:latest - -# Wait a few minutes -curl localhost:5010/get/ -# result: xxx.xxx.xxx.xxx:xxxx - -curl localhost:5010/get_all/ -``` - -### 5、使用 -　　定时任务启动后，会通过GetFreeProxy中的方法抓取代理存入数据库并验证。此后默认每10分钟会重复执行一次。定时任务启动大概一两分钟后，便可在[SSDB](https://github.com/jhao104/SSDBAdmin)中看到刷新出来的可用的代理： - -![useful_proxy](https://pic2.zhimg.com/v2-12f9b7eb72f60663212f317535a113d1_b.png) - -　　启动ProxyApi.py后即可在浏览器中使用接口获取代理，一下是浏览器中的截图: - -　　index页面: - -![index](https://pic3.zhimg.com/v2-a867aa3db1d413fea8aeeb4c693f004a_b.png) - -　　get： - -![get](https://pic1.zhimg.com/v2-f54b876b428893235533de20f2edbfe0_b.png) - -　　get_all： - -![get_all](https://pic3.zhimg.com/v2-5c79f8c07e04f9ef655b9bea406d0306_b.png) - - -　　爬虫中使用，如果要在爬虫代码中使用的话， 可以将此api封装成函数直接使用，例如: -``` -import requests - -def get_proxy(): - return requests.get("http://127.0.0.1:5010/get/").content - -def delete_proxy(proxy): - requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy)) - -# your spider code - -def spider(): - # .... - requests.get('https://www.example.com', proxies={"http": "http://{}".format(get_proxy())}) - # .... - -``` - -　　测试地址：http://123.207.35.36:5010 单机勿压测。谢谢 - -### 6、最后 -　　时间仓促，功能和代码都比较简陋，以后有时间再改进。喜欢的在github上给个star。感谢！ diff --git a/doc/release_notes.md b/doc/release_notes.md deleted file mode 100644 index 0871a2db5..000000000 --- a/doc/release_notes.md +++ /dev/null @@ -1,21 +0,0 @@ -## Release Notes - -* 1.12 (2018.4) - - 1.优化代理格式检查; - - 2.增加代理源; - - 3.fix bug [#122](https://github.com/jhao104/proxy_pool/issues/122) [#126](https://github.com/jhao104/proxy_pool/issues/126) - -* 1.11 (2017.8) - -　　1.使用多线程验证useful_pool; - -* 1.10 (2016.11) - -　　1. 第一版； - -　　2. 支持PY2/PY3; - -　　3. 代理池基本功能； \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9d1a10ba4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' +services: + proxy_pool: + build: . + container_name: proxy_pool + ports: + - "5010:5010" + links: + - proxy_redis + environment: + DB_CONN: "redis://@proxy_redis:6379/0" + proxy_redis: + image: "redis" + container_name: proxy_redis \ No newline at end of file diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 000000000..875d378c8 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,79 @@ +# API 使用 + +## 接口列表 + +启动 ProxyPool 的 `server` 后会提供如下 HTTP 接口： + +| 接口 | 方法 | 说明 | 参数 | +|------|------|------|------| +| `/` | GET | 返回 API 列表 | 无 | +| `/get` | GET | 随机返回一个代理 | 可选：`?type=https` 过滤 HTTPS 代理 | +| `/pop` | GET | 返回并删除一个代理 | 可选：`?type=https` 过滤 HTTPS 代理 | +| `/all` | GET | 返回所有代理 | 可选：`?type=https` 过滤 HTTPS 代理 | +| `/count` | GET | 返回代理数量统计 | 无 | +| `/delete` | GET | 删除指定代理 | `?proxy=host:port` | + +## 调用示例 + +### 在爬虫中使用 + +通过调用 API 接口来使用代理池： + +```python +import requests + + +def get_proxy(): + return requests.get("http://127.0.0.1:5010/get/").json() + + +def delete_proxy(proxy): + requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy)) + + +def get_html(): + retry_count = 5 + proxy = get_proxy().get("proxy") + while retry_count > 0: + try: + # 使用代理访问 + html = requests.get( + "http://www.example.com", + proxies={ + "http": "http://{}".format(proxy), + "https": "https://{}".format(proxy), + }, + ) + return html + except Exception: + retry_count -= 1 + # 删除代理池中代理 + delete_proxy(proxy) + return None +``` + +本例中在本地 `127.0.0.1` 启动端口为 `5010` 的 `server`，使用 `/get` 接口获取代理，`/delete` 删除代理。 + +### 获取 HTTPS 代理 + +```python +# 只获取支持 HTTPS 的代理 +proxy = requests.get("http://127.0.0.1:5010/get/?type=https").json() +``` + +### 获取代理统计 + +```python +# 返回代理数量、类型分布、来源分布 +stats = requests.get("http://127.0.0.1:5010/count/").json() +# 示例返回: {"http_type": {"http": 10, "https": 5}, "source": {"freeProxy01": 8, "freeProxy02": 7}, "count": 15} +``` + +## 直接读取数据库 + +除了通过 API 接口，也可以直接读取数据库获取代理。目前支持两种数据库：Redis 和 SSDB。 + +- **Redis**：存储结构为 hash，hash name 为配置项中的 `TABLE_NAME`（默认 `use_proxy`） +- **SSDB**：存储结构为 hash，hash name 为配置项中的 `TABLE_NAME` + +可以在代码中自行读取数据库获取代理列表。 \ No newline at end of file diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 000000000..3f1c5c65b --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,12 @@ + + + + + + + + + P + + + \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 000000000..75001fac0 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,116 @@ +# 变更日志 + +## Next (unreleased) + +1. 新增代理源 **谷德代理**; (2026-05-14) +2. 引入tox自动化测试, **放弃Python 3.7以下版本支持**; (2026-05-14) +3. 新增统一服务管理脚本 ``proxy_pool.sh``, 支持start/stop/restart/status命令; (2026-05-26) +4. 优化CI配置, 避免PR时重复触发测试; (2026-05-26) +5. 迁移文档从 Sphinx/ReadTheDocs 到 MkDocs Material/GitHub Pages; (2026-05-27) +6. **重写测试套件**: 使用pytest重构全部测试, 覆盖unit/api/integration三层; (2026-05-28) +7. **重构代理采集模块**: 将 `proxyFetcher.py` 拆分为独立文件 + `BaseFetcher` 基类插件架构，支持自动扫描和运行时热更新; (2026-05-31) + - 每个代理源独立文件（`fetcher/sources/`），继承 `BaseFetcher` 基类 + - 自动扫描启用的代理源，无需在配置中列举 + - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 + - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 +8. 新增代理源 **Proxifly**; (2026-06-01) +9. 移除失效代理源 **FreeProxyList**; (2026-06-02) +10. 新增代理源 **66代理 (daili66)**; (2026-06-08) +11. 移除失效代理源 **66代理 (ip66)**; (2026-06-08) +12. 新增代理源 **RoundProxies**; (2026-06-09) +13. 移除失效代理源 **免费代理库 (jiangxianli)**; (2026-06-09) + + + +## 2.4.2 (2024-01-18) + +1. 代理格式检查支持需认证的代理格式 `username:password@ip:port`; (2023-03-10) +2. 新增代理源 **稻壳代理**; (2023-05-15) +3. 新增代理源 **冰凌代理**; (2023-01-18) + +## 2.4.1 (2022-07-17) + +1. 新增代理源 **FreeProxyList**; (2022-07-21) +2. 新增代理源 **FateZero**; (2022-08-01) +3. 新增代理属性 `region`; (2022-08-16) + +## 2.4.0 (2021-11-17) + +1. 移除无效代理源 **神鸡代理**; (2021-11-16) +2. 移除无效代理源 **极速代理**; (2021-11-16) +3. 移除代理源 **西拉代理**; (2021-11-16) +4. 新增代理源 **蝶鸟IP**; (2021-11-16) +5. 新增代理源 **PROXY11**; (2021-11-16) +6. 多线程采集代理; (2021-11-17) + +## 2.3.0 (2021-05-27) + +1. 修复Dockerfile时区问题; (2021-04-12) +2. 新增Proxy属性 `source`, 标记代理来源; (2021-04-13) +3. 新增Proxy属性 `https`, 标记支持https的代理; (2021-05-27) + +## 2.2.0 (2021-04-08) + +1. 启动时检查数据库连通性; +2. 新增免费代理源 **米扑代理**; +3. 新增免费代理源 **Pzzqz**; +4. 新增免费代理源 **神鸡代理**; +5. 新增免费代理源 **极速代理**; +6. 新增免费代理源 **小幻代理**; + +## 2.1.1 (2021-02-23) + +1. Fix Bug [#493](https://github.com/jhao104/proxy_pool/issues/493), 新增时区配置; (2020-08-12) +2. 修复 **66代理** 采集; (2020-11-04) +3. 修复 **全网代理** 采集, 解决HTML端口加密问题; (2020-11-04) +4. 新增 **代理盒子** 免费源; (2020-11-04) +5. 新增 `POOL_SIZE_MIN` 配置项, runProxyCheck时, 剩余代理少于POOL_SIZE_MIN触发抓取; (2021-02-23) + +## 2.1.0 (2020-07) + +1. 新增免费代理源 **西拉代理**; (2020-03-30) +2. Fix Bug [#356](https://github.com/jhao104/proxy_pool/issues/356) [#401](https://github.com/jhao104/proxy_pool/issues/401) +3. 优化Docker镜像体积; (2020-06-19) +4. 优化配置方式; +5. 优化代码结构; +6. 不再储存raw_proxy, 抓取后直接验证入库; + +## 2.0.1 (2019-10) + +1. 新增免费代理源 **89免费代理**; +2. 新增免费代理源 **齐云代理**; + +## 2.0.0 (2019-08) + +1. WebApi集成Gunicorn方式启动, Windows平台暂不支持; +2. 优化Proxy调度程序; +3. 扩展Proxy属性; +4. 新增cli工具, 更加方便启动proxyPool; + +## 1.14 (2019-07) + +1. 修复 Queue阻塞导致的 `ProxyValidSchedule` 假死bug; +2. 修改代理源 **云代理** 抓取; +3. 修改代理源 **码农代理** 抓取; +4. 修改代理源 **代理66** 抓取, 引入 `PyExecJS` 模块破解加速乐动态Cookies加密; + +## 1.13 (2019-02) + +1. 使用.py文件替换.ini作为配置文件; +2. 优化代理采集部分; + +## 1.12 (2018-04) + +1. 优化代理格式检查; +2. 增加代理源; +3. Fix Bug [#122](https://github.com/jhao104/proxy_pool/issues/122) [#126](https://github.com/jhao104/proxy_pool/issues/126) + +## 1.11 (2017-08) + +1. 使用多线程验证useful_pool; + +## 1.10 (2016-11) + +1. 第一版; +2. 支持PY2/PY3; +3. 代理池基本功能; \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..bd71fc7a5 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,118 @@ +# 配置参考 + +配置文件 `setting.py` 位于项目的主目录下，配置主要分为五类：**服务配置**、**数据库配置**、**采集配置**、**校验配置**、**调度配置**。 + +## 服务配置 + +### `HOST` + +API 服务监听的 IP。本机访问设置为 `127.0.0.1`，开启远程访问设置为 `0.0.0.0`。 + +- 默认值：`"0.0.0.0"` + +### `PORT` + +API 服务监听的端口。 + +- 默认值：`5010` + +## 数据库配置 + +### `DB_CONN` + +存放代理 IP 的数据库 URI，配置格式为： + +``` +db_type://[[user]:[pwd]]@ip:port/[db] +``` + +目前支持的 `db_type`：`redis`、`ssdb`。 + +配置示例： + +```python +# Redis +DB_CONN = 'redis://@127.0.0.1:6379' +DB_CONN = 'redis://:123456@127.0.0.1:6379' +DB_CONN = 'redis://:123456@127.0.0.1:6379/15' + +# SSDB +DB_CONN = 'ssdb://@127.0.0.1:8888' +DB_CONN = 'ssdb://:123456@127.0.0.1:8888' +``` + +### `TABLE_NAME` + +存放代理的数据载体名称。SSDB 和 Redis 的存放结构为 hash。 + +- 默认值：`"use_proxy"` + +## 采集配置 + +代理采集采用插件架构，调度器自动扫描 `fetcher/sources/` 目录，加载所有 `enabled=True` 的代理源。新增代理源只需在 `sources/` 下创建文件，无需修改配置。 + +查看当前启用的代理源： + +```bash +python proxyPool.py fetcher +``` + +### `PROXY_FETCHER_EXCLUDE` + +代理源黑名单。列表中的类名对应的代理源不会被加载，即使 `enabled=True`。适用于临时禁用某个代理源而不修改其源文件。 + +```python +PROXY_FETCHER_EXCLUDE = [ + # "BinglxFetcher", # 临时禁用冰凌代理 +] +``` + +如需永久禁用，建议直接在源文件中设置 `enabled = False`。 + +## 校验配置 + +### `HTTP_URL` + +用于检验代理是否可用的地址。 + +- 默认值：`"http://httpbin.org"` + +### `HTTPS_URL` + +用于检验代理是否支持 HTTPS 的地址。 + +- 默认值：`"https://www.qq.com"` + +### `VERIFY_TIMEOUT` + +检验代理的超时时间，单位秒。使用代理访问 `HTTP_URL` / `HTTPS_URL` 耗时超过 `VERIFY_TIMEOUT` 时，视为代理不可用。 + +- 默认值：`10` + +### `MAX_FAIL_COUNT` + +检验代理允许的最大失败次数。超过则剔除代理。 + +- 默认值：`0`（即失败一次即删除） + +### `POOL_SIZE_MIN` + +代理检测定时任务运行前，若代理数量小于 `POOL_SIZE_MIN`，则先运行抓取程序。 + +- 默认值：`20` + +## 代理属性 + +### `PROXY_REGION` + +是否启用代理地域属性。开启后会尝试解析代理 IP 的地理位置信息。 + +- 默认值：`True` + +## 调度配置 + +### `TIMEZONE` + +调度器的时区设置。如果在虚拟机上运行时出现 `ValueError: Timezone offset does not match system offset` 错误，请设置该配置项。 + +- 默认值：`"Asia/Shanghai"` \ No newline at end of file diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 000000000..bd620ae6a --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,54 @@ +# Docker 部署 + +## 使用 Docker 镜像 + +拉取并运行 Docker 镜像： + +```console +docker pull jhao104/proxy_pool:latest + +docker run --env DB_CONN=redis://:password@ip:port/0 -p 5010:5010 jhao104/proxy_pool:latest +``` + +`DB_CONN` 环境变量会覆盖 `setting.py` 中的数据库连接配置。 + +## 使用 docker-compose + +项目根目录下的 `docker-compose.yml` 定义了 ProxyPool 和 Redis 两个服务： + +```yaml +version: '2' +services: + proxy_pool: + build: . + container_name: proxy_pool + ports: + - "5010:5010" + links: + - proxy_redis + environment: + DB_CONN: "redis://@proxy_redis:6379/0" + proxy_redis: + image: "redis" + container_name: proxy_redis +``` + +启动： + +```console +docker-compose up -d +``` + +## 容器环境注意事项 + +在 Docker 容器中，建议使用前台模式启动服务： + +```console +./proxy_pool.sh start --fg +``` + +Dockerfile 中的 ENTRYPOINT 配置： + +```dockerfile +ENTRYPOINT ["tini", "--", "bash", "proxy_pool.sh", "start", "--fg"] +``` \ No newline at end of file diff --git a/docs/extending/fetcher.md b/docs/extending/fetcher.md new file mode 100644 index 000000000..0b9a2107f --- /dev/null +++ b/docs/extending/fetcher.md @@ -0,0 +1,118 @@ +# 扩展代理源 + +项目默认包含多个免费的代理获取源，但是免费的毕竟质量有限，如果直接运行可能拿到的代理质量不理想。因此提供了用户自定义扩展代理获取的方法。 + +## 添加新的代理源 + +### 第一步：创建代理源文件 + +在 `fetcher/sources/` 目录下新建一个 `.py` 文件，继承 `BaseFetcher` 基类，实现 `fetch()` 方法： + +```python +# fetcher/sources/mySource.py + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class MySourceFetcher(BaseFetcher): + """我的代理源 https://example.com/proxy""" + + name = "mysource" # 唯一标识，用于日志 + url = "https://example.com/proxy" # 源网站首页 + enabled = True # 设为 False 可禁用 + + def fetch(self): + """yield "host:port" 格式的代理字符串""" + r = WebRequest().get(self.url, timeout=10) + for proxy in self.parseProxiesFromText(r.text): + yield proxy + + +if __name__ == '__main__': + for proxy in MySourceFetcher().fetch(): + print(proxy) +``` + +添加完成后，调度器下一轮采集（默认 5 分钟）会自动发现并启用新源，**无需修改任何配置文件**。 + +### 第二步：验证 + +独立调试代理源： + +```bash +python -m fetcher.sources.mySource +``` + +查看当前启用的代理源： + +```bash +python proxyPool.py fetcher +``` + +## 禁用代理源 + +有两种方式禁用某个代理源： + +**方式一：修改源文件**（推荐） + +在对应py文件中将 `enabled` 设为 `False`： + +```python +class MySourceFetcher(BaseFetcher): + enabled = False # 禁用该源 +``` + +**方式二：黑名单配置** + +在 `setting.py` 的 `PROXY_FETCHER_EXCLUDE` 列表中添加类名，无需修改源文件： + +```python +PROXY_FETCHER_EXCLUDE = ["MySourceFetcher"] +``` + +## BaseFetcher 基类 + +所有代理源必须继承 `BaseFetcher`，基类提供以下约定和工具： + +### 必须声明的属性 + +| 属性 | 类型 | 说明 | +|------|------|------| +| `name` | str | 唯一标识，用于日志和代理来源标记 | +| `url` | str | 源网站首页 URL | + +### 可选属性 + +| 属性 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| `enabled` | bool | `True` | 是否启用 | + +### 必须实现的方法 + +| 方法 | 说明 | +|------|------| +| `fetch(self)` | 生成器，yield `"host:port"` 格式字符串 | + +### 共享解析工具 + +| 方法 | 说明 | +|------|------| +| `parseProxiesFromText(text)` | 从纯文本中用正则提取 ip:port | +| `yieldUniqueProxies(proxies)` | 去重 yield | + +## 运行时热更新 + +调度器每轮采集时会重新扫描 `fetcher/sources/` 目录并 reload 模块，因此： + +- 新增文件 → 下一轮自动启用 +- 修改文件内容 → 下一轮自动加载新版本 +- 删除文件 → 下一轮自动移除（建议先从 `PROXY_FETCHER_EXCLUDE` 或 `enabled` 中禁用） + +## 命名规范 + +| 元素 | 风格 | 示例 | +|------|------|------| +| 文件名 | 小写 | `mysource.py` | +| 类名 | PascalCase | `MySourceFetcher` | +| `name` 属性 | 小写 | `"mysource"` | \ No newline at end of file diff --git a/docs/extending/validator.md b/docs/extending/validator.md new file mode 100644 index 000000000..1a844cac1 --- /dev/null +++ b/docs/extending/validator.md @@ -0,0 +1,68 @@ +# 扩展校验器 + +## 内置校验 + +项目中使用的代理校验方法全部定义在 `helper/validator.py` 中，通过 `ProxyValidator` 类中提供的装饰器来区分。校验方法返回 `True` 表示校验通过，返回 `False` 表示校验不通过。 + +代理校验方法分为三类： + +| 类型 | 装饰器 | 说明 | +|------|--------|------| +| `preValidator` | `@ProxyValidator.addPreValidator` | 预校验，在代理抓取后验证前调用 | +| `httpValidator` | `@ProxyValidator.addHttpValidator` | 代理可用性校验，通过则认为代理可用 | +| `httpsValidator` | `@ProxyValidator.addHttpsValidator` | 校验代理是否支持 HTTPS | + +每种校验可以定义多个方法，只有**所有**方法都返回 `True` 的情况下才视为该校验通过。 + +### 校验执行顺序 + +```mermaid +graph LR + A[抓取代理] --> B[preValidator] + B -->|通过| C[httpValidator] + B -->|失败| F[丢弃] + C -->|通过| D[代理可用] + C -->|失败| F + D --> E[httpsValidator] + E -->|通过| G[标记 HTTPS=True] + E -->|失败| H[标记 HTTPS=False] +``` + +- `preValidator` 校验通过的代理才会进入可用性校验 +- `httpValidator` 校验通过后认为代理可用，更新入代理池 +- `httpsValidator` 校验通过后视为代理支持 HTTPS，更新代理的 `https` 属性为 `True` + +## 扩展校验 + +在 `helper/validator.py` 中已有自定义校验的示例，自定义函数需返回 `True` 或者 `False`，使用 `ProxyValidator` 中提供的装饰器来区分校验类型。 + +### 示例 1：自定义代理可用性校验 + +```python +@ProxyValidator.addHttpValidator +def customValidatorExample01(proxy): + """自定义代理可用性校验函数""" + proxies = {"http": "http://{proxy}".format(proxy=proxy)} + try: + r = requests.get("http://www.baidu.com/", headers=HEADER, proxies=proxies, timeout=5) + return True if r.status_code == 200 and len(r.content) > 200 else False + except Exception as e: + return False +``` + +### 示例 2：自定义 HTTPS 校验 + +```python +@ProxyValidator.addHttpsValidator +def customValidatorExample02(proxy): + """自定义代理是否支持 HTTPS 校验函数""" + proxies = {"https": "https://{proxy}".format(proxy=proxy)} + try: + r = requests.get("https://www.baidu.com/", headers=HEADER, proxies=proxies, timeout=5, verify=False) + return True if r.status_code == 200 and len(r.content) > 200 else False + except Exception as e: + return False +``` + +!!! note + 在运行代理可用性校验时，所有被 `ProxyValidator.addHttpValidator` 装饰的函数会依次按定义顺序执行，只有当所有函数都返回 `True` 时才会判断代理可用。`HttpsValidator` 运行机制也是如此。 \ No newline at end of file diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..62073ea1b --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,160 @@ +# 快速开始 + +## 下载代码 + +本项目需要下载代码到本地运行，通过 `git` 下载： + +```console +git clone https://github.com/jhao104/proxy_pool.git +``` + +或者下载特定的 [release](https://github.com/jhao104/proxy_pool/releases) 版本。 + +## 安装依赖 + +到项目目录下使用 `pip` 安装依赖库： + +```console +pip install -r requirements.txt +``` + +## 更新配置 + +配置文件 `setting.py` 位于项目的主目录下，常用的配置项： + +```python +# API 服务 +HOST = "0.0.0.0" # 监听 IP +PORT = 5010 # 监听端口 + +# 数据库 +DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0' + +# 代理采集方法 +PROXY_FETCHER = [ + "freeProxy01", # 所有 fetch 方法位于 fetcher/proxyFetcher.py + "freeProxy02", + # .... +] +``` + +更多配置请参考 [配置参考](configuration.md)。 + +## 启动项目 + +完整程序包含两部分：`schedule` 调度程序和 `server` API 服务。调度程序负责采集和验证代理，API 服务提供代理服务 HTTP 接口。 + +### 方式一：使用 `proxy_pool.sh`（推荐） + +`proxy_pool.sh` 提供统一的服务管理接口，支持后台运行和进程管理： + +```console +# 后台启动所有服务 +./proxy_pool.sh start + +# 前台启动（容器环境） +./proxy_pool.sh start --fg + +# 停止服务 +./proxy_pool.sh stop + +# 查看状态 +./proxy_pool.sh status + +# 重启服务 +./proxy_pool.sh restart +``` + +### 方式二：使用 `proxyPool.py` + +`proxyPool.py` 是项目的 Python CLI 入口，可以分别启动调度程序和 API 服务： + +```console +# 启动调度程序 +python proxyPool.py schedule + +# 启动 API 服务 +python proxyPool.py server +``` + +## 服务管理 + +### proxy_pool.sh 可用命令 + +| 命令 | 说明 | +|------|------| +| `start` | 启动所有服务（默认后台运行） | +| `start --fg` | 前台启动，适用于容器环境 | +| `stop` | 停止所有服务 | +| `restart` | 重启所有服务 | +| `status` | 查看服务运行状态 | + +### PID 文件 + +服务启动后会在项目根目录生成 `proxy_pool.pid` 文件，记录所有子进程的 PID。该文件用于 `stop` 命令识别需要终止的进程、`status` 命令检查进程状态、防止重复启动。`stop` 命令执行后会自动删除该文件。 + +## 故障排除 + +### 服务启动失败 + +使用前台启动查看详细日志排查错误： + +```console +./proxy_pool.sh start --fg +``` + +### 端口被占用 + +修改 `setting.py` 中的 `PORT` 配置： + +```python +PORT = 5010 # 修改为其他端口 +``` + +### 无法停止服务 + +手动终止进程： + +```console +# 查看 PID 文件 +cat proxy_pool.pid + +# 手动终止进程 +kill + +# 删除 PID 文件 +rm proxy_pool.pid +``` + +## 运行测试 + +### 安装测试依赖 + +```console +pip install -r requirements-test.txt +``` + +### 运行全部测试 + +```console +pytest +``` + +### 分层运行 + +```console +# 单元测试（零外部依赖，CI 必跑） +pytest tests/unit/ + +# API 路由测试 +pytest tests/api/ + +# 集成测试（RedisClient/SsdbClient CRUD，使用 fakeredis 模拟） +pytest tests/integration/ +``` + +### 查看覆盖率 + +```console +pytest --cov=. --cov-report=term-missing +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..1ed6a938a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,117 @@ +--- +hide: + - navigation + - toc +--- + +

+ +# ProxyPool + +**Python爬虫代理IP池** — 定时采集、验证、存储免费代理，通过 RESTful API 提供服务。 + +[:octicons-mark-github-16: GitHub](https://github.com/jhao104/proxy_pool){ .md-button } +[:octicons-rocket-16: 快速开始](getting-started.md){ .md-button .md-button--primary } + +
+ +
+ +
+ +### :material-access-point-network: 多源采集 + +内置 15+ 免费代理源，支持自定义扩展，定时自动采集。 + +
+ +
+ +### :material-shield-check: 自动验证 + +HTTP/HTTPS 可用性自动校验，剔除失效代理，保证代理质量。 + +
+ +
+ +### :material-database: 持久存储 + +Redis/SSDB 持久化存储，支持集群部署，数据不丢失。 + +
+ +
+ +### :material-api: RESTful API + +提供 `/get`、`/pop`、`/all`、`/count`、`/delete` 等接口，开箱即用。 + +
+ +
+ +### :material-docker: Docker 部署 + +一条命令启动，支持 docker-compose，自带 Redis 服务。 + +
+ +
+ +### :material-clock-fast: 定时调度 + +APScheduler 驱动，自动维护代理池数量，无需人工干预。 + +
+ +
+ +--- + +## 快速开始 + +```bash +# 克隆项目 +git clone https://github.com/jhao104/proxy_pool.git +cd proxy_pool + +# 安装依赖 +pip install -r requirements.txt + +# 启动调度程序（采集和验证代理） +python proxyPool.py schedule + +# 启动 API 服务 +python proxyPool.py server +``` + +启动后访问 `http://127.0.0.1:5010/get` 即可获取一个代理。 + +## API 示例 + +```python +import requests + +# 获取代理 +proxy = requests.get("http://127.0.0.1:5010/get/").json() + +# 使用代理 +html = requests.get( + "http://www.example.com", + proxies={"http": f"http://{proxy['proxy']}"} +) +``` + +## 文档导航 + +| 章节 | 说明 | +|------|------| +| [快速开始](getting-started.md) | 安装、配置、启动项目 | +| [项目结构](project-structure.md) | 目录结构与核心模块说明 | +| [配置参考](configuration.md) | `setting.py` 全部配置项详解 | +| [API 使用](api.md) | RESTful API 端点与调用示例 | +| [Docker 部署](docker.md) | Docker / docker-compose 部署方式 | +| [扩展代理源](extending/fetcher.md) | 自定义代理采集方法 | +| [扩展校验器](extending/validator.md) | 自定义代理校验规则 | +| [变更日志](changelog.md) | 版本发布记录 | \ No newline at end of file diff --git a/docs/project-structure.md b/docs/project-structure.md new file mode 100644 index 000000000..30077d8b2 --- /dev/null +++ b/docs/project-structure.md @@ -0,0 +1,87 @@ +# 项目结构 + +ProxyPool 项目目录结构如下： + +``` +proxy_pool/ +├── api/ # API 服务 +│ └── proxyApi.py # Flask RESTful 接口 +├── db/ # 数据库层 +│ ├── dbClient.py # 抽象数据库接口 +│ ├── redisClient.py # Redis 实现 +│ └── ssdbClient.py # SSDB 实现 +├── fetcher/ # 代理采集器 +│ ├── baseFetcher.py # BaseFetcher 基类（共享解析方法） +│ └── sources/ # 各代理源独立文件 +│ ├── zdaye.py # 站大爷 +│ ├── kxdaili.py # 开心代理 +│ ├── kuaidaili.py # 快代理 +│ ├── geonode.py # Geonode +│ └── ... # 其他代理源 +├── handler/ # 业务处理器 +│ ├── configHandler.py # 配置读取 +│ ├── logHandler.py # 日志处理 +│ └── proxyHandler.py # 代理 CRUD 逻辑 +├── helper/ # 核心辅助模块 +│ ├── scheduler.py # APScheduler 定时调度 +│ ├── validator.py # 代理可用性校验 +│ ├── proxy.py # 代理数据模型 +│ ├── fetch.py # 采集任务执行 +│ └── check.py # 校验任务执行 +├── util/ # 工具库 +│ ├── singleton.py # 单例元类 +│ ├── lazyProperty.py # 惰性属性装饰器 +│ ├── six.py # Python 2/3 兼容层 +│ └── webRequest.py # HTTP 请求封装 +├── tests/ # 测试 +│ ├── conftest.py # 共享 fixtures +│ ├── unit/ # 单元测试（零外部依赖） +│ ├── api/ # API 路由测试（Flask test client） +│ └── integration/ # 集成测试（RedisClient/SsdbClient CRUD） +├── docs/ # MkDocs 文档源文件 +├── proxyPool.py # CLI 入口（click） +├── proxy_pool.sh # 服务管理脚本 +├── setting.py # 全局配置文件 +├── requirements.txt # Python 依赖 +├── requirements-test.txt # 测试依赖（pytest、pytest-cov、fakeredis） +├── pyproject.toml # pytest 配置 +├── Dockerfile # Docker 镜像构建 +├── docker-compose.yml # Docker Compose 编排 +└── tox.ini # 多版本测试配置 +``` + +## 核心模块说明 + +### `proxyPool.py` — 入口 + +基于 click 的命令行入口，提供 `schedule`、`server`、`fetcher` 三个子命令。`schedule` 启动代理采集和验证调度器，`server` 启动 Flask API 服务，`fetcher` 查看当前启用的代理源列表。 + +### `api/proxyApi.py` — API 服务 + +Flask 应用，提供 `/get`、`/pop`、`/all`、`/count`、`/delete` 等接口，运行在 `setting.py` 配置的 `HOST:PORT`（默认 `0.0.0.0:5010`）。 + +### `db/` — 数据库层 + +通过 `dbClient.py` 定义统一接口，`redisClient.py` 和 `ssdbClient.py` 分别实现 Redis 和 SSDB 的存取逻辑。使用 `setting.py` 中的 `DB_CONN` 连接字符串选择后端。 + +### `fetcher/` — 代理采集 + +采用插件架构。`baseFetcher.py` 定义 `BaseFetcher` 基类，提供共享解析方法和 `name`/`url`/`enabled` 属性约定。每个代理源在 `sources/` 目录下独立一个文件，继承 `BaseFetcher` 并实现 `fetch()` 方法。调度器自动扫描 `sources/` 目录，加载 `enabled=True` 的源。通过 `setting.py` 的 `PROXY_FETCHER_EXCLUDE` 黑名单可临时禁用指定源。 + +### `helper/scheduler.py` — 定时调度 + +基于 APScheduler，按配置间隔驱动采集器和验证器运行，自动维护代理池数量。 + +### `helper/validator.py` — 代理校验 + +使用 `HTTP_URL` 和 `HTTPS_URL` 测试代理可用性，超过 `MAX_FAIL_COUNT` 次失败的代理会被移除。 + +### `handler/` — 业务处理 + +- `configHandler.py`：封装 `setting.py` 配置项的读取 +- `logHandler.py`：统一日志配置 +- `proxyHandler.py`：代理的增删改查操作 + +### `setting.py` — 配置中心 + +所有运行时配置集中在此文件，包括 API 地址、数据库连接、采集器列表、校验参数等。详见 [配置参考](configuration.md)。 diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 000000000..3e8d0df5d --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,127 @@ +/* 科技感主题定制 */ + +/* 首页 Hero 区域 */ +.md-typeset .tx-hero { + background: linear-gradient(135deg, #3949ab 0%, #1a237e 100%); + color: #fff; + padding: 3rem 2rem; + border-radius: 12px; + margin-bottom: 2rem; +} + +.md-typeset .tx-hero h1 { + color: #fff !important; + font-weight: 700; + font-size: 2.4rem; + margin-bottom: 0.5rem; +} + +.md-typeset .tx-hero p { + color: rgba(255, 255, 255, 0.85); + font-size: 1.1rem; + line-height: 1.6; +} + +/* 特性卡片 */ +.tx-features { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 1.2rem; + margin: 2rem 0; +} + +.tx-feature { + padding: 1.5rem; + border: 1px solid var(--md-default-fg-color--lightest); + border-radius: 8px; + transition: border-color 0.2s, box-shadow 0.2s; +} + +.tx-feature:hover { + border-color: var(--md-accent-fg-color); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.08); +} + +.tx-feature h3 { + margin-top: 0; + font-size: 1rem; +} + +.tx-feature p { + margin-bottom: 0; + color: var(--md-default-fg-color--light); + font-size: 0.85rem; +} + +/* 快速开始按钮 */ +.md-typeset .tx-hero .md-button { + border-color: rgba(255, 255, 255, 0.5); + color: #fff; + font-weight: 600; +} + +.md-typeset .tx-hero .md-button:hover { + background-color: rgba(255, 255, 255, 0.15); + border-color: #fff; +} + +.md-typeset .tx-hero .md-button--primary { + background-color: #fff; + color: #1a237e; + border-color: #fff; +} + +.md-typeset .tx-hero .md-button--primary:hover { + background-color: rgba(255, 255, 255, 0.9); +} + +/* 顶部导航栏 */ +.md-header { + background: linear-gradient(90deg, #1a237e 0%, #3949ab 100%); +} + +.md-header__title { + font-weight: 600; +} + +/* 代码块增强 */ +.md-typeset code { + font-size: 0.82rem; +} + +/* 导航标签页 */ +.md-tabs { + background: var(--md-primary-fg-color); +} + +.md-tabs__link { + font-weight: 500; +} + +/* 搜索框 */ +.md-search__form { + background-color: rgba(255, 255, 255, 0.15); +} + +/* 侧边栏 */ +.md-sidebar--primary .md-sidebar__scrollwrap { + background: linear-gradient(180deg, rgba(57, 73, 171, 0.05) 0%, transparent 100%); +} + +/* 表格增强 */ +.md-typeset table:not([class]) { + border-radius: 8px; + overflow: hidden; +} + +.md-typeset table:not([class]) th { + background-color: var(--md-primary-fg-color--light); + color: #fff; + font-weight: 600; +} + +/* Admonition 增强 */ +.md-typeset .admonition, +.md-typeset details { + border-radius: 8px; +} \ No newline at end of file diff --git a/ProxyGetter/__init__.py b/fetcher/__init__.py similarity index 89% rename from ProxyGetter/__init__.py rename to fetcher/__init__.py index d1c5cc292..54820a3ba 100644 --- a/ProxyGetter/__init__.py +++ b/fetcher/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------- - File Name： __init__.py.py + File Name： __init__.py Description : Author : JHao date： 2016/11/25 diff --git a/fetcher/baseFetcher.py b/fetcher/baseFetcher.py new file mode 100644 index 000000000..f22f31e25 --- /dev/null +++ b/fetcher/baseFetcher.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： baseFetcher.py + Description : 代理源基类 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re + + +class BaseFetcher(object): + """代理源基类""" + + # ---- 子类必须声明 ---- + name = "" # 唯一标识，如 "zdaye" + url = "" # 源网站首页 URL + + # ---- 子类可覆盖 ---- + enabled = True # 是否启用，设为 False 可禁用该源 + + def fetch(self): + """爬取代理，yield "host:port" 字符串""" + raise NotImplementedError + + @staticmethod + def parseProxiesFromText(text): + """从文本中用正则提取 ip:port""" + if not text: + return [] + proxy_pattern = re.compile( + r'(?= 2: + ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) + port_match = re.match(r'^\d{2,5}$', cells[1]) + if ip_match and port_match: + proxies.append("%s:%s" % (cells[0], cells[1])) + proxies.extend(self.parseProxiesFromText(r.text)) + for proxy in self.yieldUniqueProxies(proxies): + yield proxy + + +if __name__ == '__main__': + for proxy in FreeVPNNodeFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/geonode.py b/fetcher/sources/geonode.py new file mode 100644 index 000000000..8e3171047 --- /dev/null +++ b/fetcher/sources/geonode.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： geonode.py + Description : Geonode代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class GeonodeFetcher(BaseFetcher): + """Geonode Free Proxy https://geonode.com/""" + + name = "geonode" + url = "https://geonode.com/" + + def fetch(self): + url = ("https://proxylist.geonode.com/api/proxy-list?" + "filterLastChecked=10&page=1&limit=100&sort_by=lastChecked&sort_type=desc") + r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) + try: + proxies = [] + for item in r.json.get("data", []): + ip = item.get("ip", "") + port = item.get("port", "") + if ip and port: + proxies.append("%s:%s" % (ip, port)) + if not proxies: + proxies = self.parseProxiesFromText(r.text) + for proxy in self.yieldUniqueProxies(proxies): + yield proxy + except Exception as e: + logger.error("ProxyFetch - geonode: %s" % e) + + +if __name__ == '__main__': + for proxy in GeonodeFetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/fetcher/sources/goodips.py b/fetcher/sources/goodips.py new file mode 100644 index 000000000..fd1ba2a10 --- /dev/null +++ b/fetcher/sources/goodips.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： goodips.py + Description : 谷德代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class GoodipsFetcher(BaseFetcher): + """谷德代理 https://www.goodips.com/""" + + name = "goodips" + url = "https://www.goodips.com/" + + def fetch(self): + url = "https://www.goodips.com/" + tree = WebRequest().get(url, verify=False).tree + for item in tree.xpath("//div[@class='table-list']"): + ip = "".join(item.xpath("./ul/li[1]/text()")).strip() + port = "".join(item.xpath("./ul/li[2]/text()")).strip() + if ip and port: + yield "%s:%s" % (ip, port) + + +if __name__ == '__main__': + for proxy in GoodipsFetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/fetcher/sources/ihuan.py b/fetcher/sources/ihuan.py new file mode 100644 index 000000000..2981f1133 --- /dev/null +++ b/fetcher/sources/ihuan.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： ihuan.py + Description : 小幻代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from lxml import etree +import requests + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class IhuanFetcher(BaseFetcher): + """小幻代理 https://ip.ihuan.me/""" + + name = "ihuan" + url = "https://ip.ihuan.me/" + enabled = True + + def fetch(self): + wb = WebRequest() + session = requests.session() + headers = wb.header + session.get(self.url, headers=headers, verify=False) # 必须先请求一起获取cookie + res = session.get(self.url, headers=headers, verify=False) + tree = etree.HTML(res.text) + for item in tree.xpath("//table[@class='table table-hover table-bordered']//tr"): + ip = "".join(item.xpath("./td[1]//text()")).strip() + port = "".join(item.xpath("./td[2]//text()")).strip() + if ip and port: + yield "%s:%s" % (ip, port) + + +if __name__ == '__main__': + for proxy in IhuanFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/ip3366.py b/fetcher/sources/ip3366.py new file mode 100644 index 000000000..d80bc3f90 --- /dev/null +++ b/fetcher/sources/ip3366.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： ip3366.py + Description : 云代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class Ip3366Fetcher(BaseFetcher): + """云代理 http://www.ip3366.net/""" + + name = "ip3366" + url = "http://www.ip3366.net/" + + def fetch(self): + urls = [ + 'http://www.ip3366.net/free/?stype=1', + "http://www.ip3366.net/free/?stype=2", + ] + for url in urls: + r = WebRequest().get(url, timeout=10) + proxies = re.findall( + r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?(\d+)', + r.text) + for proxy in proxies: + yield ":".join(proxy) + + +if __name__ == '__main__': + for proxy in Ip3366Fetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/fetcher/sources/ip89.py b/fetcher/sources/ip89.py new file mode 100644 index 000000000..baa2038b9 --- /dev/null +++ b/fetcher/sources/ip89.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： ip89.py + Description : 89免费代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class Ip89Fetcher(BaseFetcher): + """89免费代理 https://www.89ip.cn/""" + + name = "ip89" + url = "https://www.89ip.cn/" + + def fetch(self): + r = WebRequest().get("https://www.89ip.cn/index_1.html", timeout=10) + proxies = re.findall( + r'[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?[\s\S]*?[\s\S]*?(\d+)[\s\S]*?', + r.text) + for proxy in proxies: + yield ':'.join(proxy) + + +if __name__ == '__main__': + for proxy in Ip89Fetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/fetcher/sources/kuaidaili.py b/fetcher/sources/kuaidaili.py new file mode 100644 index 000000000..3489d7ecf --- /dev/null +++ b/fetcher/sources/kuaidaili.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： kuaidaili.py + Description : 快代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from time import sleep + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class KuaidailiFetcher(BaseFetcher): + """快代理 https://www.kuaidaili.com""" + + name = "kuaidaili" + url = "https://www.kuaidaili.com" + + def fetch(self, page_count=1): + url_pattern = [ + 'https://www.kuaidaili.com/free/inha/{}/', + 'https://www.kuaidaili.com/free/intr/{}/', + ] + url_list = [] + for page_index in range(1, page_count + 1): + for pattern in url_pattern: + url_list.append(pattern.format(page_index)) + + for url in url_list: + tree = WebRequest().get(url).tree + proxy_list = tree.xpath('.//table//tr') + sleep(1) # 必须sleep 不然第二条请求不到数据 + for tr in proxy_list[1:]: + yield ':'.join(tr.xpath('./td/text()')[0:2]) + + +if __name__ == '__main__': + for proxy in KuaidailiFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/kxdaili.py b/fetcher/sources/kxdaili.py new file mode 100644 index 000000000..5d93786d2 --- /dev/null +++ b/fetcher/sources/kxdaili.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： kxdaili.py + Description : 开心代理代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class KxdailiFetcher(BaseFetcher): + """开心代理 http://www.kxdaili.com/""" + + name = "kxdaili" + url = "http://www.kxdaili.com/dailiip.html" + + def fetch(self): + target_urls = [ + "http://www.kxdaili.com/dailiip.html", + "http://www.kxdaili.com/dailiip/2/1.html", + ] + for url in target_urls: + tree = WebRequest().get(url).tree + for tr in tree.xpath("//table[@class='active']//tr")[1:]: + ip = "".join(tr.xpath('./td[1]/text()')).strip() + port = "".join(tr.xpath('./td[2]/text()')).strip() + yield "%s:%s" % (ip, port) + + +if __name__ == '__main__': + for proxy in KxdailiFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/proxifly.py b/fetcher/sources/proxifly.py new file mode 100644 index 000000000..4f8780ba0 --- /dev/null +++ b/fetcher/sources/proxifly.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： proxifly.py + Description : Proxifly代理源 + Author : JHao + date： 2026/06/01 +------------------------------------------------- + Change Activity: + 2026/06/01: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class ProxiFlyFetcher(BaseFetcher): + """Proxifly https://proxifly.dev""" + + name = "proxifly" + url = "https://proxifly.dev/" + + enabled = True # 是否启用 + + def fetch(self): + r = WebRequest().get("https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json", timeout=10) + try: + for each in r.json: + if each.get("geolocation", {}).get("country", "") == "CN" and each.get("protocol") == "http": + yield self.parseProxiesFromText(each.get('proxy', "")).pop() + except Exception as e: + logger.error("ProxyFetch - proxifly: %s" % e) + + +if __name__ == '__main__': + for proxy in ProxiFlyFetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/fetcher/sources/roundproxies.py b/fetcher/sources/roundproxies.py new file mode 100644 index 000000000..ba1d0af36 --- /dev/null +++ b/fetcher/sources/roundproxies.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： roundproxies.py + Description : Roundproxies + Author : JHao + date： 2026/06/09 +------------------------------------------------- + Change Activity: + 2026/06/09: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + +class RoundProxiesFetcher(BaseFetcher): + """Roundproxies https://roundproxies.com/free-proxy-list""" + + name = "roundproxies" + url = "https://roundproxies.com/free-proxy-list" + + enabled = True + + def fetch(self): + page_size = 50 + _url = f"https://roundproxies.com/api/get-free-proxies/?limit={page_size}&page=1&sort_by=lastChecked&sort_type=desc" + r = WebRequest().get(_url, timeout=10) + try: + for each in r.json.get("data", []): + yield "%s:%s" % (each["ip"], each["port"]) + except Exception as e: + logger.error("ProxyFetch - roundproxies: %s" % e) + + + +if __name__ == '__main__': + for proxy in RoundProxiesFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/scdn.py b/fetcher/sources/scdn.py new file mode 100644 index 000000000..3d4ca9c29 --- /dev/null +++ b/fetcher/sources/scdn.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： scdn.py + Description : SCDN代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re + +from lxml import etree + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class ScdnFetcher(BaseFetcher): + """SCDN 代理接口 https://proxy.scdn.io/""" + + name = "scdn" + url = "https://proxy.scdn.io/" + + def fetch(self): + url = ("https://proxy.scdn.io/get_proxies.php?" + "protocol=&country=&per_page=100&page=1") + r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) + try: + data = r.json + proxies = [] + table_html = data.get("table_html") if isinstance(data, dict) else "" + if table_html: + tree = etree.HTML("%s
" % table_html) + for tr in tree.xpath("//tr"): + cells = [" ".join(td.xpath(".//text()")).strip() for td in tr.xpath("./td")] + if len(cells) >= 2: + ip_match = re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', cells[0]) + port_match = re.match(r'^\d{2,5}$', cells[1]) + if ip_match and port_match: + proxies.append("%s:%s" % (cells[0], cells[1])) + + if not proxies: + items = data.get("data", []) if isinstance(data, dict) else [] + for item in items: + ip = item.get("ip", "") + port = item.get("port", "") + if ip and port: + proxies.append("%s:%s" % (ip, port)) + if not proxies: + proxies = self.parseProxiesFromText(r.text) + for proxy in self.yieldUniqueProxies(proxies): + yield proxy + except Exception as e: + logger.error("ProxyFetch - scdn: %s" % e) + + +if __name__ == '__main__': + for proxy in ScdnFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/zdaye.py b/fetcher/sources/zdaye.py new file mode 100644 index 000000000..8eeda8b79 --- /dev/null +++ b/fetcher/sources/zdaye.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： zdaye.py + Description : 站大爷代理源 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from time import sleep +from datetime import datetime + +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest + + +class ZdayeFetcher(BaseFetcher): + """站大爷 https://www.zdaye.com/dayProxy.html""" + + name = "zdaye" + url = "https://www.zdaye.com/dayProxy.html" + + def fetch(self): + start_url = "https://www.zdaye.com/free/" + html_tree = WebRequest().get(start_url, verify=False).tree + latest_page_time = html_tree.xpath( + "//span[@class='thread_time_info']/text()")[0].strip() + interval = datetime.now() - datetime.strptime( + latest_page_time, "%Y/%m/%d %H:%M:%S") + if interval.total_seconds() < 300: + target_url = ("https://www.zdaye.com/" + + html_tree.xpath("//h3[@class='thread_title']/a/@href")[0].strip()) + while target_url: + _tree = WebRequest().get(target_url, verify=False).tree + for tr in _tree.xpath("//table//tr"): + ip = "".join(tr.xpath("./td[1]/text()")).strip() + port = "".join(tr.xpath("./td[2]/text()")).strip() + yield "%s:%s" % (ip, port) + next_page = _tree.xpath( + "//div[@class='page']/a[@title='下一页']/@href") + target_url = ("https://www.zdaye.com/" + next_page[0].strip() + if next_page else False) + sleep(5) + + +if __name__ == '__main__': + for proxy in ZdayeFetcher().fetch(): + print(proxy) diff --git a/Manager/__init__.py b/handler/__init__.py similarity index 75% rename from Manager/__init__.py rename to handler/__init__.py index e94e59d11..9a42cea96 100644 --- a/Manager/__init__.py +++ b/handler/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------- - File Name： __init__.py.py + File Name： __init__.py Description : Author : JHao date： 2016/12/3 @@ -10,4 +10,6 @@ 2016/12/3: ------------------------------------------------- """ -__author__ = 'JHao' \ No newline at end of file +__author__ = 'JHao' + +# from handler.ProxyManager import ProxyManager diff --git a/handler/configHandler.py b/handler/configHandler.py new file mode 100644 index 000000000..37a923493 --- /dev/null +++ b/handler/configHandler.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： configHandler + Description : + Author : JHao + date： 2020/6/22 +------------------------------------------------- + Change Activity: + 2020/6/22: +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import setting +from util.singleton import Singleton +from util.lazyProperty import LazyProperty +from util.six import reload_six, withMetaclass + + +class ConfigHandler(withMetaclass(Singleton)): + + def __init__(self): + pass + + @LazyProperty + def serverHost(self): + return os.environ.get("HOST", setting.HOST) + + @LazyProperty + def serverPort(self): + return os.environ.get("PORT", setting.PORT) + + @LazyProperty + def dbConn(self): + return os.getenv("DB_CONN", setting.DB_CONN) + + @LazyProperty + def tableName(self): + return os.getenv("TABLE_NAME", setting.TABLE_NAME) + + @property + def fetcherExclude(self): + reload_six(setting) + return getattr(setting, 'PROXY_FETCHER_EXCLUDE', []) + + @LazyProperty + def httpUrl(self): + return os.getenv("HTTP_URL", setting.HTTP_URL) + + @LazyProperty + def httpsUrl(self): + return os.getenv("HTTPS_URL", setting.HTTPS_URL) + + @LazyProperty + def verifyTimeout(self): + return int(os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)) + + # @LazyProperty + # def proxyCheckCount(self): + # return int(os.getenv("PROXY_CHECK_COUNT", setting.PROXY_CHECK_COUNT)) + + @LazyProperty + def maxFailCount(self): + return int(os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT)) + + # @LazyProperty + # def maxFailRate(self): + # return int(os.getenv("MAX_FAIL_RATE", setting.MAX_FAIL_RATE)) + + @LazyProperty + def poolSizeMin(self): + return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN)) + + @LazyProperty + def proxyRegion(self): + return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION)) + + @LazyProperty + def timezone(self): + return os.getenv("TIMEZONE", setting.TIMEZONE) + diff --git a/Util/LogHandler.py b/handler/logHandler.py similarity index 84% rename from Util/LogHandler.py rename to handler/logHandler.py index 6e7341c1b..45cd1201d 100644 --- a/Util/LogHandler.py +++ b/handler/logHandler.py @@ -7,15 +7,16 @@ date： 2017/3/6 ------------------------------------------------- Change Activity: - 2017/3/6: log handler - 2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出) + 2017/03/06: log handler + 2017/09/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出) + 2020/07/13: Windows下TimedRotatingFileHandler线程不安全, 不再使用 ------------------------------------------------- """ __author__ = 'JHao' import os - import logging +import platform from logging.handlers import TimedRotatingFileHandler @@ -33,6 +34,12 @@ ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir) LOG_PATH = os.path.join(ROOT_PATH, 'log') +if not os.path.exists(LOG_PATH): + try: + os.mkdir(LOG_PATH) + except FileExistsError: + pass + class LogHandler(logging.Logger): """ @@ -46,7 +53,8 @@ def __init__(self, name, level=DEBUG, stream=True, file=True): if stream: self.__setStreamHandler__() if file: - self.__setFileHandler__() + if platform.system() != "Windows": + self.__setFileHandler__() def __setFileHandler__(self, level=None): """ @@ -83,16 +91,6 @@ def __setStreamHandler__(self, level=None): stream_handler.setLevel(level) self.addHandler(stream_handler) - def resetName(self, name): - """ - reset name - :param name: - :return: - """ - self.name = name - self.removeHandler(self.file_handler) - self.__setFileHandler__() - if __name__ == '__main__': log = LogHandler('test') diff --git a/handler/proxyHandler.py b/handler/proxyHandler.py new file mode 100644 index 000000000..32e215e5d --- /dev/null +++ b/handler/proxyHandler.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： ProxyHandler.py + Description : + Author : JHao + date： 2016/12/3 +------------------------------------------------- + Change Activity: + 2016/12/03: + 2020/05/26: 区分http和https +------------------------------------------------- +""" +__author__ = 'JHao' + +from helper.proxy import Proxy +from db.dbClient import DbClient +from handler.configHandler import ConfigHandler + + +class ProxyHandler(object): + """ Proxy CRUD operator""" + + def __init__(self): + self.conf = ConfigHandler() + self.db = DbClient(self.conf.dbConn) + self.db.changeTable(self.conf.tableName) + + def get(self, https=False): + """ + return a proxy + Args: + https: True/False + Returns: + """ + proxy = self.db.get(https) + return Proxy.createFromJson(proxy) if proxy else None + + def pop(self, https): + """ + return and delete a useful proxy + :return: + """ + proxy = self.db.pop(https) + if proxy: + return Proxy.createFromJson(proxy) + return None + + def put(self, proxy): + """ + put proxy into use proxy + :return: + """ + self.db.put(proxy) + + def delete(self, proxy): + """ + delete useful proxy + :param proxy: + :return: + """ + return self.db.delete(proxy.proxy) + + def getAll(self, https=False): + """ + get all proxy from pool as Proxy list + :return: + """ + proxies = self.db.getAll(https) + return [Proxy.createFromJson(_) for _ in proxies] + + def exists(self, proxy): + """ + check proxy exists + :param proxy: + :return: + """ + return self.db.exists(proxy.proxy) + + def getCount(self): + """ + return raw_proxy and use_proxy count + :return: + """ + total_use_proxy = self.db.getCount() + return {'count': total_use_proxy} diff --git a/log/__init__.py b/helper/__init__.py similarity index 100% rename from log/__init__.py rename to helper/__init__.py diff --git a/helper/check.py b/helper/check.py new file mode 100644 index 000000000..0b732b84d --- /dev/null +++ b/helper/check.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： check + Description : 执行代理校验 + Author : JHao + date： 2019/8/6 +------------------------------------------------- + Change Activity: + 2019/08/06: 执行代理校验 + 2021/05/25: 分别校验http和https + 2022/08/16: 获取代理Region信息 +------------------------------------------------- +""" +__author__ = 'JHao' + +from util.six import Empty +from threading import Thread +from datetime import datetime +from util.webRequest import WebRequest +from handler.logHandler import LogHandler +from helper.validator import ProxyValidator +from handler.proxyHandler import ProxyHandler +from handler.configHandler import ConfigHandler + + +class DoValidator(object): + """ 执行校验 """ + + conf = ConfigHandler() + + @classmethod + def validator(cls, proxy, work_type): + """ + 校验入口 + Args: + proxy: Proxy Object + work_type: raw/use + Returns: + Proxy Object + """ + http_r = cls.httpValidator(proxy) + https_r = False if not http_r else cls.httpsValidator(proxy) + + proxy.check_count += 1 + proxy.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + proxy.last_status = True if http_r else False + if http_r: + if proxy.fail_count > 0: + proxy.fail_count -= 1 + proxy.https = True if https_r else False + if work_type == "raw": + proxy.region = cls.regionGetter(proxy) if cls.conf.proxyRegion else "" + else: + proxy.fail_count += 1 + return proxy + + @classmethod + def httpValidator(cls, proxy): + for func in ProxyValidator.http_validator: + if not func(proxy.proxy): + return False + return True + + @classmethod + def httpsValidator(cls, proxy): + for func in ProxyValidator.https_validator: + if not func(proxy.proxy): + return False + return True + + @classmethod + def preValidator(cls, proxy): + for func in ProxyValidator.pre_validator: + if not func(proxy): + return False + return True + + @classmethod + def regionGetter(cls, proxy): + try: + url = 'https://api.ip.sb/geoip/%s' % proxy.proxy.split(':')[0] + r = WebRequest().get(url=url, retry_time=1, timeout=2).json + return r.get('country_code') + except: + return 'error' + + +class _ThreadChecker(Thread): + """ 多线程检测 """ + + def __init__(self, work_type, target_queue, thread_name): + Thread.__init__(self, name=thread_name) + self.work_type = work_type + self.log = LogHandler("checker") + self.proxy_handler = ProxyHandler() + self.target_queue = target_queue + self.conf = ConfigHandler() + + def run(self): + self.log.info("{}ProxyCheck - {}: start".format(self.work_type.title(), self.name)) + while True: + try: + proxy = self.target_queue.get(block=False) + except Empty: + self.log.info("{}ProxyCheck - {}: complete".format(self.work_type.title(), self.name)) + break + proxy = DoValidator.validator(proxy, self.work_type) + if self.work_type == "raw": + self.__ifRaw(proxy) + else: + self.__ifUse(proxy) + self.target_queue.task_done() + + def __ifRaw(self, proxy): + if proxy.last_status: + if self.proxy_handler.exists(proxy): + self.log.info('RawProxyCheck - {}: {} exist'.format(self.name, proxy.proxy.ljust(23))) + else: + self.log.info('RawProxyCheck - {}: {} pass'.format(self.name, proxy.proxy.ljust(23))) + self.proxy_handler.put(proxy) + else: + self.log.info('RawProxyCheck - {}: {} fail'.format(self.name, proxy.proxy.ljust(23))) + + def __ifUse(self, proxy): + if proxy.last_status: + self.log.info('UseProxyCheck - {}: {} pass'.format(self.name, proxy.proxy.ljust(23))) + self.proxy_handler.put(proxy) + else: + if proxy.fail_count > self.conf.maxFailCount: + self.log.info('UseProxyCheck - {}: {} fail, count {} delete'.format(self.name, + proxy.proxy.ljust(23), + proxy.fail_count)) + self.proxy_handler.delete(proxy) + else: + self.log.info('UseProxyCheck - {}: {} fail, count {} keep'.format(self.name, + proxy.proxy.ljust(23), + proxy.fail_count)) + self.proxy_handler.put(proxy) + + +def Checker(tp, queue): + """ + run Proxy ThreadChecker + :param tp: raw/use + :param queue: Proxy Queue + :return: + """ + thread_list = list() + for index in range(20): + thread_list.append(_ThreadChecker(tp, queue, "thread_%s" % str(index).zfill(2))) + + for thread in thread_list: + thread.setDaemon(True) + thread.start() + + for thread in thread_list: + thread.join() diff --git a/helper/fetch.py b/helper/fetch.py new file mode 100644 index 000000000..62970685a --- /dev/null +++ b/helper/fetch.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： fetch.py + Description : 代理采集 + Author : JHao + date： 2019/8/6 +------------------------------------------------- + Change Activity: + 2019/08/06: 多线程采集 + 2026/05/31: 重构为动态加载 fetcher 插件 +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import sys +import importlib +from threading import Thread + +from helper.proxy import Proxy +from helper.check import DoValidator +from handler.logHandler import LogHandler +from handler.configHandler import ConfigHandler +from fetcher.baseFetcher import BaseFetcher + +_logger = LogHandler("fetch") + +# 模块缓存: {module_name: (mtime, module)} +_module_cache = {} + + +def _get_sources_dir(): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), '..', 'fetcher', 'sources') + + +def _load_module(module_name, filepath): + """加载或 reload 模块，仅在文件 mtime 变化时 reload""" + global _module_cache + mtime = os.path.getmtime(filepath) + cached = _module_cache.get(module_name) + if cached and cached[0] == mtime: + return cached[1] + try: + if module_name in sys.modules: + module = importlib.reload(sys.modules[module_name]) + else: + module = importlib.import_module(module_name) + _module_cache[module_name] = (mtime, module) + return module + except Exception as e: + _logger.warning("ProxyFetch : load %s error - %s" % (module_name, e)) + return None + + +def _discover_fetchers(exclude_list): + """ + 自动扫描 sources/ 目录，返回所有 enabled=True 且不在黑名单中的 fetcher 类列表。 + 仅在文件 mtime 变化时重新加载模块，支持运行时热更新。 + """ + global _module_cache + sources_dir = _get_sources_dir() + fetcher_classes = [] + seen_modules = set() + + for filename in os.listdir(sources_dir): + if not filename.endswith('.py') or filename.startswith('_'): + continue + module_name = "fetcher.sources.%s" % filename[:-3] + seen_modules.add(module_name) + filepath = os.path.join(sources_dir, filename) + module = _load_module(module_name, filepath) + if module is None: + continue + for attr_name in dir(module): + attr = getattr(module, attr_name, None) + if (attr and isinstance(attr, type) + and issubclass(attr, BaseFetcher) + and attr is not BaseFetcher + and attr.name + and attr.enabled + and attr.__name__ not in exclude_list): + fetcher_classes.append(attr) + + # 清理已删除文件的缓存 + for name in list(_module_cache): + if name not in seen_modules: + del _module_cache[name] + + return sorted(fetcher_classes, key=lambda c: c.name) + + +class _ThreadFetcher(Thread): + + def __init__(self, fetcher_class, proxy_dict): + Thread.__init__(self) + self.fetcher_class = fetcher_class + self.proxy_dict = proxy_dict + self.log = LogHandler("fetcher") + + def run(self): + fetcher_name = self.fetcher_class.name + self.log.info("ProxyFetch - {func}: start".format(func=fetcher_name)) + try: + for proxy in self.fetcher_class().fetch(): + self.log.info('ProxyFetch - %s: %s ok' % (fetcher_name, proxy.ljust(23))) + proxy = proxy.strip() + if proxy in self.proxy_dict: + self.proxy_dict[proxy].add_source(fetcher_name) + else: + self.proxy_dict[proxy] = Proxy( + proxy, source=fetcher_name) + except Exception as e: + self.log.error("ProxyFetch - {func}: error".format(func=fetcher_name)) + self.log.error(str(e)) + + +class Fetcher(object): + name = "fetcher" + + def __init__(self): + self.log = LogHandler(self.name) + self.conf = ConfigHandler() + + def run(self): + """ + fetch proxy with fetcher plugins + :return: + """ + proxy_dict = dict() + thread_list = list() + self.log.info("ProxyFetch : start") + + exclude_list = self.conf.fetcherExclude + fetcher_classes = _discover_fetchers(exclude_list) + self.log.info("ProxyFetch : active fetchers [%s]" % ", ".join(c.name for c in fetcher_classes)) + + for fetcher_class in fetcher_classes: + thread_list.append(_ThreadFetcher(fetcher_class, proxy_dict)) + + for thread in thread_list: + thread.setDaemon(True) + thread.start() + + for thread in thread_list: + thread.join() + + self.log.info("ProxyFetch - all complete!") + for _ in proxy_dict.values(): + if DoValidator.preValidator(_.proxy): + yield _ diff --git a/helper/launcher.py b/helper/launcher.py new file mode 100644 index 000000000..1f336644f --- /dev/null +++ b/helper/launcher.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： launcher + Description : 启动器 + Author : JHao + date： 2021/3/26 +------------------------------------------------- + Change Activity: + 2021/3/26: 启动器 +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys +from db.dbClient import DbClient +from handler.logHandler import LogHandler +from handler.configHandler import ConfigHandler + +log = LogHandler('launcher') + + +def startServer(): + __beforeStart() + from api.proxyApi import runFlask + runFlask() + + +def startScheduler(): + __beforeStart() + from helper.scheduler import runScheduler + runScheduler() + + +def __beforeStart(): + __showVersion() + __showConfigure() + if __checkDBConfig(): + log.info('exit!') + sys.exit() + + +def __showVersion(): + from setting import VERSION + log.info("ProxyPool Version: %s" % VERSION) + + +def __showConfigure(): + conf = ConfigHandler() + log.info("ProxyPool configure HOST: %s" % conf.serverHost) + log.info("ProxyPool configure PORT: %s" % conf.serverPort) + exclude = conf.fetcherExclude + if exclude: + log.info("ProxyPool configure PROXY_FETCHER_EXCLUDE: %s" % exclude) + log.info("ProxyPool configure PROXY_FETCHER: auto-scan (enabled=True, exclude=%s)" % exclude) + + +def __checkDBConfig(): + conf = ConfigHandler() + db = DbClient(conf.dbConn) + log.info("============ DATABASE CONFIGURE ================") + log.info("DB_TYPE: %s" % db.db_type) + log.info("DB_HOST: %s" % db.db_host) + log.info("DB_PORT: %s" % db.db_port) + log.info("DB_NAME: %s" % db.db_name) + log.info("DB_USER: %s" % db.db_user) + log.info("=================================================") + return db.test() diff --git a/helper/proxy.py b/helper/proxy.py new file mode 100644 index 000000000..396a84239 --- /dev/null +++ b/helper/proxy.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： Proxy + Description : 代理对象类型封装 + Author : JHao + date： 2019/7/11 +------------------------------------------------- + Change Activity: + 2019/7/11: 代理对象类型封装 +------------------------------------------------- +""" +__author__ = 'JHao' + +import json + + +class Proxy(object): + + def __init__(self, proxy, fail_count=0, region="", anonymous="", + source="", check_count=0, last_status="", last_time="", https=False): + self._proxy = proxy + self._fail_count = fail_count + self._region = region + self._anonymous = anonymous + self._source = source.split('/') + self._check_count = check_count + self._last_status = last_status + self._last_time = last_time + self._https = https + + @classmethod + def createFromJson(cls, proxy_json): + _dict = json.loads(proxy_json) + return cls(proxy=_dict.get("proxy", ""), + fail_count=_dict.get("fail_count", 0), + region=_dict.get("region", ""), + anonymous=_dict.get("anonymous", ""), + source=_dict.get("source", ""), + check_count=_dict.get("check_count", 0), + last_status=_dict.get("last_status", ""), + last_time=_dict.get("last_time", ""), + https=_dict.get("https", False) + ) + + @property + def proxy(self): + """ 代理 ip:port """ + return self._proxy + + @property + def fail_count(self): + """ 检测失败次数 """ + return self._fail_count + + @property + def region(self): + """ 地理位置(国家/城市) """ + return self._region + + @property + def anonymous(self): + """ 匿名 """ + return self._anonymous + + @property + def source(self): + """ 代理来源 """ + return '/'.join(self._source) + + @property + def check_count(self): + """ 代理检测次数 """ + return self._check_count + + @property + def last_status(self): + """ 最后一次检测结果 True -> 可用; False -> 不可用""" + return self._last_status + + @property + def last_time(self): + """ 最后一次检测时间 """ + return self._last_time + + @property + def https(self): + """ 是否支持https """ + return self._https + + @property + def to_dict(self): + """ 属性字典 """ + return {"proxy": self.proxy, + "https": self.https, + "fail_count": self.fail_count, + "region": self.region, + "anonymous": self.anonymous, + "source": self.source, + "check_count": self.check_count, + "last_status": self.last_status, + "last_time": self.last_time} + + @property + def to_json(self): + """ 属性json格式 """ + return json.dumps(self.to_dict, ensure_ascii=False) + + @fail_count.setter + def fail_count(self, value): + self._fail_count = value + + @check_count.setter + def check_count(self, value): + self._check_count = value + + @last_status.setter + def last_status(self, value): + self._last_status = value + + @last_time.setter + def last_time(self, value): + self._last_time = value + + @https.setter + def https(self, value): + self._https = value + + @region.setter + def region(self, value): + self._region = value + + def add_source(self, source_str): + if source_str: + self._source.append(source_str) + self._source = list(set(self._source)) diff --git a/helper/scheduler.py b/helper/scheduler.py new file mode 100644 index 000000000..ef0431881 --- /dev/null +++ b/helper/scheduler.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： proxyScheduler + Description : + Author : JHao + date： 2019/8/5 +------------------------------------------------- + Change Activity: + 2019/08/05: proxyScheduler + 2021/02/23: runProxyCheck时,剩余代理少于POOL_SIZE_MIN时执行抓取 +------------------------------------------------- +""" +__author__ = 'JHao' + +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.executors.pool import ProcessPoolExecutor + +from util.six import Queue +from helper.fetch import Fetcher +from helper.check import Checker +from handler.logHandler import LogHandler +from handler.proxyHandler import ProxyHandler +from handler.configHandler import ConfigHandler + + +def __runProxyFetch(): + proxy_queue = Queue() + proxy_fetcher = Fetcher() + + for proxy in proxy_fetcher.run(): + proxy_queue.put(proxy) + + Checker("raw", proxy_queue) + + +def __runProxyCheck(): + proxy_handler = ProxyHandler() + proxy_queue = Queue() + if proxy_handler.db.getCount().get("total", 0) < proxy_handler.conf.poolSizeMin: + __runProxyFetch() + for proxy in proxy_handler.getAll(): + proxy_queue.put(proxy) + Checker("use", proxy_queue) + + +def runScheduler(): + __runProxyFetch() + + timezone = ConfigHandler().timezone + scheduler_log = LogHandler("scheduler") + scheduler = BlockingScheduler(logger=scheduler_log, timezone=timezone) + + scheduler.add_job(__runProxyFetch, 'interval', minutes=5, id="proxy_fetch", name="proxy采集") + scheduler.add_job(__runProxyCheck, 'interval', minutes=2, id="proxy_check", name="proxy检查") + executors = { + 'default': {'type': 'threadpool', 'max_workers': 20}, + 'processpool': ProcessPoolExecutor(max_workers=5) + } + job_defaults = { + 'coalesce': False, + 'max_instances': 10 + } + + scheduler.configure(executors=executors, job_defaults=job_defaults, timezone=timezone) + + scheduler.start() + + +if __name__ == '__main__': + runScheduler() diff --git a/helper/validator.py b/helper/validator.py new file mode 100644 index 000000000..136691c2e --- /dev/null +++ b/helper/validator.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： _validators + Description : 定义proxy验证方法 + Author : JHao + date： 2021/5/25 +------------------------------------------------- + Change Activity: + 2023/03/10: 支持带用户认证的代理格式 username:password@ip:port +------------------------------------------------- +""" +__author__ = 'JHao' + +import re +from requests import head +from util.six import withMetaclass +from util.singleton import Singleton +from handler.configHandler import ConfigHandler + +conf = ConfigHandler() + +HEADER = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', + 'Accept': '*/*', + 'Connection': 'keep-alive', + 'Accept-Language': 'zh-CN,zh;q=0.8'} + +IP_REGEX = re.compile(r"(.*:.*@)?\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}") + + +class ProxyValidator(withMetaclass(Singleton)): + pre_validator = [] + http_validator = [] + https_validator = [] + + @classmethod + def addPreValidator(cls, func): + cls.pre_validator.append(func) + return func + + @classmethod + def addHttpValidator(cls, func): + cls.http_validator.append(func) + return func + + @classmethod + def addHttpsValidator(cls, func): + cls.https_validator.append(func) + return func + + +@ProxyValidator.addPreValidator +def formatValidator(proxy): + """检查代理格式""" + return True if IP_REGEX.fullmatch(proxy) else False + + +@ProxyValidator.addHttpValidator +def httpTimeOutValidator(proxy): + """ http检测超时 """ + + proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)} + + try: + r = head(conf.httpUrl, headers=HEADER, proxies=proxies, timeout=conf.verifyTimeout) + return True if r.status_code == 200 else False + except Exception as e: + return False + + +@ProxyValidator.addHttpsValidator +def httpsTimeOutValidator(proxy): + """https检测超时""" + + proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)} + try: + r = head(conf.httpsUrl, headers=HEADER, proxies=proxies, timeout=conf.verifyTimeout, verify=False) + return True if r.status_code == 200 else False + except Exception as e: + return False + + +@ProxyValidator.addHttpValidator +def customValidatorExample(proxy): + """自定义validator函数，校验代理是否可用, 返回True/False""" + return True diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..60886afe6 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,93 @@ +site_name: ProxyPool +site_description: Python爬虫代理IP池 +site_author: jhao104 +site_url: https://jhao104.github.io/proxy_pool/ + +repo_name: jhao104/proxy_pool +repo_url: https://github.com/jhao104/proxy_pool + +theme: + name: material + language: zh + logo: assets/logo.svg + favicon: assets/logo.svg + icon: + repo: fontawesome/brands/github + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: light-blue + toggle: + icon: material/brightness-7 + name: 切换到暗色模式 + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: light-blue + toggle: + icon: material/brightness-4 + name: 切换到亮色模式 + features: + - navigation.instant + - navigation.instant.progress + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.top + - navigation.tracking + - search.suggest + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - content.tabs.link + - toc.follow + font: + text: Noto Sans SC + code: JetBrains Mono + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.tabbed: + alternate_style: true + - pymdownx.snippets + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - tables + - attr_list + - def_list + - md_in_html + - toc: + permalink: true + +extra_css: + - stylesheets/extra.css + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/jhao104/proxy_pool + generator: false + +plugins: + - search + +nav: + - 首页: index.md + - 用户指南: + - 快速开始: getting-started.md + - 项目结构: project-structure.md + - 配置参考: configuration.md + - API 使用: api.md + - Docker 部署: docker.md + - 开发指南: + - 扩展代理源: extending/fetcher.md + - 扩展校验器: extending/validator.md + - 变更日志: changelog.md \ No newline at end of file diff --git a/proxyPool.py b/proxyPool.py new file mode 100644 index 000000000..9b18b7845 --- /dev/null +++ b/proxyPool.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： proxy_pool + Description : proxy pool 启动入口 + Author : JHao + date： 2020/6/19 +------------------------------------------------- + Change Activity: + 2020/6/19: +------------------------------------------------- +""" +__author__ = 'JHao' + +import click +from helper.launcher import startServer, startScheduler +from setting import BANNER, VERSION + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.version_option(version=VERSION) +def cli(): + """ProxyPool cli工具""" + + +@cli.command(name="schedule") +def schedule(): + """ 启动调度程序 """ + click.echo(BANNER) + startScheduler() + + +@cli.command(name="server") +def server(): + """ 启动api服务 """ + click.echo(BANNER) + startServer() + + +@cli.command(name="fetcher") +def fetcher(): + """ 查看启用的代理源 """ + from helper.fetch import _discover_fetchers + from handler.configHandler import ConfigHandler + conf = ConfigHandler() + exclude = conf.fetcherExclude + fetcher_classes = _discover_fetchers(exclude) + click.echo("Active fetchers (%d):" % len(fetcher_classes)) + for cls in fetcher_classes: + click.echo(" - %s" % cls.name) + if exclude: + click.echo("\nExcluded: %s" % ", ".join(exclude)) + + +if __name__ == '__main__': + cli() diff --git a/proxy_pool.sh b/proxy_pool.sh new file mode 100644 index 000000000..cae60196c --- /dev/null +++ b/proxy_pool.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PID_FILE="$SCRIPT_DIR/proxy_pool.pid" +PYTHON="${PYTHON:-python}" + +# 颜色 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# 获取已启动的 PIDs +get_pids() { + if [ -f "$PID_FILE" ]; then + cat "$PID_FILE" + fi +} + +# 检查进程是否存活 +is_running() { + local pid=$1 + kill -0 "$pid" 2>/dev/null +} + +# 启动服务 +cmd_start() { + local foreground=false + + while [[ $# -gt 0 ]]; do + case $1 in + --fg|--foreground) foreground=true; shift ;; + *) log_error "Unknown option: $1"; exit 1 ;; + esac + done + + # 检查是否已运行 + local pids=$(get_pids) + if [ -n "$pids" ]; then + for pid in $pids; do + if is_running "$pid"; then + log_warn "Service already running (PID: $pid)" + log_warn "Use '$0 stop' first, or '$0 restart'" + exit 1 + fi + done + fi + + # 清理旧的 PID 文件 + rm -f "$PID_FILE" + + cd "$SCRIPT_DIR" + + if [ "$foreground" = true ]; then + # 前台模式（容器环境） + log_info "Starting in foreground mode..." + + trap 'log_info "Shutting down..."; kill $SERVER_PID $SCHEDULER_PID 2>/dev/null; wait; rm -f "$PID_FILE"; exit 0' EXIT INT TERM + + $PYTHON proxyPool.py server & + SERVER_PID=$! + + $PYTHON proxyPool.py schedule & + SCHEDULER_PID=$! + + echo "$SERVER_PID" >> "$PID_FILE" + echo "$SCHEDULER_PID" >> "$PID_FILE" + + log_info "Services started (PIDs: $SERVER_PID $SCHEDULER_PID)" + wait + else + # 后台模式（非容器环境） + log_info "Starting in background mode..." + + nohup $PYTHON proxyPool.py server > /dev/null 2>&1 & + SERVER_PID=$! + + nohup $PYTHON proxyPool.py schedule > /dev/null 2>&1 & + SCHEDULER_PID=$! + + echo "$SERVER_PID" >> "$PID_FILE" + echo "$SCHEDULER_PID" >> "$PID_FILE" + + sleep 2 + + # 验证启动 + local failed=false + if ! is_running "$SERVER_PID"; then + log_error "Server failed to start" + failed=true + fi + if ! is_running "$SCHEDULER_PID"; then + log_error "Scheduler failed to start" + failed=true + fi + + if [ "$failed" = true ]; then + cmd_stop + exit 1 + fi + + log_info "Services started" + log_info " Server PID: $SERVER_PID" + log_info " Scheduler PID: $SCHEDULER_PID" + log_info "Use '$0 stop' to stop, '$0 status' to check" + fi +} + +# 停止服务 +cmd_stop() { + local pids=$(get_pids) + + if [ -z "$pids" ]; then + log_warn "No PID file found. Services may not be running." + exit 0 + fi + + log_info "Stopping services..." + + local stopped=0 + for pid in $pids; do + if is_running "$pid"; then + kill "$pid" 2>/dev/null || true + stopped=$((stopped + 1)) + fi + done + + # 等待进程退出 + sleep 1 + + # 强制杀死仍在运行的进程 + for pid in $pids; do + if is_running "$pid"; then + log_warn "Force killing PID $pid" + kill -9 "$pid" 2>/dev/null || true + fi + done + + rm -f "$PID_FILE" + log_info "Stopped $stopped service(s)" +} + +# 重启服务 +cmd_restart() { + cmd_stop + sleep 1 + cmd_start "$@" +} + +# 查看状态 +cmd_status() { + local pids=$(get_pids) + + if [ -z "$pids" ]; then + log_info "No PID file found. Services are not running." + exit 0 + fi + + local running=0 + local dead=0 + + for pid in $pids; do + if is_running "$pid"; then + running=$((running + 1)) + else + dead=$((dead + 1)) + fi + done + + if [ $running -gt 0 ]; then + log_info "Services: $running running, $dead dead" + for pid in $pids; do + local status="stopped" + if is_running "$pid"; then + status="running" + fi + echo " PID $pid: $status" + done + else + log_warn "All services are stopped" + rm -f "$PID_FILE" + fi +} + +# 显示帮助 +cmd_help() { + cat < [options] + +Commands: + start [--fg] Start services (background by default) + --fg Run in foreground (for containers) + stop Stop all services + restart [--fg] Restart services + status Show service status + help Show this help + +Examples: + $0 start # Start in background + $0 start --fg # Start in foreground (containers) + $0 stop # Stop all services + $0 status # Check status + +Environment: + PYTHON Python executable (default: python) +EOF +} + +# 主入口 +case "${1:-help}" in + start) shift; cmd_start "$@" ;; + stop) cmd_stop ;; + restart) shift; cmd_restart "$@" ;; + status) cmd_status ;; + help|-h|--help) cmd_help ;; + *) log_error "Unknown command: $1"; cmd_help; exit 1 ;; +esac diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..2931028ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[tool.setuptools] +py-modules = [] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: 需要外部服务（如 Redis）的集成测试", +] + +[tool.coverage.run] +source = ["."] +omit = [ + "tests/*", + "docs/*", + "setup.py", +] + +[tool.coverage.report] +show_missing = true +skip_empty = true \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000..081f84acb --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,6 @@ +pytest>=7.0 +pytest-cov>=4.0 +fakeredis>=2.0,<2.26;python_version<="3.8" +fakeredis>=2.26;python_version>"3.8" +async_timeout>=3.0;python_version<"3.11" +typing_extensions>=4.0;python_version<"3.11" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 698cc8197..c658ae713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -APScheduler==3.2.0 -Flask==0.11.1 -requests==2.11.0 -lxml==3.7.1 - -pymongo==3.2.2 -redis==2.10.5 - - +requests==2.31.0 +gunicorn==19.9.0 +lxml==4.9.2 +redis>=4.2.0 +APScheduler==3.10.0;python_version>="3.10" +APScheduler==3.2.0;python_version<"3.10" +click==8.0.1 +Flask==2.1.1 +werkzeug>=2.0,<2.2 diff --git a/setting.py b/setting.py new file mode 100644 index 000000000..868f32bb7 --- /dev/null +++ b/setting.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： setting.py + Description : 配置文件 + Author : JHao + date： 2019/2/15 +------------------------------------------------- + Change Activity: + 2019/2/15: +------------------------------------------------- +""" + +BANNER = r""" +**************************************************************** +*** ______ ********************* ______ *********** _ ******** +*** | ___ \_ ******************** | ___ \ ********* | | ******** +*** | |_/ / \__ __ __ _ __ _ | |_/ /___ * ___ | | ******** +*** | __/| _// _ \ \ \/ /| | | || __// _ \ / _ \ | | ******** +*** | | | | | (_) | > < \ |_| || | | (_) | (_) || |___ **** +*** \_| |_| \___/ /_/\_\ \__ |\_| \___/ \___/ \_____/ **** +**** __ / / ***** +************************* /___ / ******************************* +************************* ******************************** +**************************************************************** +""" + +VERSION = "2.4.0" + +# ############### server config ############### +HOST = "0.0.0.0" + +PORT = 5010 + +# ############### database config ################### +# db connection uri +# example: +# Redis: redis://:password@ip:port/db +# Ssdb: ssdb://:password@ip:port +DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0' + +# proxy table name +TABLE_NAME = 'use_proxy' + + +# ###### config the proxy fetch function ###### +# 自动扫描 fetcher/sources/ 目录，加载所有 enabled=True 的 fetcher +# 如需临时禁用某个 fetcher，在下方黑名单中添加类名（不改源文件） +PROXY_FETCHER_EXCLUDE = [] + +# ############# proxy validator ################# +# 代理验证目标网站 +HTTP_URL = "http://httpbin.org" + +HTTPS_URL = "https://www.qq.com" + +# 代理验证时超时时间 +VERIFY_TIMEOUT = 10 + +# 近PROXY_CHECK_COUNT次校验中允许的最大失败次数,超过则剔除代理 +MAX_FAIL_COUNT = 0 + +# 近PROXY_CHECK_COUNT次校验中允许的最大失败率,超过则剔除代理 +# MAX_FAIL_RATE = 0.1 + +# proxyCheck时代理数量少于POOL_SIZE_MIN触发抓取 +POOL_SIZE_MIN = 20 + +# ############# proxy attributes ################# +# 是否启用代理地域属性 +PROXY_REGION = True + +# ############# scheduler config ################# + +# Set the timezone for the scheduler forcely (optional) +# If it is running on a VM, and +# "ValueError: Timezone offset does not match system offset" +# was raised during scheduling. +# Please uncomment the following line and set a timezone for the scheduler. +# Otherwise it will detect the timezone from the system automatically. + +TIMEZONE = "Asia/Shanghai" diff --git a/test.py b/test.py deleted file mode 100644 index 518710d3b..000000000 --- a/test.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： test.py - Description : - Author : JHao - date： 2017/3/7 -------------------------------------------------- - Change Activity: - 2017/3/7: -------------------------------------------------- -""" -__author__ = 'JHao' - -from Schedule import ProxyRefreshSchedule \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/test_proxy_api.py b/tests/api/test_proxy_api.py new file mode 100644 index 000000000..012a11e88 --- /dev/null +++ b/tests/api/test_proxy_api.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testProxyApi.py + Description : Flask API全路由测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock +from helper.proxy import Proxy +from api.proxyApi import JsonResponse + + +@pytest.fixture +def mocks(app): + """快捷访问 app._test_mocks""" + return app._test_mocks + + +class TestIndex: + + def test_index_returns_api_list(self, client): + resp = client.get("/") + assert resp.status_code == 200 + data = resp.get_json() + assert "url" in data + assert len(data["url"]) > 0 + + +class TestGet: + + def test_get_returns_proxy(self, client, mocks): + proxy = Proxy("1.2.3.4:8080", source="test", https=False) + mocks["get"].return_value = proxy + + resp = client.get("/get/") + assert resp.status_code == 200 + data = resp.get_json() + assert data["proxy"] == "1.2.3.4:8080" + assert data["https"] is False + + def test_get_no_proxy(self, client, mocks): + mocks["get"].return_value = None + + resp = client.get("/get/") + assert resp.status_code == 200 + data = resp.get_json() + assert data["code"] == 0 + assert data["src"] == "no proxy" + + def test_get_https_filter(self, client, mocks): + proxy = Proxy("5.6.7.8:443", source="test", https=True) + mocks["get"].return_value = proxy + + resp = client.get("/get/?type=https") + assert resp.status_code == 200 + data = resp.get_json() + assert data["https"] is True + mocks["get"].assert_called_with(True) + + def test_get_http_filter(self, client, mocks): + mocks["get"].return_value = None + + client.get("/get/") + mocks["get"].assert_called_with(False) + + +class TestPop: + + def test_pop_returns_proxy(self, client, mocks): + proxy = Proxy("1.2.3.4:8080", source="test") + mocks["pop"].return_value = proxy + + resp = client.get("/pop/") + assert resp.status_code == 200 + data = resp.get_json() + assert data["proxy"] == "1.2.3.4:8080" + + def test_pop_no_proxy(self, client, mocks): + mocks["pop"].return_value = None + + resp = client.get("/pop/") + data = resp.get_json() + assert data["code"] == 0 + + +class TestAll: + + def test_all_returns_list(self, client, mocks): + proxies = [ + Proxy("1.2.3.4:8080", source="test"), + Proxy("5.6.7.8:443", source="test", https=True), + ] + mocks["getAll"].return_value = proxies + + resp = client.get("/all/") + assert resp.status_code == 200 + data = resp.get_json() + assert len(data) == 2 + assert data[0]["proxy"] == "1.2.3.4:8080" + assert data[1]["proxy"] == "5.6.7.8:443" + + def test_all_empty(self, client, mocks): + mocks["getAll"].return_value = [] + + resp = client.get("/all/") + data = resp.get_json() + assert data == [] + + +class TestDelete: + + def test_delete_calls_handler(self, client, mocks): + mocks["delete"].return_value = True + + resp = client.get("/delete/?proxy=1.2.3.4:8080") + assert resp.status_code == 200 + data = resp.get_json() + assert data["code"] == 0 + assert data["src"] is True + mocks["delete"].assert_called_once() + + +class TestCount: + + def test_count_returns_stats(self, client, mocks): + proxies = [ + Proxy("1.2.3.4:8080", source="freeProxy01", https=False), + Proxy("5.6.7.8:443", source="freeProxy02", https=True), + ] + mocks["getAll"].return_value = proxies + + resp = client.get("/count/") + assert resp.status_code == 200 + data = resp.get_json() + assert data["count"] == 2 + assert data["http_type"]["http"] == 1 + assert data["http_type"]["https"] == 1 + assert data["source"]["freeProxy01"] == 1 + assert data["source"]["freeProxy02"] == 1 + + def test_count_empty(self, client, mocks): + mocks["getAll"].return_value = [] + + resp = client.get("/count/") + data = resp.get_json() + assert data["count"] == 0 + assert data["http_type"] == {} + assert data["source"] == {} + + +class TestRefresh: + + def test_refresh_returns_success(self, client): + resp = client.get("/refresh/") + assert resp.status_code == 200 + assert b"success" in resp.data + + +class TestJsonResponse: + + def test_force_type_with_dict(self, app): + """dict -> JSON Response""" + with app.app_context(): + resp = JsonResponse.force_type({"key": "val"}) + assert resp.content_type == "application/json" + + def test_force_type_with_list(self, app): + """list -> JSON Response""" + with app.app_context(): + resp = JsonResponse.force_type([1, 2, 3]) + assert resp.content_type == "application/json" + + +class TestRunFlask: + + @patch("api.proxyApi.platform") + @patch("api.proxyApi.app") + def test_runflask_windows_path(self, mock_app, mock_platform): + """Windows 下调用 app.run()""" + mock_platform.system.return_value = "Windows" + from api.proxyApi import runFlask + runFlask() + mock_app.run.assert_called_once() \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..4ec30bda6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： conftest.py + Description : 测试共享fixtures + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys +import os +from unittest.mock import MagicMock, patch + +import pytest +import fakeredis + +# 确保项目根目录在 sys.path 中 +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from util.singleton import Singleton +from helper.proxy import Proxy + + +# --------------- Singleton 重置 --------------- + +@pytest.fixture(autouse=True) +def reset_singleton(): + """每个测试前清空 Singleton 缓存，防止测试间状态泄漏""" + saved = Singleton._inst.copy() + Singleton._inst.clear() + yield + Singleton._inst.clear() + Singleton._inst.update(saved) + + +# --------------- Proxy 工厂 --------------- + +@pytest.fixture +def proxy_obj(): + """标准测试用 Proxy 对象""" + return Proxy("1.2.3.4:8080", source="test", https=False) + + +@pytest.fixture +def https_proxy_obj(): + """HTTPS 测试用 Proxy 对象""" + return Proxy("5.6.7.8:443", source="test", https=True) + + +# --------------- Redis / DB --------------- + +@pytest.fixture +def fake_redis(): + """fakeredis 实例，用于 RedisClient/SsdbClient 测试""" + return fakeredis.FakeRedis(decode_responses=True, protocol=2) + + +@pytest.fixture +def mock_db_client(fake_redis): + """mock DbClient，返回 fakeredis 支持的 RedisClient 行为""" + with patch("db.dbClient.DbClient") as mock_cls: + yield mock_cls, fake_redis + + +# --------------- Flask API --------------- + +@pytest.fixture +def app(): + """Flask app，proxy_handler 被 mock""" + # mock 掉 DbClient，防止 ProxyHandler 连接真实 Redis + # 必须 patch handler.proxyHandler.DbClient（已 import 到本地命名空间） + with patch("handler.proxyHandler.DbClient") as mock_db_cls: + mock_db_instance = MagicMock() + mock_db_cls.return_value = mock_db_instance + + from api.proxyApi import app as flask_app, proxy_handler + flask_app.config["TESTING"] = True + + # 替换 proxy_handler 的方法为 MagicMock，方便测试中配置返回值 + with patch.object(proxy_handler, "get") as mock_get, \ + patch.object(proxy_handler, "pop") as mock_pop, \ + patch.object(proxy_handler, "getAll") as mock_getAll, \ + patch.object(proxy_handler, "delete") as mock_delete: + flask_app._test_mocks = { + "get": mock_get, + "pop": mock_pop, + "getAll": mock_getAll, + "delete": mock_delete, + } + yield flask_app + + +@pytest.fixture +def client(app): + """Flask test client""" + return app.test_client() \ No newline at end of file diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_redis_client.py b/tests/integration/test_redis_client.py new file mode 100644 index 000000000..9c048ca0f --- /dev/null +++ b/tests/integration/test_redis_client.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testRedisClient.py + Description : RedisClient集成测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import json +import pytest +import fakeredis +from unittest.mock import patch, MagicMock +from db.redisClient import RedisClient +from helper.proxy import Proxy + + +@pytest.fixture +def redis_client(fake_redis): + """RedisClient 实例，内部连接替换为 fakeredis""" + with patch("db.redisClient.BlockingConnectionPool"): + with patch("db.redisClient.Redis", return_value=fake_redis): + client = RedisClient(host="localhost", port=6379, + username=None, password=None, db="0") + client.changeTable("test_proxy") + return client + + +def _make_proxy(proxy_str, https=False, source="test"): + return Proxy(proxy_str, source=https and "https_test" or "http_test", + https=https) + + +class TestRedisPutGet: + + def test_put_and_get(self, redis_client): + proxy = _make_proxy("1.2.3.4:8080") + redis_client.put(proxy) + result = redis_client.get(https=False) + assert result is not None + data = json.loads(result) + assert data["proxy"] == "1.2.3.4:8080" + + def test_get_https(self, redis_client): + proxy = _make_proxy("5.6.7.8:443", https=True) + redis_client.put(proxy) + result = redis_client.get(https=True) + assert result is not None + data = json.loads(result) + assert data["https"] is True + + def test_get_https_excludes_http(self, redis_client): + proxy = _make_proxy("1.2.3.4:8080", https=False) + redis_client.put(proxy) + result = redis_client.get(https=True) + assert result is None + + def test_get_empty_returns_none(self, redis_client): + result = redis_client.get(https=False) + assert result is None + + +class TestRedisExists: + + def test_exists_true(self, redis_client): + proxy = _make_proxy("1.2.3.4:8080") + redis_client.put(proxy) + assert redis_client.exists("1.2.3.4:8080") is True + + def test_exists_false(self, redis_client): + assert redis_client.exists("9.9.9.9:9999") is False + + +class TestRedisDelete: + + def test_delete(self, redis_client): + proxy = _make_proxy("1.2.3.4:8080") + redis_client.put(proxy) + redis_client.delete("1.2.3.4:8080") + assert redis_client.exists("1.2.3.4:8080") is False + + +class TestRedisPop: + + def test_pop_removes_proxy(self, redis_client): + proxy = _make_proxy("1.2.3.4:8080") + redis_client.put(proxy) + popped = redis_client.pop(https=False) + assert popped is not None + assert redis_client.exists("1.2.3.4:8080") is False + + def test_pop_empty_returns_none(self, redis_client): + result = redis_client.pop(https=False) + assert result is None + + +class TestRedisGetAll: + + def test_get_all(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080")) + redis_client.put(_make_proxy("5.6.7.8:443", https=True)) + all_proxies = redis_client.getAll(https=False) + assert len(all_proxies) == 2 + + def test_get_all_https_filter(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080", https=False)) + redis_client.put(_make_proxy("5.6.7.8:443", https=True)) + https_proxies = redis_client.getAll(https=True) + assert len(https_proxies) == 1 + + +class TestRedisGetCount: + + def test_get_count(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080", https=False)) + redis_client.put(_make_proxy("5.6.7.8:443", https=True)) + count = redis_client.getCount() + assert count["total"] == 2 + assert count["https"] == 1 + + def test_get_count_empty(self, redis_client): + count = redis_client.getCount() + assert count["total"] == 0 + assert count["https"] == 0 + + +class TestRedisClear: + + def test_clear(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080")) + redis_client.put(_make_proxy("5.6.7.8:443")) + redis_client.clear() + count = redis_client.getCount() + assert count["total"] == 0 + + +class TestRedisChangeTable: + + def test_change_table_isolation(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080")) + redis_client.changeTable("other_table") + assert redis_client.getCount()["total"] == 0 + redis_client.changeTable("test_proxy") + assert redis_client.getCount()["total"] == 1 \ No newline at end of file diff --git a/tests/integration/test_ssdb_client.py b/tests/integration/test_ssdb_client.py new file mode 100644 index 000000000..ab046ef7f --- /dev/null +++ b/tests/integration/test_ssdb_client.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testSsdbClient.py + Description : SsdbClient集成测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import json +import pytest +from unittest.mock import patch +from db.ssdbClient import SsdbClient +from helper.proxy import Proxy + + +@pytest.fixture +def ssdb_client(fake_redis): + """SsdbClient 实例，内部连接替换为 fakeredis""" + with patch("db.ssdbClient.BlockingConnectionPool"): + with patch("db.ssdbClient.Redis", return_value=fake_redis): + client = SsdbClient(host="localhost", port=8888, + username=None, password=None) + client.changeTable("test_proxy") + return client + + +def _make_proxy(proxy_str, https=False, source="test"): + return Proxy(proxy_str, source=https and "https_test" or "http_test", + https=https) + + +class TestSsdbPutGet: + + def test_put_and_get(self, ssdb_client): + proxy = _make_proxy("1.2.3.4:8080") + ssdb_client.put(proxy) + result = ssdb_client.get(https=False) + assert result is not None + data = json.loads(result) + assert data["proxy"] == "1.2.3.4:8080" + + def test_get_https(self, ssdb_client): + proxy = _make_proxy("5.6.7.8:443", https=True) + ssdb_client.put(proxy) + result = ssdb_client.get(https=True) + assert result is not None + data = json.loads(result) + assert data["https"] is True + + def test_get_https_excludes_http(self, ssdb_client): + proxy = _make_proxy("1.2.3.4:8080", https=False) + ssdb_client.put(proxy) + result = ssdb_client.get(https=True) + assert result is None + + def test_get_empty_returns_none(self, ssdb_client): + result = ssdb_client.get(https=False) + assert result is None + + +class TestSsdbExists: + + def test_exists_true(self, ssdb_client): + proxy = _make_proxy("1.2.3.4:8080") + ssdb_client.put(proxy) + assert ssdb_client.exists("1.2.3.4:8080") is True + + def test_exists_false(self, ssdb_client): + assert ssdb_client.exists("9.9.9.9:9999") is False + + +class TestSsdbDelete: + + def test_delete(self, ssdb_client): + proxy = _make_proxy("1.2.3.4:8080") + ssdb_client.put(proxy) + ssdb_client.delete("1.2.3.4:8080") + assert ssdb_client.exists("1.2.3.4:8080") is False + + +class TestSsdbPop: + + def test_pop_removes_proxy(self, ssdb_client): + proxy = _make_proxy("1.2.3.4:8080") + ssdb_client.put(proxy) + popped = ssdb_client.pop(https=False) + assert popped is not None + assert ssdb_client.exists("1.2.3.4:8080") is False + + def test_pop_empty_returns_none(self, ssdb_client): + result = ssdb_client.pop(https=False) + assert result is None + + +class TestSsdbGetAll: + + def test_get_all(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080")) + ssdb_client.put(_make_proxy("5.6.7.8:443", https=True)) + all_proxies = list(ssdb_client.getAll(https=False)) + assert len(all_proxies) == 2 + + def test_get_all_https_filter(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080", https=False)) + ssdb_client.put(_make_proxy("5.6.7.8:443", https=True)) + https_proxies = list(ssdb_client.getAll(https=True)) + assert len(https_proxies) == 1 + + +class TestSsdbGetCount: + + def test_get_count(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080", https=False)) + ssdb_client.put(_make_proxy("5.6.7.8:443", https=True)) + count = ssdb_client.getCount() + assert count["total"] == 2 + assert count["https"] == 1 + + def test_get_count_empty(self, ssdb_client): + count = ssdb_client.getCount() + assert count["total"] == 0 + assert count["https"] == 0 + + +class TestSsdbClear: + + def test_clear(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080")) + ssdb_client.put(_make_proxy("5.6.7.8:443")) + ssdb_client.clear() + count = ssdb_client.getCount() + assert count["total"] == 0 + + +class TestSsdbChangeTable: + + def test_change_table_isolation(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080")) + ssdb_client.changeTable("other_table") + assert ssdb_client.getCount()["total"] == 0 + ssdb_client.changeTable("test_proxy") + assert ssdb_client.getCount()["total"] == 1 \ No newline at end of file diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/test_base_fetcher.py b/tests/unit/test_base_fetcher.py new file mode 100644 index 000000000..d086c900c --- /dev/null +++ b/tests/unit/test_base_fetcher.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_base_fetcher.py + Description : BaseFetcher 基类测试 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher + + +class TestParseProxiesFromText(object): + """parseProxiesFromText 测试""" + + def test_basic_ip_port(self): + text = "1.2.3.4:8080" + result = BaseFetcher.parseProxiesFromText(text) + assert result == ["1.2.3.4:8080"] + + def test_multiple_proxies(self): + text = "1.2.3.4:8080\n5.6.7.8:3128\n9.10.11.12:80" + result = BaseFetcher.parseProxiesFromText(text) + assert result == ["1.2.3.4:8080", "5.6.7.8:3128", "9.10.11.12:80"] + + def test_ip_port_with_spaces(self): + text = "1.2.3.4 8080" + result = BaseFetcher.parseProxiesFromText(text) + assert result == ["1.2.3.4:8080"] + + def test_ip_port_in_html_text(self): + """HTML标签中的ip:port需要跨标签匹配，parseProxiesFromText只处理纯文本""" + text = '1.2.3.48080' + result = BaseFetcher.parseProxiesFromText(text) + assert result == [] + + def test_empty_text(self): + assert BaseFetcher.parseProxiesFromText("") == [] + assert BaseFetcher.parseProxiesFromText(None) == [] + + def test_no_proxies(self): + text = "no proxies here" + assert BaseFetcher.parseProxiesFromText(text) == [] + + def test_ip_with_colon_port(self): + text = "192.168.1.1:3128" + result = BaseFetcher.parseProxiesFromText(text) + assert result == ["192.168.1.1:3128"] + + def test_port_range(self): + text = "1.2.3.4:80 1.2.3.4:65535" + result = BaseFetcher.parseProxiesFromText(text) + assert "1.2.3.4:80" in result + assert "1.2.3.4:65535" in result + + +class TestYieldUniqueProxies(object): + """yieldUniqueProxies 测试""" + + def test_unique_proxies(self): + proxies = ["1.2.3.4:8080", "5.6.7.8:3128"] + result = list(BaseFetcher.yieldUniqueProxies(proxies)) + assert result == ["1.2.3.4:8080", "5.6.7.8:3128"] + + def test_duplicates_removed(self): + proxies = ["1.2.3.4:8080", "5.6.7.8:3128", "1.2.3.4:8080"] + result = list(BaseFetcher.yieldUniqueProxies(proxies)) + assert result == ["1.2.3.4:8080", "5.6.7.8:3128"] + + def test_empty_list(self): + assert list(BaseFetcher.yieldUniqueProxies([])) == [] + + def test_preserves_order(self): + proxies = ["3.3.3.3:80", "1.1.1.1:80", "2.2.2.2:80", "3.3.3.3:80"] + result = list(BaseFetcher.yieldUniqueProxies(proxies)) + assert result == ["3.3.3.3:80", "1.1.1.1:80", "2.2.2.2:80"] + + def test_is_generator(self): + import types + gen = BaseFetcher.yieldUniqueProxies([]) + assert isinstance(gen, types.GeneratorType) + + +class TestBaseFetcherInterface(object): + """BaseFetcher 接口约定测试""" + + def test_name_attribute(self): + assert hasattr(BaseFetcher, 'name') + assert BaseFetcher.name == "" + + def test_url_attribute(self): + assert hasattr(BaseFetcher, 'url') + assert BaseFetcher.url == "" + + def test_fetch_raises_not_implemented(self): + fetcher = BaseFetcher() + try: + fetcher.fetch() + assert False, "Should have raised NotImplementedError" + except NotImplementedError: + pass diff --git a/tests/unit/test_check.py b/tests/unit/test_check.py new file mode 100644 index 000000000..eb1daf8f1 --- /dev/null +++ b/tests/unit/test_check.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_check.py + Description : helper/check.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock, PropertyMock +from datetime import datetime + +from helper.proxy import Proxy +from helper.check import DoValidator, _ThreadChecker + + +class TestDoValidator: + """DoValidator.validator 测试""" + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_pass_https_pass(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + HTTPS 通过 -> https=True, fail_count 不变""" + mock_pv = MagicMock() + mock_pv.http_validator = [MagicMock(return_value=True)] + mock_pv.https_validator = [MagicMock(return_value=True)] + mock_pv_cls.http_validator = mock_pv.http_validator + mock_pv_cls.https_validator = mock_pv.https_validator + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + mock_conf_cls.return_value = mock_conf + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 0 + + # Patch DoValidator.conf at class level + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.https is True + assert result.last_status is True + assert result.check_count == 1 + assert result.fail_count == 0 + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_pass_https_fail(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + HTTPS 失败 -> https=False""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=False)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.https is False + assert result.last_status is True + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_http_fail(self, mock_pv_cls, mock_conf_cls): + """HTTP 失败 -> fail_count += 1, last_status=False""" + mock_pv_cls.http_validator = [MagicMock(return_value=False)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 0 + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.last_status is False + assert result.fail_count == 1 + + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_fail_count_decrement(self, mock_pv_cls, mock_conf_cls): + """HTTP 通过 + fail_count > 0 -> fail_count -= 1""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.fail_count = 3 + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "use") + + assert result.fail_count == 2 + + @patch("helper.check.DoValidator.regionGetter", return_value="US") + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_raw_sets_region(self, mock_pv_cls, mock_conf_cls, mock_region): + """work_type='raw' + proxyRegion=True -> regionGetter 被调用""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = True + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + result = DoValidator.validator(proxy, "raw") + + assert result.region == "US" + mock_region.assert_called_once_with(proxy) + + @patch("helper.check.DoValidator.regionGetter") + @patch("helper.check.ConfigHandler") + @patch("helper.check.ProxyValidator") + def test_validator_use_skips_region(self, mock_pv_cls, mock_conf_cls, mock_region): + """work_type='use' -> 不调用 regionGetter""" + mock_pv_cls.http_validator = [MagicMock(return_value=True)] + mock_pv_cls.https_validator = [MagicMock(return_value=True)] + + mock_conf = MagicMock() + mock_conf.proxyRegion = True + + proxy = Proxy("1.2.3.4:8080", source="test") + + with patch.object(DoValidator, "conf", mock_conf): + DoValidator.validator(proxy, "use") + + mock_region.assert_not_called() + + +class TestRegionGetter: + """DoValidator.regionGetter 测试""" + + @patch("helper.check.WebRequest") + def test_success_returns_country_code(self, mock_wr_cls): + """正常返回 -> country_code""" + mock_wr = MagicMock() + mock_wr.get.return_value.json = {"country_code": "CN"} + mock_wr_cls.return_value = mock_wr + + proxy = Proxy("1.2.3.4:8080") + result = DoValidator.regionGetter(proxy) + assert result == "CN" + + @patch("helper.check.WebRequest") + def test_exception_returns_error(self, mock_wr_cls): + """异常 -> 'error'""" + mock_wr = MagicMock() + mock_wr.get.side_effect = Exception("timeout") + mock_wr_cls.return_value = mock_wr + + proxy = Proxy("1.2.3.4:8080") + result = DoValidator.regionGetter(proxy) + assert result == "error" + + +def _make_checker(work_type, proxy_handler, conf=None): + """构造手动注入依赖的 _ThreadChecker（绕过 Thread.__init__）""" + checker = _ThreadChecker.__new__(_ThreadChecker) + # 手动初始化 Thread 所需的状态 + checker._initialized = True + checker._name = "test_thread" + checker._target = None + checker._args = () + checker._kwargs = {} + checker._daemonic = False + checker._ident = None + checker._tstate_lock = None + checker._started = MagicMock() + checker._is_stopped = False + checker._block = MagicMock() + checker._waiters = [] + checker._stderr = None + # 注入依赖 + checker.proxy_handler = proxy_handler + checker.log = MagicMock() + checker.work_type = work_type + checker.conf = conf or MagicMock() + return checker + + +class TestThreadCheckerIfRaw: + """_ThreadChecker.__ifRaw 测试""" + + def test_ifraw_new_proxy_gets_put(self): + """last_status=True, exists=False -> put""" + mock_ph = MagicMock() + mock_ph.exists.return_value = False + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_called_once_with(proxy) + + def test_ifraw_existing_proxy_skipped(self): + """last_status=True, exists=True -> 不 put""" + mock_ph = MagicMock() + mock_ph.exists.return_value = True + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_not_called() + + def test_ifraw_failed_proxy_not_put(self): + """last_status=False -> 不 put""" + mock_ph = MagicMock() + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + + checker = _make_checker("raw", mock_ph) + checker._ThreadChecker__ifRaw(proxy) + mock_ph.put.assert_not_called() + + +class TestThreadCheckerIfUse: + """_ThreadChecker.__ifUse 测试""" + + def test_ifuse_pass_gets_put(self): + """last_status=True -> put""" + mock_ph = MagicMock() + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = True + + checker = _make_checker("use", mock_ph) + checker._ThreadChecker__ifUse(proxy) + mock_ph.put.assert_called_once_with(proxy) + + def test_ifuse_fail_exceeds_max_deleted(self): + """fail_count > maxFailCount -> delete""" + mock_ph = MagicMock() + mock_conf = MagicMock() + mock_conf.maxFailCount = 3 + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + proxy.fail_count = 5 + + checker = _make_checker("use", mock_ph, mock_conf) + checker._ThreadChecker__ifUse(proxy) + mock_ph.delete.assert_called_once_with(proxy) + mock_ph.put.assert_not_called() + + def test_ifuse_fail_below_max_kept(self): + """fail_count <= maxFailCount -> put""" + mock_ph = MagicMock() + mock_conf = MagicMock() + mock_conf.maxFailCount = 3 + + proxy = Proxy("1.2.3.4:8080", source="test") + proxy.last_status = False + proxy.fail_count = 2 + + checker = _make_checker("use", mock_ph, mock_conf) + checker._ThreadChecker__ifUse(proxy) + mock_ph.put.assert_called_once_with(proxy) + mock_ph.delete.assert_not_called() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 000000000..8c8fe662a --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_cli.py + Description : proxyPool CLI 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock +from click.testing import CliRunner + +from proxyPool import cli +from setting import VERSION + + +@pytest.fixture +def runner(): + return CliRunner() + + +class TestCli: + + def test_version_flag(self, runner): + """--version 显示版本号""" + result = runner.invoke(cli, ["--version"]) + assert result.exit_code == 0 + assert VERSION in result.output + + @patch("proxyPool.startScheduler") + def test_schedule_command(self, mock_scheduler, runner): + """schedule 命令调用 startScheduler""" + result = runner.invoke(cli, ["schedule"]) + assert result.exit_code == 0 + mock_scheduler.assert_called_once() + + @patch("proxyPool.startServer") + def test_server_command(self, mock_server, runner): + """server 命令调用 startServer""" + result = runner.invoke(cli, ["server"]) + assert result.exit_code == 0 + mock_server.assert_called_once() + + @patch("handler.configHandler.ConfigHandler") + @patch("helper.fetch._discover_fetchers") + def test_fetcher_command(self, mock_discover, mock_conf_cls, runner): + """fetcher 命令输出启用的代理源列表""" + mock_cls1 = MagicMock() + mock_cls1.name = "freeProxy01" + mock_cls2 = MagicMock() + mock_cls2.name = "freeProxy02" + mock_discover.return_value = [mock_cls1, mock_cls2] + mock_conf = MagicMock() + mock_conf.fetcherExclude = [] + mock_conf_cls.return_value = mock_conf + + result = runner.invoke(cli, ["fetcher"]) + assert result.exit_code == 0 + assert "freeProxy01" in result.output + assert "freeProxy02" in result.output \ No newline at end of file diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 000000000..abf3b8a66 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testConfig.py + Description : ConfigHandler环境变量测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import pytest +import setting +from handler.configHandler import ConfigHandler + + +@pytest.fixture(autouse=True) +def clean_env(): + """测试前后清理可能设置的环境变量""" + env_keys = ["DB_CONN", "PORT", "HOST", "TABLE_NAME", "HTTP_URL", + "HTTPS_URL", "VERIFY_TIMEOUT", "MAX_FAIL_COUNT", + "POOL_SIZE_MIN", "PROXY_REGION", "TIMEZONE"] + saved = {k: os.environ.get(k) for k in env_keys} + for k in env_keys: + os.environ.pop(k, None) + yield + for k, v in saved.items(): + if v is not None: + os.environ[k] = v + else: + os.environ.pop(k, None) + + +@pytest.fixture +def conf(): + return ConfigHandler() + + +class TestConfigHandlerDefaults: + + def test_db_conn_default(self, conf): + assert conf.dbConn == setting.DB_CONN + + def test_server_host_default(self, conf): + assert conf.serverHost == setting.HOST + + def test_server_port_default(self, conf): + assert str(conf.serverPort) == str(setting.PORT) + + def test_table_name_default(self, conf): + assert conf.tableName == setting.TABLE_NAME + + def test_http_url_default(self, conf): + assert conf.httpUrl == setting.HTTP_URL + + def test_https_url_default(self, conf): + assert conf.httpsUrl == setting.HTTPS_URL + + def test_verify_timeout_default(self, conf): + assert conf.verifyTimeout == setting.VERIFY_TIMEOUT + + def test_max_fail_count_default(self, conf): + assert conf.maxFailCount == setting.MAX_FAIL_COUNT + + def test_pool_size_min_default(self, conf): + assert conf.poolSizeMin == setting.POOL_SIZE_MIN + + def test_timezone_default(self, conf): + assert conf.timezone == setting.TIMEZONE + + def test_fetcher_exclude_is_list(self, conf): + assert isinstance(conf.fetcherExclude, list) + + +class TestConfigHandlerEnvOverride: + + def test_db_conn_override(self): + os.environ["DB_CONN"] = "redis://:newpwd@10.0.0.1:6380/3" + conf = ConfigHandler() + assert conf.dbConn == "redis://:newpwd@10.0.0.1:6380/3" + + def test_port_override(self): + os.environ["PORT"] = "8080" + conf = ConfigHandler() + assert str(conf.serverPort) == "8080" + + def test_verify_timeout_override(self): + os.environ["VERIFY_TIMEOUT"] = "30" + conf = ConfigHandler() + assert conf.verifyTimeout == 30 + + def test_max_fail_count_override(self): + os.environ["MAX_FAIL_COUNT"] = "5" + conf = ConfigHandler() + assert conf.maxFailCount == 5 \ No newline at end of file diff --git a/tests/unit/test_db_client.py b/tests/unit/test_db_client.py new file mode 100644 index 000000000..49ae429c4 --- /dev/null +++ b/tests/unit/test_db_client.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testDbClient.py + Description : DbClient URI解析单元测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import MagicMock, patch + +from db.dbClient import DbClient + + +class TestParseDbConn: + + def test_redis_uri(self): + DbClient.parseDbConn("redis://:password@127.0.0.1:6379/1") + assert DbClient.db_type == "REDIS" + assert DbClient.db_pwd == "password" + assert DbClient.db_host == "127.0.0.1" + assert DbClient.db_port == 6379 + assert DbClient.db_name == "1" + + def test_ssdb_uri(self): + DbClient.parseDbConn("ssdb://:password@127.0.0.1:8888") + assert DbClient.db_type == "SSDB" + assert DbClient.db_pwd == "password" + assert DbClient.db_host == "127.0.0.1" + assert DbClient.db_port == 8888 + + def test_redis_uri_no_password(self): + DbClient.parseDbConn("redis://127.0.0.1:6379/0") + assert DbClient.db_type == "REDIS" + assert DbClient.db_pwd is None + assert DbClient.db_host == "127.0.0.1" + assert DbClient.db_port == 6379 + assert DbClient.db_name == "0" + + def test_ssdb_uri_no_password(self): + DbClient.parseDbConn("ssdb://@127.0.0.1:8888") + assert DbClient.db_type == "SSDB" + assert DbClient.db_host == "127.0.0.1" + assert DbClient.db_port == 8888 + + def test_unknown_db_type_raises(self): + with pytest.raises(AssertionError): + DbClient("mysql://127.0.0.1:3306") + + @pytest.mark.parametrize("uri,expected_type", [ + ("redis://:pwd@10.0.0.1:6380/2", "REDIS"), + ("ssdb://:pwd@10.0.0.1:8899", "SSDB"), + ]) + def test_parse_returns_cls(self, uri, expected_type): + """parseDbConn 返回 cls 以支持链式调用""" + result = DbClient.parseDbConn(uri) + assert result is DbClient + assert DbClient.db_type == expected_type + + +class TestDbClientInit: + + @patch("db.dbClient.DbClient.parseDbConn") + def test_redis_init(self, mock_parse): + """Redis URI -> RedisClient 实例""" + with patch.object(DbClient, "_DbClient__initDbClient") as mock_init: + db = DbClient.__new__(DbClient) + DbClient.__init__(db, "redis://:pwd@127.0.0.1:6379/0") + mock_parse.assert_called_once_with("redis://:pwd@127.0.0.1:6379/0") + mock_init.assert_called_once() + + @patch("db.dbClient.DbClient.parseDbConn") + def test_ssdb_init(self, mock_parse): + """SSDB URI -> SsdbClient 实例""" + with patch.object(DbClient, "_DbClient__initDbClient") as mock_init: + db = DbClient.__new__(DbClient) + DbClient.__init__(db, "ssdb://:pwd@127.0.0.1:8888") + mock_parse.assert_called_once_with("ssdb://:pwd@127.0.0.1:8888") + mock_init.assert_called_once() + + +class TestDbClientDelegation: + """所有委托方法测试""" + + def _make_client(self): + """构造注入 mock client 的 DbClient""" + db = DbClient.__new__(DbClient) + db.client = MagicMock() + return db + + def test_get(self): + db = self._make_client() + db.client.get.return_value = '{"proxy": "1.2.3.4:8080"}' + result = db.get(True) + db.client.get.assert_called_once_with(True) + assert result == '{"proxy": "1.2.3.4:8080"}' + + def test_put(self): + db = self._make_client() + db.put("1.2.3.4:8080") + db.client.put.assert_called_once_with("1.2.3.4:8080") + + def test_update(self): + db = self._make_client() + db.update("key", "value") + db.client.update.assert_called_once_with("key", "value") + + def test_delete(self): + db = self._make_client() + db.delete("1.2.3.4:8080") + db.client.delete.assert_called_once_with("1.2.3.4:8080") + + def test_exists(self): + db = self._make_client() + db.client.exists.return_value = True + result = db.exists("1.2.3.4:8080") + db.client.exists.assert_called_once_with("1.2.3.4:8080") + assert result is True + + def test_pop(self): + db = self._make_client() + db.client.pop.return_value = '{"proxy": "1.2.3.4:8080"}' + result = db.pop(True) + db.client.pop.assert_called_once_with(True) + assert result == '{"proxy": "1.2.3.4:8080"}' + + def test_getAll(self): + db = self._make_client() + db.client.getAll.return_value = [] + result = db.getAll(False) + db.client.getAll.assert_called_once_with(False) + assert result == [] + + def test_clear(self): + db = self._make_client() + db.clear() + db.client.clear.assert_called_once() + + def test_changeTable(self): + db = self._make_client() + db.changeTable("use_proxy") + db.client.changeTable.assert_called_once_with("use_proxy") + + def test_getCount(self): + db = self._make_client() + db.client.getCount.return_value = 42 + result = db.getCount() + db.client.getCount.assert_called_once() + assert result == 42 + + def test_test(self): + db = self._make_client() + db.client.test.return_value = True + result = db.test() + db.client.test.assert_called_once() + assert result is True \ No newline at end of file diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py new file mode 100644 index 000000000..73e3b29e5 --- /dev/null +++ b/tests/unit/test_fetch.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_fetch.py + Description : helper/fetch.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import os +import sys +import pytest +from unittest.mock import patch, MagicMock + +import helper.fetch as fetch_mod +from helper.fetch import _get_sources_dir, _load_module, _discover_fetchers, _ThreadFetcher +from helper.proxy import Proxy +from fetcher.baseFetcher import BaseFetcher + + +class TestGetSourcesDir: + + def test_returns_correct_path(self): + """返回 fetcher/sources/ 目录路径""" + path = _get_sources_dir() + assert path.endswith(os.path.join("fetcher", "sources")) + assert os.path.isdir(path) + + +class TestLoadModule: + + def setup_method(self): + """每个测试前清空缓存""" + fetch_mod._module_cache.clear() + + def test_fresh_load(self): + """缓存为空 -> importlib.import_module""" + # 使用一个已知存在的模块 + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + result = _load_module("fetcher.sources.kuaidaili", filepath) + assert result is not None + assert "fetcher.sources.kuaidaili" in fetch_mod._module_cache + + def test_cache_hit(self): + """mtime 不变 -> 返回缓存""" + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + first = _load_module("fetcher.sources.kuaidaili", filepath) + second = _load_module("fetcher.sources.kuaidaili", filepath) + assert first is second + + def test_cache_miss_reload(self): + """mtime 变化 -> importlib.reload""" + filepath = os.path.join(_get_sources_dir(), "kuaidaili.py") + first = _load_module("fetcher.sources.kuaidaili", filepath) + # 模拟 mtime 变化 + fetch_mod._module_cache["fetcher.sources.kuaidaili"] = (0, first) + second = _load_module("fetcher.sources.kuaidaili", filepath) + assert second is not None + + @patch("helper.fetch.os.path.getmtime", return_value=0) + @patch("helper.fetch.importlib") + def test_import_exception_returns_none(self, mock_importlib, mock_mtime): + """import 失败 -> 返回 None""" + mock_importlib.import_module.side_effect = ImportError("not found") + # 确保模块不在 sys.modules 中，避免走 reload 分支 + mock_importlib.reload.side_effect = ImportError("not found") + saved = sys.modules.pop("fetcher.sources.nonexistent", None) + try: + result = _load_module("fetcher.sources.nonexistent", "/fake/path.py") + assert result is None + finally: + if saved is not None: + sys.modules["fetcher.sources.nonexistent"] = saved + + +class TestDiscoverFetchers: + + def setup_method(self): + fetch_mod._module_cache.clear() + + def test_filters_enabled_only(self): + """enabled=False 的 fetcher 被排除""" + # 使用真实扫描，检查结果中所有 fetcher 都是 enabled=True + fetchers = _discover_fetchers([]) + for f in fetchers: + assert f.enabled is True + + def test_filters_exclude_list(self): + """exclude_list 中的被排除""" + all_fetchers = _discover_fetchers([]) + if not all_fetchers: + pytest.skip("No fetchers available") + first_name = all_fetchers[0].__name__ + filtered = _discover_fetchers([first_name]) + filtered_names = [f.__name__ for f in filtered] + assert first_name not in filtered_names + + def test_returns_sorted_by_name(self): + """返回结果按 name 排序""" + fetchers = _discover_fetchers([]) + names = [f.name for f in fetchers] + assert names == sorted(names) + + def test_prunes_stale_cache(self): + """已删除文件的缓存被清理""" + fetch_mod._module_cache["fetcher.sources.deleted_module"] = (0, MagicMock()) + _discover_fetchers([]) + assert "fetcher.sources.deleted_module" not in fetch_mod._module_cache + + +class TestThreadFetcher: + + def test_collects_proxies(self): + """fetcher.fetch() yield 代理 -> proxy_dict 有值""" + mock_cls = MagicMock() + mock_cls.name = "test_fetcher" + mock_cls.return_value.fetch.return_value = ["1.2.3.4:8080", "5.6.7.8:443"] + + proxy_dict = {} + thread = _ThreadFetcher(mock_cls, proxy_dict) + thread.run() + + assert "1.2.3.4:8080" in proxy_dict + assert "5.6.7.8:443" in proxy_dict + assert isinstance(proxy_dict["1.2.3.4:8080"], Proxy) + + def test_merges_duplicate_sources(self): + """同一代理出现两次 -> add_source 被调用""" + mock_cls = MagicMock() + mock_cls.name = "test_fetcher" + mock_cls.return_value.fetch.return_value = ["1.2.3.4:8080", "1.2.3.4:8080"] + + proxy_dict = {} + thread = _ThreadFetcher(mock_cls, proxy_dict) + thread.run() + + assert "1.2.3.4:8080" in proxy_dict + # source 应该包含两次 "test_fetcher"（add_source 去重，但只出现一次） + assert "test_fetcher" in proxy_dict["1.2.3.4:8080"].source diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py new file mode 100644 index 000000000..0c1649b6c --- /dev/null +++ b/tests/unit/test_fetcher_sources.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_fetcher_sources.py + Description : 各代理源 fetcher 测试 + Author : JHao + date： 2026/5/31 +------------------------------------------------- + Change Activity: + 2026/05/31: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re +from unittest.mock import patch, MagicMock + +from lxml import etree + +from fetcher.baseFetcher import BaseFetcher + + +# --------------- 辅助工具 --------------- + +def _make_response(text="", tree=None, json_data=None): + """构造 mock 的 WebRequest 返回对象""" + resp = MagicMock() + resp.text = text + resp.tree = tree + resp.json = json_data if json_data is not None else {} + return resp + + +def _html_table(rows, has_header=False): + """快速生成 HTML table 字符串""" + html = "" + if has_header: + html += "" + for ip, port in rows: + html += "" % (ip, port) + html += "
IPPort
%s%s
" + return html + + +def _assert_valid_proxies(proxies): + """验证所有 proxy 符合 ip:port 格式""" + pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}$') + for p in proxies: + assert pattern.match(p), f"Invalid proxy format: {p}" + + +# --------------- 接口约定测试 --------------- + +class TestFetcherInterface(object): + """所有 fetcher 的接口约定""" + + FETCHER_CLASSES = [ + ("fetcher.sources.kxdaili", "KxdailiFetcher"), + ("fetcher.sources.ip3366", "Ip3366Fetcher"), + ("fetcher.sources.ip89", "Ip89Fetcher"), + ("fetcher.sources.docip", "DocipFetcher"), + ("fetcher.sources.goodips", "GoodipsFetcher"), + ("fetcher.sources.geonode", "GeonodeFetcher"), + + ("fetcher.sources.kuaidaili", "KuaidailiFetcher"), + ("fetcher.sources.freevpnnode", "FreeVPNNodeFetcher"), + ("fetcher.sources.scdn", "ScdnFetcher"), + ("fetcher.sources.zdaye", "ZdayeFetcher"), + ("fetcher.sources.ihuan", "IhuanFetcher"), + ("fetcher.sources.proxifly", "ProxiFlyFetcher"), + ("fetcher.sources.daili66", "DaiLi66Fetcher"), + ("fetcher.sources.roundproxies", "RoundProxiesFetcher"), + ] + + def test_all_fetchers_have_name_url_enabled(self): + for module_path, class_name in self.FETCHER_CLASSES: + module = __import__(module_path, fromlist=[class_name]) + cls = getattr(module, class_name) + assert cls.name, f"{class_name} missing name" + assert cls.url, f"{class_name} missing url" + assert hasattr(cls, 'enabled'), f"{class_name} missing enabled" + + def test_all_fetchers_subclass_base(self): + for module_path, class_name in self.FETCHER_CLASSES: + module = __import__(module_path, fromlist=[class_name]) + cls = getattr(module, class_name) + assert issubclass(cls, BaseFetcher), f"{class_name} not subclass of BaseFetcher" + + def test_all_fetchers_have_fetch_method(self): + for module_path, class_name in self.FETCHER_CLASSES: + module = __import__(module_path, fromlist=[class_name]) + cls = getattr(module, class_name) + assert hasattr(cls, 'fetch'), f"{class_name} missing fetch method" + + +# --------------- 各 fetcher 逻辑测试 --------------- + +class TestKxdailiFetcher(object): + + @patch("fetcher.sources.kxdaili.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.kxdaili import KxdailiFetcher + html = '' \ + '
IPPort
1.2.3.48080
' + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response(tree=tree) + result = list(KxdailiFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestIp3366Fetcher(object): + + @patch("fetcher.sources.ip3366.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.ip3366 import Ip3366Fetcher + html = '1.2.3.480805.6.7.83128' + mock_wr.return_value.get.return_value = _make_response(text=html) + result = list(Ip3366Fetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + +class TestIp89Fetcher(object): + + @patch("fetcher.sources.ip89.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.ip89 import Ip89Fetcher + html = '1.2.3.48080' + mock_wr.return_value.get.return_value = _make_response(text=html) + result = list(Ip89Fetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestDocipFetcher(object): + + @patch("fetcher.sources.docip.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.docip import DocipFetcher + json_data = {"data": [{"ip": "1.2.3.4:8080"}, {"ip": "5.6.7.8:3128"}]} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(DocipFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + +class TestGoodipsFetcher(object): + + @patch("fetcher.sources.goodips.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.goodips import GoodipsFetcher + html = '
  • 1.2.3.4
  • 8080
' + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response(tree=tree) + result = list(GoodipsFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestGeonodeFetcher(object): + + @patch("fetcher.sources.geonode.WebRequest") + def test_fetch_json(self, mock_wr): + from fetcher.sources.geonode import GeonodeFetcher + json_data = {"data": [{"ip": "1.2.3.4", "port": "8080"}]} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(GeonodeFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.geonode.WebRequest") + def test_fetch_text_fallback(self, mock_wr): + from fetcher.sources.geonode import GeonodeFetcher + mock_wr.return_value.get.return_value = _make_response( + json_data={}, text="1.2.3.4:8080") + result = list(GeonodeFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestKuaidailiFetcher(object): + + @patch("fetcher.sources.kuaidaili.WebRequest") + @patch("fetcher.sources.kuaidaili.sleep", return_value=None) + def test_fetch(self, mock_sleep, mock_wr): + from fetcher.sources.kuaidaili import KuaidailiFetcher + # kuaidaili 使用 proxy_list[1:] 跳过第一行 + html = _html_table([("IP", "Port"), ("1.2.3.4", "8080")]) + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response(tree=tree) + result = list(KuaidailiFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestFreeVPNNodeFetcher(object): + + @patch("fetcher.sources.freevpnnode.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.freevpnnode import FreeVPNNodeFetcher + html = _html_table([("1.2.3.4", "8080")]) + tree = etree.HTML(html) + mock_wr.return_value.get.return_value = _make_response( + tree=tree, text="1.2.3.4:8080 5.6.7.8:3128") + result = list(FreeVPNNodeFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + +class TestScdnFetcher(object): + + @patch("fetcher.sources.scdn.WebRequest") + def test_fetch_json(self, mock_wr): + from fetcher.sources.scdn import ScdnFetcher + json_data = {"data": [{"ip": "1.2.3.4", "port": "8080"}]} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ScdnFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.scdn.WebRequest") + def test_fetch_table_html(self, mock_wr): + from fetcher.sources.scdn import ScdnFetcher + table_html = '1.2.3.48080' + json_data = {"table_html": table_html} + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ScdnFetcher().fetch()) + assert "1.2.3.4:8080" in result + + +class TestZdayeFetcher(object): + + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.sleep", return_value=None) + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_recent(self, mock_dt, mock_sleep, mock_wr): + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 模拟最新帖子时间在5分钟内 + mock_dt.now.return_value = real_datetime(2026, 5, 31, 12, 0, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 31, 11, 58, 0) + + index_tree = etree.HTML( + '2026/05/31 11:58:00' + '

test

') + detail_tree = etree.HTML(_html_table([("1.2.3.4", "8080")])) + + def side_effect(url, **kwargs): + resp = MagicMock() + if "free" in url: + resp.tree = index_tree + else: + resp.tree = detail_tree + return resp + + mock_wr.return_value.get.side_effect = side_effect + result = list(ZdayeFetcher().fetch()) + assert "1.2.3.4:8080" in result + + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_old_returns_empty(self, mock_dt, mock_wr): + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 模拟最新帖子时间超过5分钟 + mock_dt.now.return_value = real_datetime(2026, 5, 31, 12, 0, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 31, 10, 0, 0) + + index_tree = etree.HTML( + '2026/05/31 10:00:00' + '

test

') + mock_wr.return_value.get.return_value = _make_response(tree=index_tree) + result = list(ZdayeFetcher().fetch()) + assert result == [] + + @patch("fetcher.sources.zdaye.WebRequest") + @patch("fetcher.sources.zdaye.datetime") + def test_fetch_old_cross_day_returns_empty(self, mock_dt, mock_wr): + """跨天帖子应判定为过期（total_seconds 而非 seconds）""" + from fetcher.sources.zdaye import ZdayeFetcher + from datetime import datetime as real_datetime + # 帖子是昨天 23:59，当前是今天 00:01（差 2 分钟，但跨天） + mock_dt.now.return_value = real_datetime(2026, 5, 31, 0, 1, 0) + mock_dt.strptime.return_value = real_datetime(2026, 5, 30, 23, 59, 0) + + index_tree = etree.HTML( + '2026/05/30 23:59:00' + '

test

') + mock_wr.return_value.get.return_value = _make_response(tree=index_tree) + result = list(ZdayeFetcher().fetch()) + assert result == [] + + +class TestIhuanFetcher(object): + + @patch("fetcher.sources.ihuan.requests") + def test_fetch(self, mock_requests): + from fetcher.sources.ihuan import IhuanFetcher + html = ( + '' + '' + '' + '
1.2.3.48080
5.6.7.83128
' + ) + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.text = html + # 第一次 get 获取 cookie，第二次 get 返回数据 + mock_session.get.return_value = mock_resp + mock_requests.session.return_value = mock_session + + result = list(IhuanFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + assert mock_session.get.call_count == 2 + + @patch("fetcher.sources.ihuan.requests") + def test_fetch_empty_table_returns_empty(self, mock_requests): + from fetcher.sources.ihuan import IhuanFetcher + html = '
' + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.text = html + mock_session.get.return_value = mock_resp + mock_requests.session.return_value = mock_session + + result = list(IhuanFetcher().fetch()) + assert result == [] + + +class TestProxiFlyFetcher(object): + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "5.6.7.8:3128", "protocol": "http", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_cn(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "http", "geolocation": {"country": "US"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_http(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "https", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result + + +class TestDaiLi66Fetcher(object): + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + json_data = { + "data": [ + {"ip": "1.2.3.4", "port": "8080"}, + {"ip": "5.6.7.8", "port": "3128"}, + ] + } + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(DaiLi66Fetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch_empty_data_returns_empty(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + mock_wr.return_value.get.return_value = _make_response(json_data={}) + result = list(DaiLi66Fetcher().fetch()) + assert result == [] + + +class TestRoundProxiesFetcher(object): + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + json_data = { + "data": [ + {"ip": "1.2.3.4", "port": "8080"}, + {"ip": "5.6.7.8", "port": "3128"}, + ] + } + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(RoundProxiesFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch_empty_data_returns_empty(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + mock_wr.return_value.get.return_value = _make_response(json_data={}) + result = list(RoundProxiesFetcher().fetch()) + assert result == [] + + @patch("fetcher.sources.roundproxies.WebRequest") + def test_fetch_exception_returns_empty(self, mock_wr): + from fetcher.sources.roundproxies import RoundProxiesFetcher + mock_wr.return_value.get.return_value = _make_response(json_data=None) + result = list(RoundProxiesFetcher().fetch()) + assert result == [] diff --git a/tests/unit/test_launcher.py b/tests/unit/test_launcher.py new file mode 100644 index 000000000..20ceb9ea4 --- /dev/null +++ b/tests/unit/test_launcher.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_launcher.py + Description : helper/launcher.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock + +import helper.launcher as launcher_mod + + +class TestStartServer: + + def test_calls_before_start_then_flask(self): + """startServer 先调用 __beforeStart 再调用 runFlask""" + with patch.object(launcher_mod, "__beforeStart") as mock_before, \ + patch("api.proxyApi.runFlask") as mock_flask: + launcher_mod.startServer() + mock_before.assert_called_once() + + +class TestStartScheduler: + + def test_calls_before_start_then_scheduler(self): + """startScheduler 先调用 __beforeStart 再调用 runScheduler""" + with patch.object(launcher_mod, "__beforeStart") as mock_before, \ + patch("helper.scheduler.runScheduler") as mock_sched: + launcher_mod.startScheduler() + mock_before.assert_called_once() + + +class TestBeforeStart: + + def test_exits_when_db_check_fails(self): + """DB 检查失败 -> sys.exit()""" + with patch.object(launcher_mod, "__showVersion"), \ + patch.object(launcher_mod, "__showConfigure"), \ + patch.object(launcher_mod, "__checkDBConfig", return_value=True), \ + patch("helper.launcher.sys") as mock_sys: + getattr(launcher_mod, "__beforeStart")() + mock_sys.exit.assert_called_once() + + def test_continues_when_db_check_passes(self): + """DB 检查通过 -> 不调用 sys.exit""" + with patch.object(launcher_mod, "__showVersion"), \ + patch.object(launcher_mod, "__showConfigure"), \ + patch.object(launcher_mod, "__checkDBConfig", return_value=False), \ + patch("helper.launcher.sys") as mock_sys: + getattr(launcher_mod, "__beforeStart")() + mock_sys.exit.assert_not_called() + + +class TestCheckDBConfig: + + def test_returns_db_test_result(self): + """返回 db.test() 的结果""" + with patch.object(launcher_mod, "DbClient") as mock_db_cls, \ + patch.object(launcher_mod, "ConfigHandler") as mock_conf_cls: + mock_conf = MagicMock() + mock_conf.dbConn = "redis://:@127.0.0.1:6379/0" + mock_conf_cls.return_value = mock_conf + + mock_db = MagicMock() + mock_db.test.return_value = False + mock_db_cls.return_value = mock_db + + result = getattr(launcher_mod, "__checkDBConfig")() + + assert result is False + mock_db.test.assert_called_once() \ No newline at end of file diff --git a/tests/unit/test_log_handler.py b/tests/unit/test_log_handler.py new file mode 100644 index 000000000..10f099466 --- /dev/null +++ b/tests/unit/test_log_handler.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_log_handler.py + Description : LogHandler 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import logging +import pytest +from unittest.mock import patch, MagicMock + +from handler.logHandler import LogHandler, DEBUG, INFO, ERROR + + +class TestLogHandlerInit: + """__init__ 测试""" + + def test_default_creates_stream_handler(self): + """默认参数创建 stream handler""" + log = LogHandler("test_default_stream", stream=True, file=False) + handler_types = [type(h) for h in log.handlers] + assert logging.StreamHandler in handler_types + + @patch("handler.logHandler.platform") + def test_file_handler_on_linux(self, mock_platform): + """Linux 下创建 file handler""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_linux_file", stream=False, file=True) + has_file = any(isinstance(h, logging.handlers.TimedRotatingFileHandler) for h in log.handlers) + assert has_file + + @patch("handler.logHandler.platform") + def test_no_file_handler_on_windows(self, mock_platform): + """Windows 下不创建 file handler""" + mock_platform.system.return_value = "Windows" + log = LogHandler("test_windows_no_file", stream=False, file=True) + has_file = any(isinstance(h, logging.handlers.TimedRotatingFileHandler) for h in log.handlers) + assert not has_file + + def test_no_stream_handler_when_disabled(self): + """stream=False 时不创建 stream handler""" + log = LogHandler("test_no_stream", stream=False, file=False) + assert len(log.handlers) == 0 + + +class TestLogHandlerStreamLevel: + """stream handler level 测试""" + + def test_default_level_used_when_no_override(self): + """未指定 level 时使用 self.level""" + log = LogHandler("test_stream_level", level=ERROR, stream=True, file=False) + stream_handlers = [h for h in log.handlers if isinstance(h, logging.StreamHandler) + and not isinstance(h, logging.handlers.TimedRotatingFileHandler)] + assert len(stream_handlers) > 0 + assert stream_handlers[0].level == ERROR + + def test_explicit_level_overrides_default(self): + """显式指定 level 覆盖默认值""" + log = LogHandler("test_stream_override", level=DEBUG, stream=True, file=False) + log.__setStreamHandler__(level=ERROR) + # 最后添加的 handler 应该是 ERROR 级别 + last_handler = log.handlers[-1] + assert last_handler.level == ERROR + + +class TestLogHandlerFileLevel: + """file handler level 测试""" + + @patch("handler.logHandler.platform") + def test_file_handler_default_level(self, mock_platform): + """file handler 未指定 level 时使用 self.level""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_file_level", level=INFO, stream=False, file=True) + file_handlers = [h for h in log.handlers + if isinstance(h, logging.handlers.TimedRotatingFileHandler)] + assert len(file_handlers) > 0 + assert file_handlers[0].level == INFO + + @patch("handler.logHandler.platform") + def test_file_handler_explicit_level(self, mock_platform): + """file handler 显式指定 level""" + mock_platform.system.return_value = "Linux" + log = LogHandler("test_file_override", level=DEBUG, stream=False, file=True) + log.__setFileHandler__(level=ERROR) + last_handler = log.handlers[-1] + assert last_handler.level == ERROR + + +class TestLogHandlerDirCreation: + """log 目录创建测试""" + + @patch("os.path.exists", return_value=False) + @patch("os.mkdir") + def test_creates_log_dir_when_missing(self, mock_mkdir, mock_exists): + """log 目录不存在时创建""" + # 重新 import 触发模块级代码（无法直接测试，验证模块级逻辑） + # 这里测试的是 FileExistsError 处理 + import handler.logHandler as lh + # 模块加载时已执行，此处验证 LOG_PATH 存在 + assert lh.LOG_PATH is not None + + @patch("os.path.exists", return_value=False) + @patch("os.mkdir", side_effect=FileExistsError) + def test_handles_file_exists_race_condition(self, mock_mkdir, mock_exists): + """处理 mkdir 时的 FileExistsError 竞态条件""" + # 验证模块级代码不会因 FileExistsError 崩溃 + import handler.logHandler as lh + assert lh.LOG_PATH is not None \ No newline at end of file diff --git a/tests/unit/test_proxy.py b/tests/unit/test_proxy.py new file mode 100644 index 000000000..d8f1bb680 --- /dev/null +++ b/tests/unit/test_proxy.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testProxy.py + Description : Proxy类单元测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import json +import pytest +from helper.proxy import Proxy + + +class TestProxyInit: + """Proxy 构造测试""" + + def test_default_values(self): + p = Proxy("1.2.3.4:8080") + assert p.proxy == "1.2.3.4:8080" + assert p.fail_count == 0 + assert p.region == "" + assert p.anonymous == "" + assert p.source == "" + assert p.check_count == 0 + assert p.last_status == "" + assert p.last_time == "" + assert p.https is False + + def test_custom_values(self): + p = Proxy( + "5.6.7.8:443", + fail_count=3, + region="US", + anonymous="high", + source="freeProxy01", + check_count=10, + last_status=True, + last_time="2024-01-01 00:00:00", + https=True, + ) + assert p.proxy == "5.6.7.8:443" + assert p.fail_count == 3 + assert p.region == "US" + assert p.anonymous == "high" + assert p.source == "freeProxy01" + assert p.check_count == 10 + assert p.last_status is True + assert p.last_time == "2024-01-01 00:00:00" + assert p.https is True + + def test_source_with_slash(self): + """source 含 / 时应被拆分为列表，读回时用 / 连接""" + p = Proxy("1.2.3.4:8080", source="freeProxy01/freeProxy02") + assert p.source == "freeProxy01/freeProxy02" + + +class TestProxySerialization: + """序列化 / 反序列化测试""" + + def test_to_dict_keys(self): + p = Proxy("1.2.3.4:8080") + d = p.to_dict + expected_keys = {"proxy", "https", "fail_count", "region", "anonymous", + "source", "check_count", "last_status", "last_time"} + assert set(d.keys()) == expected_keys + + def test_to_dict_values(self): + p = Proxy("1.2.3.4:8080", source="test", https=True) + d = p.to_dict + assert d["proxy"] == "1.2.3.4:8080" + assert d["https"] is True + assert d["source"] == "test" + assert d["fail_count"] == 0 + + def test_to_json_is_valid_json(self): + p = Proxy("1.2.3.4:8080", source="test") + j = p.to_json + d = json.loads(j) + assert d["proxy"] == "1.2.3.4:8080" + + def test_create_from_json_roundtrip(self): + """to_json -> createFromJson 往返一致性""" + original = Proxy("10.0.0.1:3128", source="freeProxy01/freeProxy02", + https=True, fail_count=2, region="CN") + restored = Proxy.createFromJson(original.to_json) + assert restored.proxy == original.proxy + assert restored.https == original.https + assert restored.fail_count == original.fail_count + assert restored.region == original.region + assert restored.source == original.source + + def test_create_from_json_minimal(self): + """createFromJson 缺少字段时使用默认值""" + j = '{"proxy": "1.2.3.4:8080"}' + p = Proxy.createFromJson(j) + assert p.proxy == "1.2.3.4:8080" + assert p.fail_count == 0 + assert p.https is False + + def test_create_from_json_with_slash_source(self): + """source 含 / 的 JSON 反序列化""" + j = '{"proxy": "1.2.3.4:8080", "source": "freeProxy01/freeProxy02", "https": false}' + p = Proxy.createFromJson(j) + assert p.source == "freeProxy01/freeProxy02" + + def test_to_dict_to_json_consistency(self): + """to_dict 和 to_json 数据一致""" + p = Proxy("1.2.3.4:8080", source="test", https=True, fail_count=1) + d = p.to_dict + j = json.loads(p.to_json) + assert d == j + + +class TestProxySetters: + """setter 测试""" + + def test_fail_count_setter(self): + p = Proxy("1.2.3.4:8080") + p.fail_count = 5 + assert p.fail_count == 5 + + def test_check_count_setter(self): + p = Proxy("1.2.3.4:8080") + p.check_count = 10 + assert p.check_count == 10 + + def test_last_status_setter(self): + p = Proxy("1.2.3.4:8080") + p.last_status = True + assert p.last_status is True + + def test_last_time_setter(self): + p = Proxy("1.2.3.4:8080") + p.last_time = "2024-01-01 12:00:00" + assert p.last_time == "2024-01-01 12:00:00" + + def test_https_setter(self): + p = Proxy("1.2.3.4:8080") + p.https = True + assert p.https is True + + def test_region_setter(self): + p = Proxy("1.2.3.4:8080") + p.region = "US" + assert p.region == "US" + + +class TestProxyAddSource: + """add_source 测试""" + + def test_add_source(self): + p = Proxy("1.2.3.4:8080", source="src1") + p.add_source("src2") + assert "src1" in p.source + assert "src2" in p.source + + def test_add_source_dedup(self): + """重复 source 不应重复添加""" + p = Proxy("1.2.3.4:8080", source="src1") + p.add_source("src1") + assert p.source.count("src1") == 1 + + def test_add_source_empty_string(self): + """空字符串不应添加""" + p = Proxy("1.2.3.4:8080", source="src1") + p.add_source("") + assert p.source == "src1" + + def test_add_source_none(self): + """None 不应添加""" + p = Proxy("1.2.3.4:8080", source="src1") + p.add_source(None) + assert p.source == "src1" \ No newline at end of file diff --git a/tests/unit/test_proxy_handler.py b/tests/unit/test_proxy_handler.py new file mode 100644 index 000000000..441b89a75 --- /dev/null +++ b/tests/unit/test_proxy_handler.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_proxy_handler.py + Description : ProxyHandler 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import MagicMock, patch + +from handler.proxyHandler import ProxyHandler +from helper.proxy import Proxy + + +def _make_handler(): + """构造注入 mock DbClient 的 ProxyHandler 实例""" + with patch("handler.proxyHandler.DbClient") as mock_db_cls, \ + patch("handler.proxyHandler.ConfigHandler") as mock_conf_cls: + mock_db = MagicMock() + mock_db_cls.return_value = mock_db + mock_conf = MagicMock() + mock_conf.dbConn = "redis://:test@127.0.0.1:6379/0" + mock_conf.tableName = "use_proxy" + mock_conf_cls.return_value = mock_conf + handler = ProxyHandler() + handler._mock_db = mock_db + return handler + + +class TestProxyHandlerGet: + """get() 测试""" + + def test_get_returns_proxy(self): + """DbClient 返回 JSON -> Proxy 对象""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test", https=False) + handler._mock_db.get.return_value = proxy.to_json + + result = handler.get(https=False) + + assert result is not None + assert result.proxy == "1.2.3.4:8080" + assert result.https is False + handler._mock_db.get.assert_called_once_with(False) + + def test_get_returns_none_when_empty(self): + """DbClient 返回 None -> None""" + handler = _make_handler() + handler._mock_db.get.return_value = None + + result = handler.get(https=False) + + assert result is None + + def test_get_https_forwarded(self): + """https=True 转发给 DbClient""" + handler = _make_handler() + proxy = Proxy("5.6.7.8:443", source="test", https=True) + handler._mock_db.get.return_value = proxy.to_json + + result = handler.get(https=True) + + assert result is not None + assert result.https is True + handler._mock_db.get.assert_called_once_with(True) + + +class TestProxyHandlerPop: + """pop() 测试""" + + def test_pop_returns_proxy(self): + """pop 正常返回 Proxy 对象""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + handler._mock_db.pop.return_value = proxy.to_json + + result = handler.pop(https=False) + + assert result is not None + assert result.proxy == "1.2.3.4:8080" + handler._mock_db.pop.assert_called_once_with(False) + + def test_pop_returns_none_when_empty(self): + """pop 无数据时返回 None""" + handler = _make_handler() + handler._mock_db.pop.return_value = None + + result = handler.pop(https=False) + + assert result is None + + +class TestProxyHandlerPut: + """put() 测试""" + + def test_put_delegates_to_db(self): + """put 调用 DbClient.put""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + + handler.put(proxy) + + handler._mock_db.put.assert_called_once_with(proxy) + + +class TestProxyHandlerDelete: + """delete() 测试""" + + def test_delete_delegates_to_db(self): + """delete 传入 proxy.proxy 字符串给 DbClient""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + + handler.delete(proxy) + + handler._mock_db.delete.assert_called_once_with("1.2.3.4:8080") + + +class TestProxyHandlerGetAll: + """getAll() 测试""" + + def test_getAll_returns_proxy_list(self): + """getAll 返回 Proxy 对象列表""" + handler = _make_handler() + proxy1 = Proxy("1.2.3.4:8080", source="test").to_json + proxy2 = Proxy("5.6.7.8:443", source="test", https=True).to_json + handler._mock_db.getAll.return_value = [proxy1, proxy2] + + result = handler.getAll(https=False) + + assert len(result) == 2 + assert result[0].proxy == "1.2.3.4:8080" + assert result[1].proxy == "5.6.7.8:443" + handler._mock_db.getAll.assert_called_once_with(False) + + def test_getAll_empty_returns_empty_list(self): + """getAll 无数据返回空列表""" + handler = _make_handler() + handler._mock_db.getAll.return_value = [] + + result = handler.getAll() + + assert result == [] + + +class TestProxyHandlerExists: + """exists() 测试""" + + def test_exists_delegates_to_db(self): + """exists 传入 proxy.proxy 字符串给 DbClient""" + handler = _make_handler() + proxy = Proxy("1.2.3.4:8080", source="test") + handler._mock_db.exists.return_value = True + + result = handler.exists(proxy) + + assert result is True + handler._mock_db.exists.assert_called_once_with("1.2.3.4:8080") + + +class TestProxyHandlerGetCount: + """getCount() 测试""" + + def test_getCount_returns_dict(self): + """getCount 返回 {'count': N}""" + handler = _make_handler() + handler._mock_db.getCount.return_value = 42 + + result = handler.getCount() + + assert result == {"count": 42} \ No newline at end of file diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py new file mode 100644 index 000000000..81c6e4e3d --- /dev/null +++ b/tests/unit/test_scheduler.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_scheduler.py + Description : helper/scheduler.py 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys +import pytest +from unittest.mock import patch, MagicMock + + +# apscheduler 依赖 pkg_resources，在 tox/uv 环境中可能缺失 +# 在 import 前 mock 掉，避免 collection 阶段报错 +_apscheduler_mock = MagicMock() +sys.modules.setdefault("apscheduler", _apscheduler_mock) +sys.modules.setdefault("apscheduler.schedulers", _apscheduler_mock.schedulers) +sys.modules.setdefault("apscheduler.schedulers.blocking", _apscheduler_mock.schedulers.blocking) +sys.modules.setdefault("apscheduler.executors", _apscheduler_mock.executors) +sys.modules.setdefault("apscheduler.executors.pool", _apscheduler_mock.executors.pool) + +import helper.scheduler as scheduler_mod + + +def _get_attr(name): + """获取模块中双下划线开头的属性（绕过类内 name mangling）""" + return getattr(scheduler_mod, name) + + +class TestRunProxyFetch: + + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.Fetcher") + def test_fetcher_yields_go_to_queue(self, mock_fetcher_cls, mock_checker): + """Fetcher yield 的代理放入 queue，传给 Checker""" + mock_proxy = MagicMock() + mock_fetcher = MagicMock() + mock_fetcher.run.return_value = iter([mock_proxy]) + mock_fetcher_cls.return_value = mock_fetcher + + _get_attr("__runProxyFetch")() + + mock_fetcher_cls.assert_called_once() + mock_checker.assert_called_once() + call_args = mock_checker.call_args + assert call_args[0][0] == "raw" + + +class TestRunProxyCheck: + + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.ProxyHandler") + def test_triggers_fetch_when_pool_low(self, mock_ph_cls, mock_checker, mock_fetch): + """count < poolSizeMin -> 触发 __runProxyFetch""" + mock_ph = MagicMock() + mock_ph.db.getCount.return_value = {"total": 5} + mock_ph.conf.poolSizeMin = 20 + mock_ph.getAll.return_value = [] + mock_ph_cls.return_value = mock_ph + + _get_attr("__runProxyCheck")() + + mock_fetch.assert_called_once() + + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.Checker") + @patch("helper.scheduler.ProxyHandler") + def test_skips_fetch_when_pool_sufficient(self, mock_ph_cls, mock_checker, mock_fetch): + """count >= poolSizeMin -> 不触发 __runProxyFetch""" + mock_ph = MagicMock() + mock_ph.db.getCount.return_value = {"total": 50} + mock_ph.conf.poolSizeMin = 20 + mock_ph.getAll.return_value = [] + mock_ph_cls.return_value = mock_ph + + _get_attr("__runProxyCheck")() + + mock_fetch.assert_not_called() + + +class TestRunScheduler: + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_adds_two_jobs(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """runScheduler 添加两个定时任务""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + assert mock_sched.add_job.call_count == 2 + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_fetch_job_interval_5min(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """采集任务间隔 5 分钟""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + calls = mock_sched.add_job.call_args_list + first_call = calls[0] + assert first_call[0][1] == "interval" + assert first_call[1]["minutes"] == 5 + + @patch("helper.scheduler.BlockingScheduler") + @patch("helper.scheduler.__runProxyFetch") + @patch("helper.scheduler.ConfigHandler") + @patch("helper.scheduler.LogHandler") + def test_check_job_interval_2min(self, mock_log, mock_conf_cls, mock_fetch, mock_sched_cls): + """检查任务间隔 2 分钟""" + mock_conf = MagicMock() + mock_conf.timezone = "Asia/Shanghai" + mock_conf_cls.return_value = mock_conf + mock_sched = MagicMock() + mock_sched_cls.return_value = mock_sched + + scheduler_mod.runScheduler() + + calls = mock_sched.add_job.call_args_list + second_call = calls[1] + assert second_call[0][1] == "interval" + assert second_call[1]["minutes"] == 2 \ No newline at end of file diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py new file mode 100644 index 000000000..effe0e24d --- /dev/null +++ b/tests/unit/test_validator.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： testValidator.py + Description : formatValidator正则测试 + Author : JHao + date： 2026/5/28 +------------------------------------------------- + Change Activity: + 2026/05/28: +------------------------------------------------- +""" +__author__ = 'JHao' + +import re +import pytest +from unittest.mock import patch, MagicMock + +# 直接导入 IP_REGEX 和 formatValidator，不导入整个 validator 模块（避免模块级副作用） +from helper.validator import IP_REGEX, formatValidator, httpTimeOutValidator, httpsTimeOutValidator, customValidatorExample + + +class TestIPRegex: + + @pytest.mark.parametrize("proxy", [ + "1.2.3.4:8080", + "192.168.1.1:3128", + "10.0.0.1:80", + "255.255.255.255:65535", + "0.0.0.0:1", + "1.2.3.4:99999", # regex 不校验端口范围 + "999.1.1.1:80", # regex 不校验 IP 范围 + "user:pass@1.2.3.4:8080", + "admin:secret@192.168.1.1:443", + ]) + def test_valid_proxy_format(self, proxy): + assert IP_REGEX.fullmatch(proxy) is not None, f"应匹配: {proxy}" + + @pytest.mark.parametrize("proxy", [ + "", + "abc", + "1.2.3.4", + "1.2.3.4:", + ":8080", + "1.2.3.4:abc", + "1.2.3.4:8080:extra", + "host:8080", + ]) + def test_invalid_proxy_format(self, proxy): + assert IP_REGEX.fullmatch(proxy) is None, f"不应匹配: {proxy}" + + +class TestFormatValidator: + + @pytest.mark.parametrize("proxy", [ + "1.2.3.4:8080", + "192.168.1.1:3128", + "user:pass@10.0.0.1:80", + ]) + def test_valid_returns_true(self, proxy): + assert formatValidator(proxy) is True + + @pytest.mark.parametrize("proxy", [ + "", + "abc", + "1.2.3.4", + ]) + def test_invalid_returns_false(self, proxy): + assert formatValidator(proxy) is False + + +class TestHttpTimeOutValidator: + """httpTimeOutValidator 测试""" + + @patch("helper.validator.head") + def test_returns_true_on_200(self, mock_head): + """status_code=200 -> True""" + mock_head.return_value = MagicMock(status_code=200) + assert httpTimeOutValidator("1.2.3.4:8080") is True + + @patch("helper.validator.head") + def test_returns_false_on_non_200(self, mock_head): + """status_code=502 -> False""" + mock_head.return_value = MagicMock(status_code=502) + assert httpTimeOutValidator("1.2.3.4:8080") is False + + @patch("helper.validator.head") + def test_returns_false_on_exception(self, mock_head): + """head() raise Timeout -> False""" + mock_head.side_effect = TimeoutError("connection timed out") + assert httpTimeOutValidator("1.2.3.4:8080") is False + + +class TestHttpsTimeOutValidator: + """httpsTimeOutValidator 测试""" + + @patch("helper.validator.head") + def test_returns_true_on_200(self, mock_head): + """status_code=200 -> True""" + mock_head.return_value = MagicMock(status_code=200) + assert httpsTimeOutValidator("1.2.3.4:8080") is True + # 验证 verify=False 被传递 + call_kwargs = mock_head.call_args + assert call_kwargs[1]["verify"] is False + + @patch("helper.validator.head") + def test_returns_false_on_non_200(self, mock_head): + """status_code=502 -> False""" + mock_head.return_value = MagicMock(status_code=502) + assert httpsTimeOutValidator("1.2.3.4:8080") is False + + @patch("helper.validator.head") + def test_returns_false_on_exception(self, mock_head): + """head() raise Timeout -> False""" + mock_head.side_effect = TimeoutError("connection timed out") + assert httpsTimeOutValidator("1.2.3.4:8080") is False + + +class TestCustomValidatorExample: + """customValidatorExample 测试""" + + def test_always_returns_true(self): + """customValidatorExample 始终返回 True""" + assert customValidatorExample("1.2.3.4:8080") is True \ No newline at end of file diff --git a/tests/unit/test_web_request.py b/tests/unit/test_web_request.py new file mode 100644 index 000000000..a62aad79d --- /dev/null +++ b/tests/unit/test_web_request.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： test_web_request.py + Description : WebRequest 单元测试 + Author : JHao + date： 2026/6/15 +------------------------------------------------- + Change Activity: + 2026/06/15: +------------------------------------------------- +""" +__author__ = 'JHao' + +import pytest +from unittest.mock import patch, MagicMock +from requests.models import Response + +from util.webRequest import WebRequest + + +def _mock_response(status_code=200, text=None, content=None, json_data=None): + """构造 mock Response""" + resp = Response() + resp.status_code = status_code + if json_data is not None: + import json + resp._content = json.dumps(json_data).encode("utf-8") + resp.json = lambda: json_data + elif content is not None: + resp._content = content + elif text is not None: + resp._content = text.encode("utf-8") + else: + resp._content = b"ok" + return resp + + +class TestWebRequestGet: + """get() 测试""" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_success_path(self, mock_get, mock_sleep): + """正常返回 -> self.response 被设置""" + mock_get.return_value = _mock_response(200, "hello") + wr = WebRequest() + result = wr.get("http://example.com", retry_time=1, retry_interval=0, timeout=1) + + assert result is wr + assert wr.response.status_code == 200 + assert wr.text == "hello" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_custom_header_merge(self, mock_get, mock_sleep): + """自定义 header 合并到默认 header""" + mock_get.return_value = _mock_response(200) + wr = WebRequest() + wr.get("http://example.com", header={"X-Custom": "v"}, retry_time=1, retry_interval=0, timeout=1) + + call_kwargs = mock_get.call_args[1] + assert call_kwargs["headers"]["X-Custom"] == "v" + assert "User-Agent" in call_kwargs["headers"] + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.get") + def test_retry_exhaustion(self, mock_get, mock_sleep): + """全部失败 -> 返回 fallback（注意 get() 的 bug: 未赋值 self.response）""" + mock_get.side_effect = TimeoutError("timeout") + wr = WebRequest() + result = wr.get("http://example.com", retry_time=2, retry_interval=0, timeout=1) + + assert result is wr + # 注意：get() 在 retry 耗尽时创建了 resp 但未赋值给 self.response + # 所以 self.response 仍为初始的空 Response + assert mock_get.call_count == 2 + + +class TestWebRequestPost: + """post() 测试""" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_success_path(self, mock_post, mock_sleep): + """正常返回 -> self.response 被设置""" + mock_post.return_value = _mock_response(200, "posted") + wr = WebRequest() + result = wr.post("http://example.com", retry_time=1, retry_interval=0, timeout=1) + + assert result is wr + assert wr.response.status_code == 200 + assert wr.text == "posted" + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_custom_header_merge(self, mock_post, mock_sleep): + """自定义 header 合并到默认 header""" + mock_post.return_value = _mock_response(200) + wr = WebRequest() + wr.post("http://example.com", header={"X-Custom": "v"}, retry_time=1, retry_interval=0, timeout=1) + + call_kwargs = mock_post.call_args[1] + assert call_kwargs["headers"]["X-Custom"] == "v" + assert "User-Agent" in call_kwargs["headers"] + + @patch("util.webRequest.time.sleep") + @patch("util.webRequest.requests.post") + def test_retry_exhaustion(self, mock_post, mock_sleep): + """全部失败 -> self.response 被正确赋值（与 get() 不同）""" + mock_post.side_effect = TimeoutError("timeout") + wr = WebRequest() + result = wr.post("http://example.com", retry_time=2, retry_interval=0, timeout=1) + + assert result is wr + # post() 正确赋值 self.response = resp + assert wr.response.status_code == 200 + assert mock_post.call_count == 2 + + +class TestWebRequestTree: + """tree 属性测试""" + + def test_empty_content_returns_none(self): + """空 content -> None""" + wr = WebRequest() + wr.response = _mock_response(200, content=b"") + assert wr.tree is None + + def test_valid_html_returns_element(self): + """有效 HTML -> lxml element""" + wr = WebRequest() + html = b"

hello

" + wr.response = _mock_response(200, content=html) + tree = wr.tree + assert tree is not None + assert tree.xpath("//p/text()") == ["hello"] + + +class TestWebRequestText: + """text 属性测试""" + + def test_returns_response_text(self): + """返回 response.text""" + wr = WebRequest() + wr.response = _mock_response(200, text="hello world") + assert wr.text == "hello world" + + +class TestWebRequestJson: + """json 属性测试""" + + def test_valid_json_returns_dict(self): + """有效 JSON -> dict""" + wr = WebRequest() + wr.response = _mock_response(200, json_data={"key": "val"}) + assert wr.json == {"key": "val"} + + def test_invalid_json_returns_empty_dict(self): + """无效 JSON -> {}""" + wr = WebRequest() + resp = _mock_response(200, content=b"not json") + resp.json = lambda: (_ for _ in ()).throw(ValueError("Invalid JSON")) + wr.response = resp + assert wr.json == {} + + +class TestWebRequestProperties: + """header/user_agent 属性测试""" + + def test_user_agent_returns_string(self): + wr = WebRequest() + ua = wr.user_agent + assert isinstance(ua, str) + assert len(ua) > 0 + + def test_header_contains_user_agent(self): + wr = WebRequest() + h = wr.header + assert "User-Agent" in h + assert "Accept" in h + assert "Connection" in h diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..aa50babb5 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py38,py39,py310,py311 +skip_missing_interpreters = true + +[testenv] +skip_install = true +recreate = true +deps = + -r requirements.txt + -r requirements-test.txt +commands = pytest \ No newline at end of file diff --git a/Util/__init__.py b/util/__init__.py similarity index 56% rename from Util/__init__.py rename to util/__init__.py index d1c5cc292..4a81052c3 100644 --- a/Util/__init__.py +++ b/util/__init__.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- """ ------------------------------------------------- - File Name： __init__.py.py - Description : - Author : JHao - date： 2016/11/25 + File Name： __init__ + Description : + Author : JHao + date： 2020/7/6 ------------------------------------------------- Change Activity: - 2016/11/25: + 2020/7/6: ------------------------------------------------- -""" \ No newline at end of file +""" +__author__ = 'JHao' diff --git a/util/lazyProperty.py b/util/lazyProperty.py new file mode 100644 index 000000000..f028192d2 --- /dev/null +++ b/util/lazyProperty.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： lazyProperty + Description : + Author : JHao + date： 2016/12/3 +------------------------------------------------- + Change Activity: + 2016/12/3: +------------------------------------------------- +""" +__author__ = 'JHao' + + +class LazyProperty(object): + """ + LazyProperty + explain: http://www.spiderpy.cn/blog/5/ + """ + + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + if instance is None: + return self + else: + value = self.func(instance) + setattr(instance, self.func.__name__, value) + return value diff --git a/util/singleton.py b/util/singleton.py new file mode 100644 index 000000000..1abb7a7c3 --- /dev/null +++ b/util/singleton.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： singleton + Description : + Author : JHao + date： 2016/12/3 +------------------------------------------------- + Change Activity: + 2016/12/3: +------------------------------------------------- +""" +__author__ = 'JHao' + + +class Singleton(type): + """ + Singleton Metaclass + """ + + _inst = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._inst: + cls._inst[cls] = super(Singleton, cls).__call__(*args) + return cls._inst[cls] diff --git a/util/six.py b/util/six.py new file mode 100644 index 000000000..d31e12138 --- /dev/null +++ b/util/six.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： six + Description : + Author : JHao + date： 2020/6/22 +------------------------------------------------- + Change Activity: + 2020/6/22: +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + def iteritems(d, **kw): + return iter(d.items(**kw)) +else: + def iteritems(d, **kw): + return d.iteritems(**kw) + +if PY3: + from urllib.parse import urlparse +else: + from urlparse import urlparse + +if PY3: + try: + from importlib import reload as reload_six + except ImportError: + from imp import reload as reload_six +else: + reload_six = reload + +if PY3: + from queue import Empty, Queue +else: + from Queue import Empty, Queue + + +def withMetaclass(meta, *bases): + """Create a base class with a metaclass.""" + + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class MetaClass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + return type.__new__(MetaClass, 'temporary_class', (), {}) diff --git a/util/webRequest.py b/util/webRequest.py new file mode 100644 index 000000000..97164773a --- /dev/null +++ b/util/webRequest.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： WebRequest + Description : Network Requests Class + Author : J_hao + date： 2017/7/31 +------------------------------------------------- + Change Activity: + 2017/7/31: +------------------------------------------------- +""" +__author__ = 'J_hao' + +from requests.models import Response +from lxml import etree +import requests +import random +import time + +from handler.logHandler import LogHandler + +requests.packages.urllib3.disable_warnings() + + +class WebRequest(object): + name = "web_request" + + def __init__(self, *args, **kwargs): + self.log = LogHandler(self.name, file=False) + self.response = Response() + + @property + def user_agent(self): + """ + return an User-Agent at random + :return: + """ + ua_list = [ + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)', + 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', + ] + return random.choice(ua_list) + + @property + def header(self): + """ + basic header + :return: + """ + return {'User-Agent': self.user_agent, + 'Accept': '*/*', + 'Connection': 'keep-alive', + 'Accept-Language': 'zh-CN,zh;q=0.8'} + + def get(self, url, header=None, retry_time=3, retry_interval=5, timeout=5, *args, **kwargs): + """ + get method + :param url: target url + :param header: headers + :param retry_time: retry time + :param retry_interval: retry interval + :param timeout: network timeout + :return: + """ + headers = self.header + if header and isinstance(header, dict): + headers.update(header) + while True: + try: + self.response = requests.get(url, headers=headers, timeout=timeout, *args, **kwargs) + return self + except Exception as e: + self.log.error("requests: %s error: %s" % (url, str(e))) + retry_time -= 1 + if retry_time <= 0: + resp = Response() + resp.status_code = 200 + return self + self.log.info("retry %s second after" % retry_interval) + time.sleep(retry_interval) + + def post(self, url, header=None, retry_time=3, retry_interval=5, timeout=5, *args, **kwargs): + """ + post method + :param url: target url + :param header: headers + :param retry_time: retry time + :param retry_interval: retry interval + :param timeout: network timeout + :return: + """ + headers = self.header + if header and isinstance(header, dict): + headers.update(header) + while True: + try: + self.response = requests.post(url, headers=headers, timeout=timeout, *args, **kwargs) + return self + except Exception as e: + self.log.error("requests: %s error: %s" % (url, str(e))) + retry_time -= 1 + if retry_time <= 0: + resp = Response() + resp.status_code = 200 + self.response = resp + return self + self.log.info("retry %s second after" % retry_interval) + time.sleep(retry_interval) + + @property + def tree(self): + if not self.response.content: + return None + return etree.HTML(self.response.content) + + @property + def text(self): + return self.response.text + + @property + def json(self): + try: + return self.response.json() + except Exception as e: + self.log.error(str(e)) + return {}