From 71f6f039b0d62135bbbf97ab1d71c3eb6bc269e8 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 01:20:59 +0800 Subject: [PATCH 1/8] update docker-compose --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1922009..18e7277 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: redis: image: redis:alpine - container_name: redis + container_name: redis4accountpool command: redis-server ports: - "6379:6379" @@ -16,4 +16,4 @@ services: restart: always environment: REDIS_HOST: redis - WEBSITE: antispider6 \ No newline at end of file + WEBSITE: antispider7 \ No newline at end of file From e8da4544af57ae36ea871d6c80d810a9d3bef1b2 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 02:02:43 +0800 Subject: [PATCH 2/8] add register and fix --- accountpool/processors/generator.py | 41 ++++++++++++++++++++++++++--- accountpool/processors/tester.py | 27 +++++++++++++++++++ accountpool/scheduler.py | 2 +- accountpool/setting.py | 3 +++ docker-compose.yml | 9 +++---- importer.py | 8 +++++- register.py | 27 +++++++++++++++++++ 7 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 register.py diff --git a/accountpool/processors/generator.py b/accountpool/processors/generator.py index 418a2ba..48d6f9b 100644 --- a/accountpool/processors/generator.py +++ b/accountpool/processors/generator.py @@ -14,7 +14,7 @@ def __init__(self, website=None): raise InitException self.account_operator = RedisClient(type='account', website=self.website) self.credential_operator = RedisClient(type='credential', website=self.website) - + def generate(self, username, password): """ generate method @@ -23,13 +23,13 @@ def generate(self, username, password): :return: """ raise NotImplementedError - + def init(self): """ do init """ pass - + def run(self): """ run main process @@ -47,8 +47,40 @@ def run(self): import requests -class Antispider7Generator(BaseGenerator): +class Antispider6Generator(BaseGenerator): + + def init(self): + """ + do init + """ + if self.account_operator.count() == 0: + self.account_operator.set('admin', 'admin') + self.account_operator.set('admin2', 'admin2') + + def generate(self, username, password): + """ + generate main process + """ + if self.credential_operator.get(username): + logger.debug(f'credential of {username} exists, skip') + return + login_url = 'https://antispider6.scrape.center/login' + s = requests.Session() + s.post(login_url, data={ + 'username': username, + 'password': password + }) + result = [] + for cookie in s.cookies: + print(cookie.name, cookie.value) + result.append(f'{cookie.name}={cookie.value}') + result = ';'.join(result) + logger.debug(f'get credential {result}') + self.credential_operator.set(username, result) + +class Antispider7Generator(BaseGenerator): + def generate(self, username, password): """ generate main process @@ -63,6 +95,7 @@ def generate(self, username, password): 'password': password }) if r.status_code != 200: + logger.error(f'error occurred while generating credential of {username}, error code {r.status_code}') return token = r.json().get('token') logger.debug(f'get credential {token}') diff --git a/accountpool/processors/tester.py b/accountpool/processors/tester.py index 6dc7918..7cc44d9 100644 --- a/accountpool/processors/tester.py +++ b/accountpool/processors/tester.py @@ -36,6 +36,33 @@ def run(self): self.test(username, credential) +class Antispider6Tester(BaseTester): + """ + tester for antispider6 + """ + + def __init__(self, website=None): + BaseTester.__init__(self, website) + + def test(self, username, credential): + """ + test single credential + """ + logger.info(f'testing credential for {username}') + try: + test_url = TEST_URL_MAP[self.website] + response = requests.get(test_url, headers={ + 'Cookie': credential + }, timeout=5, allow_redirects=False) + if response.status_code == 200: + logger.info('credential is valid') + else: + logger.info('credential is not valid, delete it') + self.credential_operator.delete(username) + except ConnectionError: + logger.info('test failed') + + class Antispider7Tester(BaseTester): """ tester for antispider7 diff --git a/accountpool/scheduler.py b/accountpool/scheduler.py index 3767939..12f5b9d 100644 --- a/accountpool/scheduler.py +++ b/accountpool/scheduler.py @@ -60,7 +60,7 @@ def run_server(self, _): def run(self, website): global tester_process, generator_process, server_process try: - logger.info('starting account pool...') + logger.info(f'starting account pool for website {website}...') if ENABLE_TESTER: tester_process = multiprocessing.Process(target=self.run_tester, args=(website,)) logger.info(f'starting tester, pid {tester_process.pid}...') diff --git a/accountpool/setting.py b/accountpool/setting.py index e8b666e..3176275 100644 --- a/accountpool/setting.py +++ b/accountpool/setting.py @@ -42,11 +42,13 @@ # integrated generator GENERATOR_MAP = { + 'antispider6': 'Antispider6Generator', 'antispider7': 'Antispider7Generator' } # integrated tester TESTER_MAP = { + 'antispider6': 'Antispider6Tester', 'antispider7': 'Antispider7Tester', } @@ -62,6 +64,7 @@ TEST_BATCH = env.int('TEST_BATCH', 20) # test url TEST_URL_MAP = { + 'antispider6': 'https://antispider6.scrape.center/', 'antispider7': 'https://antispider7.scrape.center/' } diff --git a/docker-compose.yml b/docker-compose.yml index 18e7277..992ac82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,18 @@ version: '3' services: - redis: + redis4accountpool: image: redis:alpine container_name: redis4accountpool command: redis-server ports: - - "6379:6379" - restart: always + - "6333:6379" accountpool: build: . image: 'germey/accountpool' container_name: accountpool ports: - "6777:6777" - restart: always environment: - REDIS_HOST: redis + REDIS_HOST: redis4accountpool + REDIS_PORT: "6379" WEBSITE: antispider7 \ No newline at end of file diff --git a/importer.py b/importer.py index d388dad..917d328 100644 --- a/importer.py +++ b/importer.py @@ -1,6 +1,12 @@ from accountpool.storages.redis import RedisClient +import argparse -conn = RedisClient('account', 'antispider7') +parser = argparse.ArgumentParser(description='AccountPool') +parser.add_argument('website', type=str, help='website') +args = parser.parse_args() +website = args.website + +conn = RedisClient('account', args.website) start = 1 end = 100 for i in range(start, end + 1): diff --git a/register.py b/register.py new file mode 100644 index 0000000..2c94c4c --- /dev/null +++ b/register.py @@ -0,0 +1,27 @@ +import argparse +from acinonyx import run +import requests +from loguru import logger + +parser = argparse.ArgumentParser(description='AccountPool') +parser.add_argument('website', type=str, help='website') +args = parser.parse_args() +website = args.website + + +@logger.catch() +def register(username, password): + logger.debug(f'register using {username} and {password}') + response = requests.post('https://antispider7.scrape.center/api/register', json={ + 'username': username, + 'password': password + }) + print(response.json()) + + +accounts = [] +for index in range(1, 1000): + accounts.append((f'admin{index}', f'admin{index}')) +print(accounts) + +run(register, accounts) From bac03cfdfd396fe2ef6c186a5638e175292c5bb3 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 02:15:04 +0800 Subject: [PATCH 3/8] update account --- docker-compose.yml | 1 + register.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 992ac82..d2e7044 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,4 +15,5 @@ services: environment: REDIS_HOST: redis4accountpool REDIS_PORT: "6379" + API_PORT: "6777" WEBSITE: antispider7 \ No newline at end of file diff --git a/register.py b/register.py index 2c94c4c..782398a 100644 --- a/register.py +++ b/register.py @@ -3,6 +3,8 @@ import requests from loguru import logger +# This is a script for registering account for antispider7, using acinonyx to accelerate. + parser = argparse.ArgumentParser(description='AccountPool') parser.add_argument('website', type=str, help='website') args = parser.parse_args() @@ -12,16 +14,15 @@ @logger.catch() def register(username, password): logger.debug(f'register using {username} and {password}') - response = requests.post('https://antispider7.scrape.center/api/register', json={ + response = requests.post(f'https://{website}.scrape.center/api/register', json={ 'username': username, 'password': password }) print(response.json()) -accounts = [] -for index in range(1, 1000): - accounts.append((f'admin{index}', f'admin{index}')) -print(accounts) - -run(register, accounts) +if __name__ == '__main__': + accounts = [] + for index in range(1, 1000): + accounts.append((f'admin{index}', f'admin{index}')) + run(register, accounts) From 7ea9b8353c50c328f4f553d7d866f8304ac07cf8 Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 02:24:19 +0800 Subject: [PATCH 4/8] update readme --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fec2276..612d80d 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,18 @@ accountpool | 2020-10-09 18:20:16,280 INFO success: tester entered RUNNING st 可以看到 Redis、Generator、Server、Tester 都已经启动成功。 -这时候访问 [http://localhost:6777/antispider6/random](http://localhost:6777/antispider6/random) 即可获取一个 [antispider6](https://antispider6.scrape.center) 的随机可用 Cookies。 +另外还需要导入一些账号信息到 Redis 数据库里面，由于已经用 Docker 启动了 Redis 数据库，运行在 6333 端口上。 + +这时候可以执行脚本： + +``` +export REDIS_PORT=6333 +python3 importer.py antispider7 +``` + +运行完成之后如果没有报错就说明账号导入成功了，可以自行连上 Redis 看下。 + +过一会访问 [http://localhost:6777/antispider7/random](http://localhost:6777/antispider7/random) 即可获取一个 [antispider7](https://antispider7.scrape.center) 的随机可用 Cookies。 ## 常规方式运行 From 111ba2c1160cbf6b03daf0d46dcf59ccd092fa3e Mon Sep 17 00:00:00 2001 From: Germey Date: Sun, 3 Jan 2021 15:41:31 +0800 Subject: [PATCH 5/8] update run --- run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run.py b/run.py index 5d3d1a4..3d88310 100644 --- a/run.py +++ b/run.py @@ -10,7 +10,6 @@ if __name__ == '__main__': # if processor set, just run it if args.processor: - print(args.processor) getattr(Scheduler(), f'run_{args.processor}')(website) else: Scheduler().run(website) From 57a03bbe613b743530e194bf30f968db4c065f9e Mon Sep 17 00:00:00 2001 From: Germey Date: Sat, 13 Mar 2021 22:14:16 +0800 Subject: [PATCH 6/8] update genrator --- accountpool/processors/generator.py | 11 ++++++++++- supervisord.conf | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/accountpool/processors/generator.py b/accountpool/processors/generator.py index 48d6f9b..7852131 100644 --- a/accountpool/processors/generator.py +++ b/accountpool/processors/generator.py @@ -80,7 +80,16 @@ def generate(self, username, password): class Antispider7Generator(BaseGenerator): - + + MAX_COUNT = 100 + + def init(self): + """ + do init + """ + for i in range(1, self.MAX_COUNT + 1): + self.account_operator.set(f'admin{i}', f'admin{i}') + def generate(self, username, password): """ generate main process diff --git a/supervisord.conf b/supervisord.conf index 291eeaf..6d9fd15 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -5,15 +5,25 @@ nodaemon=true process_name=tester command=python3 run.py %(ENV_WEBSITE)s --processor tester directory=/app - +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 [program:generator] process_name=generator command=python3 run.py %(ENV_WEBSITE)s --processor generator directory=/app - +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 [program:server] process_name=server command=python3 run.py %(ENV_WEBSITE)s --processor server directory=/app +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 \ No newline at end of file From 05d58c8f9a44029cc8edd550d7eb1ac1d3ae1e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E5=BA=86=E6=89=8D=E4=B8=A8=E9=9D=99=E8=A7=85?= Date: Sun, 8 May 2022 22:15:55 +0800 Subject: [PATCH 7/8] Update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ffe7b1..fdd46fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ requests==2.13.0 selenium==3.4.0 redis==2.10.5 -Flask==0.12.1 +Flask==1.1.4 environs==7.2.0 loguru==0.3.2 supervisor==4.1.0 +MarkupSafe==2.0.1 From 0db0c787dd910abd77297cbd4daac24edf378a01 Mon Sep 17 00:00:00 2001 From: Germey Date: Thu, 31 Aug 2023 12:10:04 +0800 Subject: [PATCH 8/8] update readme --- README.md | 99 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 612d80d..32b27e6 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ 简易高效的账号池，提供如下功能： -* 定时模拟登录账号，将 Cookies 或 JWT 等信息存储到 Redis 数据库。 -* 定时测试，剔除不可用 Cookies 或 JWT。 -* 提供 API，随机取用测试通过的可用 Cookies 或 JWT。 +- 定时模拟登录账号，将 Cookies 或 JWT 等信息存储到 Redis 数据库。 +- 定时测试，剔除不可用 Cookies 或 JWT。 +- 提供 API，随机取用测试通过的可用 Cookies 或 JWT。 ## 使用要求 @@ -18,15 +18,15 @@ 如果使用 Docker，则需要安装如下环境： -* Docker -* Docker-Compose +- Docker +- Docker-Compose ### 常规方式 常规方式要求有 Python 环境、Redis 环境，具体要求如下： -* Python>=3.6 -* Redis +- Python>=3.6 +- Redis ## Docker 运行 @@ -39,19 +39,38 @@ docker-compose up 运行结果类似如下： ``` -redis | 1:C 09 Oct 2020 18:20:13.963 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo -redis | 1:C 09 Oct 2020 18:20:13.963 # Redis version=6.0.8, bits=64, commit=00000000, modified=0, pid=1, just started -redis | 1:M 09 Oct 2020 18:20:13.964 * RDB memory usage when created 0.83 Mb -redis | 1:M 09 Oct 2020 18:20:13.964 * DB loaded from disk: 0.000 seconds -redis | 1:M 09 Oct 2020 18:20:13.964 * Ready to accept connections -accountpool | 2020-10-09 18:20:14,089 CRIT Supervisor is running as root. Privileges were not dropped because no user is specified in the config file. If you intend to run as root, you can set user=root in the config file to avoid this message. -accountpool | 2020-10-09 18:20:14,091 INFO supervisord started with pid 1 -accountpool | 2020-10-09 18:20:15,094 INFO spawned: 'generator' with pid 9 -accountpool | 2020-10-09 18:20:15,096 INFO spawned: 'server' with pid 10 -accountpool | 2020-10-09 18:20:15,098 INFO spawned: 'tester' with pid 11 -accountpool | 2020-10-09 18:20:16,280 INFO success: generator entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) -accountpool | 2020-10-09 18:20:16,280 INFO success: server entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) -accountpool | 2020-10-09 18:20:16,280 INFO success: tester entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +redis4accountpool is up-to-date +Recreating accountpool ... done +Attaching to redis4accountpool, accountpool +redis4accountpool | 1:C 31 Aug 2023 03:53:10.335 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +redis4accountpool | 1:C 31 Aug 2023 03:53:10.335 * Redis version=7.2.0, bits=64, commit=00000000, modified=0, pid=1, just started +redis4accountpool | 1:C 31 Aug 2023 03:53:10.335 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf +redis4accountpool | 1:M 31 Aug 2023 03:53:10.335 * monotonic clock: POSIX clock_gettime +redis4accountpool | 1:M 31 Aug 2023 03:53:10.336 * Running mode=standalone, port=6379. +redis4accountpool | 1:M 31 Aug 2023 03:53:10.336 * Server initialized +redis4accountpool | 1:M 31 Aug 2023 03:53:10.336 * Ready to accept connections tcp +redis4accountpool | 1:C 31 Aug 2023 04:03:11.226 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +redis4accountpool | 1:C 31 Aug 2023 04:03:11.226 * Redis version=7.2.0, bits=64, commit=00000000, modified=0, pid=1, just started +redis4accountpool | 1:C 31 Aug 2023 04:03:11.226 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf +redis4accountpool | 1:M 31 Aug 2023 04:03:11.226 * monotonic clock: POSIX clock_gettime +redis4accountpool | 1:M 31 Aug 2023 04:03:11.227 * Running mode=standalone, port=6379. +redis4accountpool | 1:M 31 Aug 2023 04:03:11.227 * Server initialized +redis4accountpool | 1:M 31 Aug 2023 04:03:11.227 * Ready to accept connections tcp +accountpool | 2023-08-31 04:06:20,737 CRIT Supervisor is running as root. Privileges were not dropped because no user is specified in the config file. If you intend to run as root, you can set user=root in the config file to avoid this message. +accountpool | 2023-08-31 04:06:20,739 INFO supervisord started with pid 1 +accountpool | 2023-08-31 04:06:21,742 INFO spawned: 'generator' with pid 10 +accountpool | 2023-08-31 04:06:21,744 INFO spawned: 'server' with pid 11 +accountpool | 2023-08-31 04:06:21,746 INFO spawned: 'tester' with pid 12 +accountpool | 2023-08-31 04:06:21.990 | DEBUG | accountpool.scheduler:run_tester:31 - tester loop 0 start... +accountpool | 2023-08-31 04:06:21.990 | DEBUG | accountpool.scheduler:run_generator:46 - getter loop 0 start... +accountpool | * Running on all addresses. +accountpool | WARNING: This is a development server. Do not use it in a production deployment. +accountpool | * Running on http://172.24.0.3:6777/ (Press CTRL+C to quit) +accountpool | 2023-08-31 04:06:22.004 | DEBUG | accountpool.processors.generator:run:39 - start to run generator +accountpool | 2023-08-31 04:06:22.005 | DEBUG | accountpool.processors.generator:run:43 - start to generate credential of admin1 +accountpool | 2023-08-31 04:06:23,007 INFO success: generator entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +accountpool | 2023-08-31 04:06:23,007 INFO success: server entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +accountpool | 2023-08-31 04:06:23,007 INFO success: tester entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) ``` 可以看到 Redis、Generator、Server、Tester 都已经启动成功。 @@ -108,7 +127,7 @@ export REDIS_CONNECTION_STRING='redis://@host:port/db' ### 安装依赖包 -这里强烈推荐使用 [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands) +这里强烈推荐使用 [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-with-commands) 或 [virtualenv](https://virtualenv.pypa.io/en/latest/user_guide.html) 创建虚拟环境，Python 版本不低于 3.6。 然后 pip 安装依赖即可： @@ -145,37 +164,37 @@ python3 run.py --processor server ### 开关 -* ENABLE_TESTER：允许 Tester 启动，默认 true -* ENABLE_GENERATOR：允许 Generator 启动，默认 true -* ENABLE_SERVER：运行 Server 启动，默认 true +- ENABLE_TESTER：允许 Tester 启动，默认 true +- ENABLE_GENERATOR：允许 Generator 启动，默认 true +- ENABLE_SERVER：运行 Server 启动，默认 true ### 环境 -* APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev -* APP_DEBUG：调试模式，可以设置 true 或 false，默认 true +- APP_ENV：运行环境，可以设置 dev、test、prod，即开发、测试、生产环境，默认 dev +- APP_DEBUG：调试模式，可以设置 true 或 false，默认 true ### Redis 连接 -* REDIS_HOST：Redis 的 Host -* REDIS_PORT：Redis 的端口 -* REDIS_PASSWORD：Redis 的密码 -* REDIS_DB：Redis 的数据库索引，如 0、1 -* REDIS_CONNECTION_STRING：Redis 连接字符串 -* REDIS_KEY：Redis 储存代理使用字典的名称 +- REDIS_HOST：Redis 的 Host +- REDIS_PORT：Redis 的端口 +- REDIS_PASSWORD：Redis 的密码 +- REDIS_DB：Redis 的数据库索引，如 0、1 +- REDIS_CONNECTION_STRING：Redis 连接字符串 +- REDIS_KEY：Redis 储存代理使用字典的名称 ### 处理器 -* CYCLE_TESTER：Tester 运行周期，即间隔多久运行一次测试，默认 20 秒 -* CYCLE_GETTER：Getter 运行周期，即间隔多久运行一次代理获取，默认 100 秒 -* API_HOST：代理 Server 运行 Host，默认 0.0.0.0 -* API_PORT：代理 Server 运行端口，默认 6777 -* API_THREADED：代理 Server 是否使用多线程，默认 true +- CYCLE_TESTER：Tester 运行周期，即间隔多久运行一次测试，默认 20 秒 +- CYCLE_GETTER：Getter 运行周期，即间隔多久运行一次代理获取，默认 100 秒 +- API_HOST：代理 Server 运行 Host，默认 0.0.0.0 +- API_PORT：代理 Server 运行端口，默认 6777 +- API_THREADED：代理 Server 是否使用多线程，默认 true ### 日志 -* LOG_DIR：日志相对路径 -* LOG_RUNTIME_FILE：运行日志文件名称 -* LOG_ERROR_FILE：错误日志文件名称 +- LOG_DIR：日志相对路径 +- LOG_RUNTIME_FILE：运行日志文件名称 +- LOG_ERROR_FILE：错误日志文件名称 ## 部署