\s*| (.*?) | \s*(.*?) | ')
- # \s* 匹配空格，起到换行作用
- re_ip_address = ip_address.findall(html)
- for address, port in re_ip_address:
- result = address + ':' + port
- yield result.replace(' ', '')
-
-
- def crawl_premproxy(self):
- for i in ['China-01','China-02','China-03','China-04','Taiwan-01']:
- start_url = 'https://premproxy.com/proxy-by-country/{}.htm'.format(i)
- html = get_page(start_url)
- if html:
- ip_address = re.compile('(.*?) | ')
- re_ip_address = ip_address.findall(html)
- for address_port in re_ip_address:
- yield address_port.replace(' ','')
-
- def crawl_xroxy(self):
- for i in ['CN','TW']:
- start_url = 'http://www.xroxy.com/proxylist.php?country={}'.format(i)
- html = get_page(start_url)
- if html:
- ip_address1 = re.compile("title='View this Proxy details'>\s*(.*).*")
- re_ip_address1 = ip_address1.findall(html)
- ip_address2 = re.compile("title='Select proxies with port number .*'>(.*)")
- re_ip_address2 = ip_address2.findall(html)
- for address,port in zip(re_ip_address1,re_ip_address2):
- address_port = address+':'+port
- yield address_port.replace(' ','')
-
- def crawl_kuaidaili(self):
- for i in range(1, 4):
- start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i)
- html = get_page(start_url)
- if html:
- ip_address = re.compile('(.*?) | ')
- re_ip_address = ip_address.findall(html)
- port = re.compile('(.*?) | ')
- re_port = port.findall(html)
- for address,port in zip(re_ip_address, re_port):
- address_port = address+':'+port
- yield address_port.replace(' ','')
-
- def crawl_xicidaili(self):
- for i in range(1, 3):
- start_url = 'http://www.xicidaili.com/nn/{}'.format(i)
- headers = {
- 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
- 'Cookie':'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRjYzc5MmM1MTBiMDMzYTUzNTZjNzA4NjBhNWRjZjliBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUp6S2tXT3g5a0FCT01ndzlmWWZqRVJNek1WanRuUDBCbTJUN21GMTBKd3M9BjsARg%3D%3D--2a69429cb2115c6a0cc9a86e0ebe2800c0d471b3',
- 'Host':'www.xicidaili.com',
- 'Referer':'http://www.xicidaili.com/nn/3',
- 'Upgrade-Insecure-Requests':'1',
- }
- html = get_page(start_url, options=headers)
- if html:
- find_trs = re.compile('(.*?)
', re.S)
- trs = find_trs.findall(html)
- for tr in trs:
- find_ip = re.compile('(\d+\.\d+\.\d+\.\d+) | ')
- re_ip_address = find_ip.findall(tr)
- find_port = re.compile('(\d+) | ')
- re_port = find_port.findall(tr)
- for address,port in zip(re_ip_address, re_port):
- address_port = address+':'+port
- yield address_port.replace(' ','')
-
- def crawl_ip3366(self):
- for i in range(1, 4):
- start_url = 'http://www.ip3366.net/?stype=1&page={}'.format(i)
- html = get_page(start_url)
- if html:
- find_tr = re.compile('(.*?)
', re.S)
- trs = find_tr.findall(html)
- for s in range(1, len(trs)):
- find_ip = re.compile('(\d+\.\d+\.\d+\.\d+) | ')
- re_ip_address = find_ip.findall(trs[s])
- find_port = re.compile('(\d+) | ')
- re_port = find_port.findall(trs[s])
- for address,port in zip(re_ip_address, re_port):
- address_port = address+':'+port
- yield address_port.replace(' ','')
-
- def crawl_iphai(self):
- start_url = 'http://www.iphai.com/'
- html = get_page(start_url)
- if html:
- find_tr = re.compile('(.*?)
', re.S)
- trs = find_tr.findall(html)
- for s in range(1, len(trs)):
- find_ip = re.compile('\s+(\d+\.\d+\.\d+\.\d+)\s+ | ', re.S)
- re_ip_address = find_ip.findall(trs[s])
- find_port = re.compile('\s+(\d+)\s+ | ', re.S)
- re_port = find_port.findall(trs[s])
- for address,port in zip(re_ip_address, re_port):
- address_port = address+':'+port
- yield address_port.replace(' ','')
-
- def crawl_89ip(self):
- start_url = 'http://www.89ip.cn/apijk/?&tqsl=1000&sxa=&sxb=&tta=&ports=&ktip=&cf=1'
- html = get_page(start_url)
- if html:
- find_ips = re.compile('(\d+\.\d+\.\d+\.\d+:\d+)', re.S)
- ip_ports = find_ips.findall(html)
- for address_port in ip_ports:
- yield address_port
-
- def crawl_data5u(self):
- start_url = 'http://www.data5u.com/free/gngn/index.shtml'
- headers = {
- 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
- 'Accept-Encoding': 'gzip, deflate',
- 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
- 'Cache-Control': 'max-age=0',
- 'Connection': 'keep-alive',
- 'Cookie': 'JSESSIONID=47AA0C887112A2D83EE040405F837A86',
- 'Host': 'www.data5u.com',
- 'Referer': 'http://www.data5u.com/free/index.shtml',
- 'Upgrade-Insecure-Requests': '1',
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
- }
- html = get_page(start_url, options=headers)
- if html:
- ip_address = re.compile('(\d+\.\d+\.\d+\.\d+).*?(\d+)', re.S)
- re_ip_address = ip_address.findall(html)
- for address, port in re_ip_address:
- result = address + ':' + port
- yield result.replace(' ', '')
-
-
-
\ No newline at end of file
diff --git a/proxypool/crawlers/__init__.py b/proxypool/crawlers/__init__.py
new file mode 100644
index 00000000..d2ad0b4c
--- /dev/null
+++ b/proxypool/crawlers/__init__.py
@@ -0,0 +1,16 @@
+import pkgutil
+from .base import BaseCrawler
+import inspect
+
+
+# load classes subclass of BaseCrawler
+classes = []
+for loader, name, is_pkg in pkgutil.walk_packages(__path__):
+ module = loader.find_module(name).load_module(name)
+ for name, value in inspect.getmembers(module):
+ globals()[name] = value
+ if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler \
+ and not getattr(value, 'ignore', False):
+ classes.append(value)
+__all__ = __ALL__ = classes
+
diff --git a/proxypool/crawlers/base.py b/proxypool/crawlers/base.py
new file mode 100644
index 00000000..611a816f
--- /dev/null
+++ b/proxypool/crawlers/base.py
@@ -0,0 +1,49 @@
+from retrying import RetryError, retry
+import requests
+from loguru import logger
+from proxypool.setting import GET_TIMEOUT
+from fake_headers import Headers
+import time
+
+
+class BaseCrawler(object):
+ urls = []
+
+ @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000)
+ def fetch(self, url, **kwargs):
+ try:
+ headers = Headers(headers=True).generate()
+ kwargs.setdefault('timeout', GET_TIMEOUT)
+ kwargs.setdefault('verify', False)
+ kwargs.setdefault('headers', headers)
+ response = requests.get(url, **kwargs)
+ if response.status_code == 200:
+ response.encoding = 'utf-8'
+ return response.text
+ except (requests.ConnectionError, requests.ReadTimeout):
+ return
+
+ def process(self, html, url):
+ """
+ used for parse html
+ """
+ for proxy in self.parse(html):
+ logger.info(f'fetched proxy {proxy.string()} from {url}')
+ yield proxy
+
+ def crawl(self):
+ """
+ crawl main method
+ """
+ try:
+ for url in self.urls:
+ logger.info(f'fetching {url}')
+ html = self.fetch(url)
+ if not html:
+ continue
+ time.sleep(.5)
+ yield from self.process(html, url)
+ except RetryError:
+ logger.error(
+ f'crawler {self} crawled proxy unsuccessfully, '
+ 'please check if target url is valid or network issue')
diff --git a/proxypool/crawlers/private/.gitignore b/proxypool/crawlers/private/.gitignore
new file mode 100644
index 00000000..44202946
--- /dev/null
+++ b/proxypool/crawlers/private/.gitignore
@@ -0,0 +1,2 @@
+*
+!__init__.py
\ No newline at end of file
diff --git a/proxypool/crawlers/private/__init__.py b/proxypool/crawlers/private/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/proxypool/crawlers/public/__init__.py b/proxypool/crawlers/public/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/proxypool/crawlers/public/daili66.py b/proxypool/crawlers/public/daili66.py
new file mode 100644
index 00000000..aec7ea68
--- /dev/null
+++ b/proxypool/crawlers/public/daili66.py
@@ -0,0 +1,32 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+
+
+BASE_URL = 'http://www.66ip.cn/{page}.html'
+MAX_PAGE = 3
+
+
+class Daili66Crawler(BaseCrawler):
+ """
+ daili66 crawler, http://www.66ip.cn/1.html
+ """
+ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ trs = doc('.containerbox table tr:gt(0)').items()
+ for tr in trs:
+ host = tr.find('td:nth-child(1)').text()
+ port = int(tr.find('td:nth-child(2)').text())
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = Daili66Crawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/data5u.py b/proxypool/crawlers/public/data5u.py
new file mode 100644
index 00000000..62158c20
--- /dev/null
+++ b/proxypool/crawlers/public/data5u.py
@@ -0,0 +1,31 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+BASE_URL = 'http://www.data5u.com'
+
+
+class Data5UCrawler(BaseCrawler):
+ """
+ data5u crawler, http://www.data5u.com
+ """
+ urls = [BASE_URL]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ items = doc('.wlist ul.l2').items()
+ for item in items:
+ host = item.find('span:first-child').text()
+ port = int(item.find('span:nth-child(2)').text())
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = Data5UCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py
new file mode 100644
index 00000000..154871fb
--- /dev/null
+++ b/proxypool/crawlers/public/docip.py
@@ -0,0 +1,38 @@
+import time
+from retrying import RetryError
+from loguru import logger
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import json
+
+BASE_URL = 'https://www.docip.net/data/free.json?t={date}'
+
+
+
+class DocipCrawler(BaseCrawler):
+ """
+ Docip crawler, https://www.docip.net/data/free.json
+ """
+ urls = [BASE_URL.format(date=time.strftime("%Y%m%d", time.localtime()))]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ try:
+ result = json.loads(html)
+ proxy_list = result['data']
+ for proxy_item in proxy_list:
+ host = proxy_item['ip']
+ port = host.split(':')[-1]
+ yield Proxy(host=host, port=port)
+ except json.JSONDecodeError:
+ print("json.JSONDecodeError")
+ return
+
+
+if __name__ == '__main__':
+ crawler = DocipCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/fatezero.py b/proxypool/crawlers/public/fatezero.py
new file mode 100644
index 00000000..681cf9e4
--- /dev/null
+++ b/proxypool/crawlers/public/fatezero.py
@@ -0,0 +1,31 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import re
+import json
+BASE_URL = 'http://proxylist.fatezero.org/proxy.list'
+
+
+class FatezeroCrawler(BaseCrawler):
+ """
+ Fatezero crawler,http://proxylist.fatezero.org
+ """
+ urls = [BASE_URL]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+
+ hosts_ports = html.split('\n')
+ for addr in hosts_ports:
+ if(addr):
+ ip_address = json.loads(addr)
+ host = ip_address['host']
+ port = ip_address['port']
+ yield Proxy(host=host, port=port)
+
+if __name__ == '__main__':
+ crawler = FatezeroCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/geonodedaili.py b/proxypool/crawlers/public/geonodedaili.py
new file mode 100644
index 00000000..f71f16ec
--- /dev/null
+++ b/proxypool/crawlers/public/geonodedaili.py
@@ -0,0 +1,71 @@
+import time
+from retrying import RetryError
+from loguru import logger
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import json
+
+BASE_URL = 'https://proxylist.geonode.com/api/proxy-list?limit=500&page={page}&sort_by=lastChecked&sort_type=desc'
+MAX_PAGE = 18
+
+
+class GeonodeCrawler(BaseCrawler):
+ """
+ Geonode crawler, https://proxylist.geonode.com/
+ """
+ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ try:
+ result = json.loads(html)
+ proxy_list = result['data']
+ for proxy_item in proxy_list:
+ host = proxy_item['ip']
+ port = proxy_item['port']
+ yield Proxy(host=host, port=port)
+ except json.JSONDecodeError:
+ print("json.JSONDecodeError")
+ return
+
+ def crawl(self):
+ """
+ override crawl main method
+ add headers
+ """
+ headers = {
+ 'authority': 'proxylist.geonode.com',
+ 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"',
+ 'accept': 'application/json, text/plain, */*',
+ 'sec-ch-ua-mobile': '?0',
+ 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36',
+ 'sec-ch-ua-platform': '"macOS"',
+ 'origin': 'https://geonode.com',
+ 'sec-fetch-site': 'same-site',
+ 'sec-fetch-mode': 'cors',
+ 'sec-fetch-dest': 'empty',
+ 'referer': 'https://geonode.com/',
+ 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7',
+ 'if-none-match': 'W/"c25d-BXjLTmP+/yYXtIz4OEcmdOWSv88"',
+ }
+ try:
+ for url in self.urls:
+ logger.info(f'fetching {url}')
+ html = self.fetch(url, headers=headers)
+ if not html:
+ continue
+ time.sleep(.5)
+ yield from self.process(html, url)
+ except RetryError:
+ logger.error(
+ f'crawler {self} crawled proxy unsuccessfully, '
+ 'please check if target url is valid or network issue')
+
+
+if __name__ == '__main__':
+ crawler = GeonodeCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/goubanjia.py b/proxypool/crawlers/public/goubanjia.py
new file mode 100644
index 00000000..57157858
--- /dev/null
+++ b/proxypool/crawlers/public/goubanjia.py
@@ -0,0 +1,44 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import re
+from pyquery import PyQuery as pq
+import time
+BASE_URL = 'http://www.goubanjia.com/'
+
+
+class GoubanjiaCrawler(BaseCrawler):
+ """
+ ip Goubanjia crawler, http://www.goubanjia.com/
+ """
+ urls = [BASE_URL]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)('.ip').items()
+ # ''.join([*filter(lambda x: x != '',re.compile('\>([\d:\.]*)\<').findall(td.html()))])
+ for td in doc:
+ trs = td.children()
+ ip_str = ''
+ for tr in trs:
+ attrib = tr.attrib
+ if 'style' in attrib and 'none' in tr.attrib['style']:
+ continue
+ ip_str+= '' if not tr.text else tr.text
+ addr_split = ip_str.split(':')
+ if(len(addr_split) == 2):
+ host = addr_split[0]
+ port = addr_split[1]
+ yield Proxy(host=host, port=port)
+ else:
+ port = trs[-1].text
+ host = ip_str.replace(port,'')
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = GoubanjiaCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/ihuan.py b/proxypool/crawlers/public/ihuan.py
new file mode 100644
index 00000000..4ca5e529
--- /dev/null
+++ b/proxypool/crawlers/public/ihuan.py
@@ -0,0 +1,36 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import re
+from pyquery import PyQuery as pq
+import time
+BASE_URL = 'https://ip.ihuan.me/today/{path}.html'
+
+
+class IhuanCrawler(BaseCrawler):
+ """
+ ip ihuan crawler, https://ip.ihuan.me
+ """
+ path = time.strftime("%Y/%m/%d/%H", time.localtime())
+ urls = [BASE_URL.format(path=path)]
+ ignore = False
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ # doc = pq(html)('.text-left')
+ ip_address = re.compile('([\d:\.]*).*?
')
+ hosts_ports = ip_address.findall(html)
+ for addr in hosts_ports:
+ addr_split = addr.split(':')
+ if(len(addr_split) == 2):
+ host = addr_split[0]
+ port = addr_split[1]
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = IhuanCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/ip3366.py b/proxypool/crawlers/public/ip3366.py
new file mode 100644
index 00000000..dfbc06f2
--- /dev/null
+++ b/proxypool/crawlers/public/ip3366.py
@@ -0,0 +1,32 @@
+from proxypool.crawlers.base import BaseCrawler
+from proxypool.schemas.proxy import Proxy
+import re
+
+
+MAX_PAGE = 3
+BASE_URL = 'http://www.ip3366.net/free/?stype={stype}&page={page}'
+
+
+class IP3366Crawler(BaseCrawler):
+ """
+ ip3366 crawler, http://www.ip3366.net/
+ """
+ urls = [BASE_URL.format(stype=stype,page=i) for stype in range(1,3) for i in range(1, 8)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ ip_address = re.compile('\s*| (.*?) | \s*(.*?) | ')
+ # \s * 匹配空格，起到换行作用
+ re_ip_address = ip_address.findall(html)
+ for address, port in re_ip_address:
+ proxy = Proxy(host=address.strip(), port=int(port.strip()))
+ yield proxy
+
+
+if __name__ == '__main__':
+ crawler = IP3366Crawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/ip89.py b/proxypool/crawlers/public/ip89.py
new file mode 100644
index 00000000..f67c3870
--- /dev/null
+++ b/proxypool/crawlers/public/ip89.py
@@ -0,0 +1,33 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import re
+
+MAX_NUM = 9999
+BASE_URL = 'http://api.89ip.cn/tqdl.html?api=1&num={MAX_NUM}&port=&address=&isp='.format(MAX_NUM=MAX_NUM)
+
+
+class Ip89Crawler(BaseCrawler):
+ """
+ 89ip crawler, http://api.89ip.cn
+ """
+ urls = [BASE_URL]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ ip_address = re.compile('([\d:\.]*)
')
+ hosts_ports = ip_address.findall(html)
+ for addr in hosts_ports:
+ addr_split = addr.split(':')
+ if(len(addr_split) == 2):
+ host = addr_split[0]
+ port = addr_split[1]
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = Ip89Crawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/iphai.py b/proxypool/crawlers/public/iphai.py
new file mode 100644
index 00000000..baa79834
--- /dev/null
+++ b/proxypool/crawlers/public/iphai.py
@@ -0,0 +1,35 @@
+from proxypool.crawlers.base import BaseCrawler
+from proxypool.schemas.proxy import Proxy
+import re
+
+
+BASE_URL = 'http://www.iphai.com/'
+
+class IPHaiCrawler(BaseCrawler):
+ """
+ iphai crawler, http://www.iphai.com/
+ """
+ urls = [BASE_URL]
+ ignore = True
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ find_tr = re.compile('
(.*?)
', re.S)
+ trs = find_tr.findall(html)
+ for s in range(1, len(trs)):
+ find_ip = re.compile('\s+(\d+\.\d+\.\d+\.\d+)\s+ | ', re.S)
+ re_ip_address = find_ip.findall(trs[s])
+ find_port = re.compile('\s+(\d+)\s+ | ', re.S)
+ re_port = find_port.findall(trs[s])
+ for address, port in zip(re_ip_address, re_port):
+ proxy = Proxy(host=address.strip(), port=int(port.strip()))
+ yield proxy
+
+if __name__ == '__main__':
+ crawler = IPHaiCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
+
diff --git a/proxypool/crawlers/public/jiangxianli.py b/proxypool/crawlers/public/jiangxianli.py
new file mode 100644
index 00000000..861dd1e5
--- /dev/null
+++ b/proxypool/crawlers/public/jiangxianli.py
@@ -0,0 +1,39 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+import json
+
+
+BASE_URL = 'https://ip.jiangxianli.com/api/proxy_ips?page={page}'
+
+MAX_PAGE = 3
+
+
+class JiangxianliCrawler(BaseCrawler):
+ """
+ jiangxianli crawler,https://ip.jiangxianli.com/
+ """
+
+ urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+
+ result = json.loads(html)
+ if result['code'] != 0:
+ return
+ MAX_PAGE = int(result['data']['last_page'])
+ hosts_ports = result['data']['data']
+ for ip_address in hosts_ports:
+ if(ip_address):
+ host = ip_address['ip']
+ port = ip_address['port']
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = JiangxianliCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/kuaidaili.py b/proxypool/crawlers/public/kuaidaili.py
new file mode 100644
index 00000000..3602833e
--- /dev/null
+++ b/proxypool/crawlers/public/kuaidaili.py
@@ -0,0 +1,33 @@
+from proxypool.crawlers.base import BaseCrawler
+from proxypool.schemas.proxy import Proxy
+import re
+from pyquery import PyQuery as pq
+
+
+BASE_URL = 'https://www.kuaidaili.com/free/{type}/{page}/'
+MAX_PAGE = 3
+
+
+class KuaidailiCrawler(BaseCrawler):
+ """
+ kuaidaili crawler, https://www.kuaidaili.com/
+ """
+ urls = [BASE_URL.format(type=type,page=page) for type in ('intr','inha') for page in range(1, MAX_PAGE + 1)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ for item in doc('table tr').items():
+ td_ip = item.find('td[data-title="IP"]').text()
+ td_port = item.find('td[data-title="PORT"]').text()
+ if td_ip and td_port:
+ yield Proxy(host=td_ip, port=td_port)
+
+
+if __name__ == '__main__':
+ crawler = KuaidailiCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/seofangfa.py b/proxypool/crawlers/public/seofangfa.py
new file mode 100644
index 00000000..1f5a20a2
--- /dev/null
+++ b/proxypool/crawlers/public/seofangfa.py
@@ -0,0 +1,34 @@
+import requests
+from pyquery import PyQuery as pq
+
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+
+requests.packages.urllib3.disable_warnings()
+BASE_URL = "https://proxy.seofangfa.com/"
+MAX_PAGE = 1
+
+
+class SeoFangFaCrawler(BaseCrawler):
+ """
+ seo方法 crawler, https://proxy.seofangfa.com/
+ """
+ urls = ["https://proxy.seofangfa.com/"]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ trs = doc('.table tr:gt(0)').items()
+ for tr in trs:
+ host = tr.find('td:nth-child(1)').text()
+ port = int(tr.find('td:nth-child(2)').text())
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = SeoFangFaCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/taiyangdaili.py b/proxypool/crawlers/public/taiyangdaili.py
new file mode 100644
index 00000000..b42388cc
--- /dev/null
+++ b/proxypool/crawlers/public/taiyangdaili.py
@@ -0,0 +1,31 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from pyquery import PyQuery as pq
+
+BaseUrl = 'http://www.taiyanghttp.com/free/page{num}'
+MAX_PAGE = 3
+
+
+class TaiyangdailiCrawler(BaseCrawler):
+ """
+ taiyangdaili crawler, http://www.taiyanghttp.com/free/
+ """
+ urls = [BaseUrl.format(num=i) for i in range(1, 6)]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ trs = doc('#ip_list .tr.ip_tr').items()
+ for tr in trs:
+ host = tr.find('div:nth-child(1)').text()
+ port = tr.find('div:nth-child(2)').text()
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = TaiyangdailiCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/uqidata.py b/proxypool/crawlers/public/uqidata.py
new file mode 100644
index 00000000..3e54b2dc
--- /dev/null
+++ b/proxypool/crawlers/public/uqidata.py
@@ -0,0 +1,49 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+BASE_URL = 'https://ip.uqidata.com/free/index.html'
+
+
+class UqidataCrawler(BaseCrawler):
+ """
+ Uqidata crawler, https://ip.uqidata.com/free/index.html
+ """
+ urls = [BASE_URL]
+ ignore = True
+
+ def encode(input_str):
+ tmp = []
+ for i in range(len(input_str)):
+ tmp.append("ABCDEFGHIZ".find(input_str[i]))
+ result = "".join(str(i) for i in tmp)
+ result = int(result) >> 0x03
+ return result
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ trs = doc('#main_container .inner table tbody tr:nth-child(n+3)').items()
+ for tr in trs:
+ ip_html = tr('td.ip').find("*").items()
+ host = ''
+ for i in ip_html:
+ if i.attr('style') is not None and 'none' in i.attr('style'):
+ continue
+ if i.text() == '':
+ continue
+ host += i.text()
+
+ port_code = tr('td.port').attr('class').split(' ')[1]
+ port = UqidataCrawler.encode(port_code)
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = UqidataCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/xiaoshudaili.py b/proxypool/crawlers/public/xiaoshudaili.py
new file mode 100644
index 00000000..f6fd0869
--- /dev/null
+++ b/proxypool/crawlers/public/xiaoshudaili.py
@@ -0,0 +1,54 @@
+import re
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+
+BASE_URL = "http://www.xsdaili.cn/"
+PAGE_BASE_URL = "http://www.xsdaili.cn/dayProxy/ip/{page}.html"
+MAX_PAGE = 3
+
+
+class XiaoShuCrawler(BaseCrawler):
+ """
+ 小舒代理 crawler, http://www.xsdaili.cn/
+ """
+
+ def __init__(self):
+ """
+ init urls
+ """
+ try:
+ html = self.fetch(url=BASE_URL)
+ except:
+ self.urls = []
+ return
+ doc = pq(html)
+ title = doc(".title:eq(0) a").items()
+ latest_page = 0
+ for t in title:
+ res = re.search(r"/(\d+)\.html", t.attr("href"))
+ latest_page = int(res.group(1)) if res else 0
+ if latest_page:
+ self.urls = [PAGE_BASE_URL.format(page=page) for page in range(
+ latest_page - MAX_PAGE, latest_page)]
+ else:
+ self.urls = []
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ contents = doc('.cont').text()
+ contents = contents.split("\n")
+ for content in contents:
+ c = content[:content.find("@")]
+ host, port = c.split(":")
+ yield Proxy(host=host, port=int(port))
+
+
+if __name__ == '__main__':
+ crawler = XiaoShuCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/xicidaili.py b/proxypool/crawlers/public/xicidaili.py
new file mode 100644
index 00000000..53a4872e
--- /dev/null
+++ b/proxypool/crawlers/public/xicidaili.py
@@ -0,0 +1,35 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+BASE_URL = 'https://www.xicidaili.com/'
+
+
+class XicidailiCrawler(BaseCrawler):
+ """
+ xididaili crawler, https://www.xicidaili.com/
+ """
+ urls = [BASE_URL]
+ ignore = True
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ items = doc('#ip_list tr:contains(高匿)').items()
+ for item in items:
+ country = item.find('td.country').text()
+ if not country or country.strip() != '高匿':
+ continue
+ host = item.find('td:nth-child(2)').text()
+ port = int(item.find('td:nth-child(3)').text())
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = XicidailiCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/xiladaili.py b/proxypool/crawlers/public/xiladaili.py
new file mode 100644
index 00000000..70a75ff1
--- /dev/null
+++ b/proxypool/crawlers/public/xiladaili.py
@@ -0,0 +1,32 @@
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from lxml import etree
+
+BASE_URL = "http://www.xiladaili.com/"
+MAX_PAGE = 5
+
+
+class XiladailiCrawler(BaseCrawler):
+ """
+ xiladaili crawler, http://www.xiladaili.com/
+ """
+ urls = ["http://www.xiladaili.com/"]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ etree_html = etree.HTML(html)
+ ip_ports = etree_html.xpath("//tbody/tr/td[1]/text()")
+
+ for ip_port in ip_ports:
+ host = ip_port.partition(":")[0]
+ port = ip_port.partition(":")[2]
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = XiladailiCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/yqie.py b/proxypool/crawlers/public/yqie.py
new file mode 100644
index 00000000..fb3feaf8
--- /dev/null
+++ b/proxypool/crawlers/public/yqie.py
@@ -0,0 +1,32 @@
+from pyquery import PyQuery as pq
+
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+
+BASE_URL = "http://ip.yqie.com/ipproxy.htm"
+MAX_PAGE = 1
+
+
+class YqIeCrawler(BaseCrawler):
+ """
+ ip yqie crawler, http://ip.yqie.com/ipproxy.htm
+ """
+ urls = [BASE_URL]
+
+ def parse(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ trs = doc('#GridViewOrder tr:gt(0)').items()
+ for tr in trs:
+ host = tr.find('td:nth-child(1)').text()
+ port = int(tr.find('td:nth-child(2)').text())
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = YqIeCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/crawlers/public/zhandaye.py b/proxypool/crawlers/public/zhandaye.py
new file mode 100755
index 00000000..1522cdf0
--- /dev/null
+++ b/proxypool/crawlers/public/zhandaye.py
@@ -0,0 +1,59 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+import re
+
+
+BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html'
+MAX_PAGE = 5 * 2
+
+
+class ZhandayeCrawler(BaseCrawler):
+ """
+ zhandaye crawler, https://www.zdaye.com/dayProxy/
+ """
+ urls_catalog = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)]
+ headers = {
+ 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+ }
+ urls = []
+ ignore = True
+
+ def crawl(self):
+ self.crawl_catalog()
+ yield from super().crawl()
+
+ def crawl_catalog(self):
+ for url in self.urls_catalog:
+ logger.info(f'fetching {url}')
+ html = self.fetch(url, headers=self.headers)
+ self.parse_catalog(html)
+
+ def parse_catalog(self, html):
+ """
+ parse html file to get proxies
+ :return:
+ """
+ doc = pq(html)
+ for item in doc('#J_posts_list .thread_item div div p a').items():
+ url = 'https://www.zdaye.com' + item.attr('href')
+ logger.info(f'get detail url: {url}')
+ self.urls.append(url)
+
+ def parse(self, html):
+ doc = pq(html)
+ trs = doc('.cont br').items()
+ for tr in trs:
+ line = tr[0].tail
+ match = re.search(r'(\d+\.\d+\.\d+\.\d+):(\d+)', line)
+ if match:
+ host = match.group(1)
+ port = match.group(2)
+ yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+ crawler = ZhandayeCrawler()
+ for proxy in crawler.crawl():
+ print(proxy)
diff --git a/proxypool/db.py b/proxypool/db.py
deleted file mode 100644
index bf108f77..00000000
--- a/proxypool/db.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import redis
-from proxypool.error import PoolEmptyError
-from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY
-from proxypool.setting import MAX_SCORE, MIN_SCORE, INITIAL_SCORE
-from random import choice
-import re
-
-
-class RedisClient(object):
- def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD):
- """
- 初始化
- :param host: Redis 地址
- :param port: Redis 端口
- :param password: Redis密码
- """
- self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=True)
-
- def add(self, proxy, score=INITIAL_SCORE):
- """
- 添加代理，设置分数为最高
- :param proxy: 代理
- :param score: 分数
- :return: 添加结果
- """
- if not re.match('\d+\.\d+\.\d+\.\d+\:\d+', proxy):
- print('代理不符合规范', proxy, '丢弃')
- return
- if not self.db.zscore(REDIS_KEY, proxy):
- return self.db.zadd(REDIS_KEY, score, proxy)
-
- def random(self):
- """
- 随机获取有效代理，首先尝试获取最高分数代理，如果不存在，按照排名获取，否则异常
- :return: 随机代理
- """
- result = self.db.zrangebyscore(REDIS_KEY, MAX_SCORE, MAX_SCORE)
- if len(result):
- return choice(result)
- else:
- result = self.db.zrevrange(REDIS_KEY, 0, 100)
- if len(result):
- return choice(result)
- else:
- raise PoolEmptyError
-
- def decrease(self, proxy):
- """
- 代理值减一分，小于最小值则删除
- :param proxy: 代理
- :return: 修改后的代理分数
- """
- score = self.db.zscore(REDIS_KEY, proxy)
- if score and score > MIN_SCORE:
- print('代理', proxy, '当前分数', score, '减1')
- return self.db.zincrby(REDIS_KEY, proxy, -1)
- else:
- print('代理', proxy, '当前分数', score, '移除')
- return self.db.zrem(REDIS_KEY, proxy)
-
- def exists(self, proxy):
- """
- 判断是否存在
- :param proxy: 代理
- :return: 是否存在
- """
- return not self.db.zscore(REDIS_KEY, proxy) == None
-
- def max(self, proxy):
- """
- 将代理设置为MAX_SCORE
- :param proxy: 代理
- :return: 设置结果
- """
- print('代理', proxy, '可用，设置为', MAX_SCORE)
- return self.db.zadd(REDIS_KEY, MAX_SCORE, proxy)
-
- def count(self):
- """
- 获取数量
- :return: 数量
- """
- return self.db.zcard(REDIS_KEY)
-
- def all(self):
- """
- 获取全部代理
- :return: 全部代理列表
- """
- return self.db.zrangebyscore(REDIS_KEY, MIN_SCORE, MAX_SCORE)
-
- def batch(self, start, stop):
- """
- 批量获取
- :param start: 开始索引
- :param stop: 结束索引
- :return: 代理列表
- """
- return self.db.zrevrange(REDIS_KEY, start, stop - 1)
-
-
-if __name__ == '__main__':
- conn = RedisClient()
- result = conn.batch(680, 688)
- print(result)
diff --git a/proxypool/error.py b/proxypool/error.py
deleted file mode 100644
index ca19569e..00000000
--- a/proxypool/error.py
+++ /dev/null
@@ -1,7 +0,0 @@
-class PoolEmptyError(Exception):
-
- def __init__(self):
- Exception.__init__(self)
-
- def __str__(self):
- return repr('代理池已经枯竭')
diff --git a/proxypool/exceptions/__init__.py b/proxypool/exceptions/__init__.py
new file mode 100644
index 00000000..b54b1e85
--- /dev/null
+++ b/proxypool/exceptions/__init__.py
@@ -0,0 +1 @@
+from .empty import PoolEmptyException
\ No newline at end of file
diff --git a/proxypool/exceptions/empty.py b/proxypool/exceptions/empty.py
new file mode 100644
index 00000000..255c7fbf
--- /dev/null
+++ b/proxypool/exceptions/empty.py
@@ -0,0 +1,7 @@
+class PoolEmptyException(Exception):
+ def __str__(self):
+ """
+ proxypool is used out
+ :return:
+ """
+ return repr('no proxy in proxypool')
diff --git a/proxypool/getter.py b/proxypool/getter.py
deleted file mode 100644
index 7af9b5c0..00000000
--- a/proxypool/getter.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from proxypool.tester import Tester
-from proxypool.db import RedisClient
-from proxypool.crawler import Crawler
-from proxypool.setting import *
-import sys
-
-class Getter():
- def __init__(self):
- self.redis = RedisClient()
- self.crawler = Crawler()
-
- def is_over_threshold(self):
- """
- 判断是否达到了代理池限制
- """
- if self.redis.count() >= POOL_UPPER_THRESHOLD:
- return True
- else:
- return False
-
- def run(self):
- print('获取器开始执行')
- if not self.is_over_threshold():
- for callback_label in range(self.crawler.__CrawlFuncCount__):
- callback = self.crawler.__CrawlFunc__[callback_label]
- # 获取代理
- proxies = self.crawler.get_proxies(callback)
- sys.stdout.flush()
- for proxy in proxies:
- self.redis.add(proxy)
diff --git a/proxypool/importer.py b/proxypool/importer.py
deleted file mode 100644
index d9651bb2..00000000
--- a/proxypool/importer.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from proxypool.db import RedisClient
-
-conn = RedisClient()
-
-
-def set(proxy):
- result = conn.add(proxy)
- print(proxy)
- print('录入成功' if result else '录入失败')
-
-
-def scan():
- print('请输入代理, 输入exit退出读入')
- while True:
- proxy = input()
- if proxy == 'exit':
- break
- set(proxy)
-
-
-if __name__ == '__main__':
- scan()
diff --git a/proxypool/processors/__init__.py b/proxypool/processors/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/proxypool/processors/getter.py b/proxypool/processors/getter.py
new file mode 100644
index 00000000..c5c16296
--- /dev/null
+++ b/proxypool/processors/getter.py
@@ -0,0 +1,46 @@
+from loguru import logger
+from proxypool.storages.redis import RedisClient
+from proxypool.setting import PROXY_NUMBER_MAX
+from proxypool.crawlers import __all__ as crawlers_cls
+from proxypool.testers import __all__ as testers_cls
+
+class Getter(object):
+ """
+ getter of proxypool
+ """
+
+ def __init__(self):
+ """
+ init db and crawlers
+ """
+ self.redis = RedisClient()
+ self.crawlers_cls = crawlers_cls
+ self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
+ self.testers_cls = testers_cls
+ self.testers = [tester_cls() for tester_cls in self.testers_cls]
+
+ def is_full(self):
+ """
+ if proxypool if full
+ return: bool
+ """
+ return self.redis.count() >= PROXY_NUMBER_MAX
+
+ @logger.catch
+ def run(self):
+ """
+ run crawlers to get proxy
+ :return:
+ """
+ if self.is_full():
+ return
+ for crawler in self.crawlers:
+ logger.info(f'crawler {crawler} to get proxy')
+ for proxy in crawler.crawl():
+ self.redis.add(proxy)
+ [self.redis.add(proxy, redis_key=tester.key) for tester in self.testers]
+
+
+if __name__ == '__main__':
+ getter = Getter()
+ getter.run()
diff --git a/proxypool/processors/server.py b/proxypool/processors/server.py
new file mode 100644
index 00000000..f95f21dc
--- /dev/null
+++ b/proxypool/processors/server.py
@@ -0,0 +1,167 @@
+import hmac
+import re
+from flask import Flask, g, request, abort
+from proxypool.exceptions import PoolEmptyException
+from proxypool.storages.redis import RedisClient
+from proxypool.setting import API_HOST, API_PORT, API_THREADED, API_KEY, IS_DEV, PROXY_RAND_KEY_DEGRADED
+import functools
+from random import choice, sample
+from proxypool.utils.geo import get_country_iso
+
+__all__ = ['app']
+
+app = Flask(__name__)
+if IS_DEV:
+ app.debug = True
+
+# allowed characters for the `key` query parameter that selects a redis sub-pool;
+# restricts to a safe charset to avoid probing arbitrary redis keys via the API
+VALID_KEY_PATTERN = re.compile(r'^[a-zA-Z0-9_:\-]{1,64}$')
+
+
+def auth_required(func):
+ @functools.wraps(func)
+ def decorator(*args, **kwargs):
+ # conditional decorator, when setting API_KEY is set, otherwise just ignore this decorator
+ if API_KEY == "":
+ return func(*args, **kwargs)
+ if request.headers.get('API-KEY', None) is not None:
+ api_key = request.headers.get('API-KEY')
+ else:
+ return {"message": "Please provide an API key in header"}, 400
+ # Check if API key is correct and valid
+ if request.method == "GET" and hmac.compare_digest(api_key, API_KEY):
+ return func(*args, **kwargs)
+ else:
+ return {"message": "The provided API key is not valid"}, 403
+
+ return decorator
+
+
+def get_conn():
+ """
+ get redis client object
+ :return:
+ """
+ if not hasattr(g, 'redis'):
+ g.redis = RedisClient()
+ return g.redis
+
+
+def get_request_key():
+ """
+ read the `key` query parameter and validate its format;
+ reject unexpected characters to avoid redis key probing/injection
+ :return: validated key or None
+ """
+ key = request.args.get('key')
+ if key and not VALID_KEY_PATTERN.match(key):
+ abort(400, description='invalid key parameter')
+ return key
+
+
+def filter_proxies_by_area(proxies, area):
+ """
+ filter proxies by country iso code (e.g. 'CN', 'US'), case-insensitive;
+ proxies whose country cannot be resolved are excluded
+ :param proxies: list of Proxy
+ :param area: country iso code, or falsy to skip filtering
+ :return: filtered list of Proxy
+ """
+ if not area:
+ return proxies
+ area = area.upper()
+ return [proxy for proxy in proxies if get_country_iso(proxy.host) == area]
+
+
+@app.route('/')
+@auth_required
+def index():
+ """
+ get home page, you can define your own templates
+ :return:
+ """
+ return 'Welcome to Proxy Pool System
'
+
+
+@app.route('/random')
+@auth_required
+def get_proxy():
+ """
+ get a random proxy, can query the specific sub-pool according the (redis) key
+ if PROXY_RAND_KEY_DEGRADED is set to True, will get a universal random proxy if no proxy found in the sub-pool
+ can pass a `count` parameter to get multiple random proxies at once
+ can pass an `area` parameter to only get proxies from a country (iso code, e.g. CN)
+ :return: get a random proxy
+ """
+ key = get_request_key()
+ count = request.args.get('count', type=int)
+ area = request.args.get('area')
+ conn = get_conn()
+ # return conn.random(key).string() if key else conn.random().string()
+ if area:
+ # area filtering needs the candidate set first, then filter by country
+ candidates = conn.all(key) if key else conn.all()
+ candidates = filter_proxies_by_area(candidates, area)
+ if not candidates and key and PROXY_RAND_KEY_DEGRADED:
+ candidates = filter_proxies_by_area(conn.all(), area)
+ if not candidates:
+ raise PoolEmptyException
+ if count and count > 1:
+ count = min(count, len(candidates))
+ return '\n'.join(proxy.string() for proxy in sample(candidates, count))
+ return choice(candidates).string()
+ if count and count > 1:
+ # return multiple random proxies, one per line
+ try:
+ proxies = conn.randoms(count, key) if key else conn.randoms(count)
+ except PoolEmptyException:
+ if key and PROXY_RAND_KEY_DEGRADED:
+ proxies = conn.randoms(count)
+ else:
+ raise
+ return '\n'.join(proxy.string() for proxy in proxies)
+ if key:
+ try:
+ return conn.random(key).string()
+ except PoolEmptyException:
+ if not PROXY_RAND_KEY_DEGRADED:
+ raise
+ return conn.random().string()
+
+
+@app.route('/all')
+@auth_required
+def get_proxy_all():
+ """
+ get all proxies, optionally filtered by `area` (country iso code, e.g. CN)
+ :return: all proxies
+ """
+ key = get_request_key()
+ area = request.args.get('area')
+
+ conn = get_conn()
+ proxies = conn.all(key) if key else conn.all()
+ proxies = filter_proxies_by_area(proxies, area)
+ proxies_string = ''
+ if proxies:
+ for proxy in proxies:
+ proxies_string += str(proxy) + '\n'
+
+ return proxies_string
+
+
+@app.route('/count')
+@auth_required
+def get_count():
+ """
+ get the count of proxies
+ :return: count, int
+ """
+ conn = get_conn()
+ key = get_request_key()
+ return str(conn.count(key)) if key else str(conn.count())
+
+
+if __name__ == '__main__':
+ app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py
new file mode 100644
index 00000000..5d70d238
--- /dev/null
+++ b/proxypool/processors/tester.py
@@ -0,0 +1,169 @@
+import asyncio
+import aiohttp
+from loguru import logger
+from proxypool.schemas import Proxy
+from proxypool.storages.redis import RedisClient
+from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS, TEST_ANONYMOUS, \
+ TEST_DONT_SET_MAX_SCORE, TEST_ANONYMOUS_URL
+from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError, ClientHttpProxyError, \
+ ClientResponseError, ContentTypeError
+from asyncio import TimeoutError
+from proxypool.testers import __all__ as testers_cls
+
+EXCEPTIONS = (
+ ClientProxyConnectionError,
+ ConnectionRefusedError,
+ TimeoutError,
+ ServerDisconnectedError,
+ ClientOSError,
+ ClientHttpProxyError,
+ ClientResponseError,
+ ContentTypeError,
+ AssertionError
+)
+
+
+class Tester(object):
+ """
+ tester for testing proxies in queue
+ """
+
+ def __init__(self):
+ """
+ init redis
+ """
+ self.redis = RedisClient()
+ self.testers_cls = testers_cls
+ self.testers = [tester_cls() for tester_cls in self.testers_cls]
+
+ async def test(self, proxy: Proxy, session: aiohttp.ClientSession):
+ """
+ test single proxy
+ :param proxy: Proxy object
+ :param session: shared aiohttp session
+ :return:
+ """
+ try:
+ logger.debug(f'testing {proxy.string()}')
+ # if TEST_ANONYMOUS is True, make sure that
+ # the proxy has the effect of hiding the real IP
+ # logger.debug(f'TEST_ANONYMOUS {TEST_ANONYMOUS}')
+ if TEST_ANONYMOUS:
+ url = TEST_ANONYMOUS_URL
+ async with session.get(url, timeout=TEST_TIMEOUT) as response:
+ resp_json = await response.json()
+ origin_ip = resp_json['origin']
+ # logger.debug(f'origin ip is {origin_ip}')
+ async with session.get(url, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT) as response:
+ resp_json = await response.json()
+ anonymous_ip = resp_json['origin']
+ logger.debug(f'anonymous ip is {anonymous_ip}')
+ assert origin_ip != anonymous_ip
+ assert proxy.host == anonymous_ip
+ async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT,
+ allow_redirects=False) as response:
+ if response.status in TEST_VALID_STATUS:
+ if TEST_DONT_SET_MAX_SCORE:
+ logger.debug(
+ f'proxy {proxy.string()} is valid, remain current score')
+ else:
+ self.redis.max(proxy)
+ logger.debug(
+ f'proxy {proxy.string()} is valid, set max score')
+ else:
+ self.redis.decrease(proxy)
+ logger.debug(
+ f'proxy {proxy.string()} is invalid, decrease score')
+ # if independent tester class found, create new set of storage and do the extra test
+ for tester in self.testers:
+ key = tester.key
+ if self.redis.exists(proxy, key):
+ test_url = tester.test_url
+ headers = tester.headers()
+ cookies = tester.cookies()
+ async with session.get(test_url, proxy=f'http://{proxy.string()}',
+ timeout=TEST_TIMEOUT,
+ headers=headers,
+ cookies=cookies,
+ allow_redirects=False) as response:
+ resp_text = await response.text()
+ is_valid = await tester.parse(resp_text, test_url, proxy.string())
+ if is_valid:
+ if tester.test_dont_set_max_score:
+ logger.info(
+ f'key[{key}] proxy {proxy.string()} is valid, remain current score')
+ else:
+ self.redis.max(
+ proxy, key, tester.proxy_score_max)
+ logger.info(
+ f'key[{key}] proxy {proxy.string()} is valid, set max score')
+ else:
+ self.redis.decrease(
+ proxy, tester.key, tester.proxy_score_min)
+ logger.info(
+ f'key[{key}] proxy {proxy.string()} is invalid, decrease score')
+
+ except EXCEPTIONS:
+ self.redis.decrease(proxy)
+ [self.redis.decrease(proxy, tester.key, tester.proxy_score_min)
+ for tester in self.testers]
+ logger.debug(
+ f'proxy {proxy.string()} is invalid, decrease score')
+
+ async def run_tests(self):
+ """
+ test all proxies in batches, reusing a single aiohttp session
+ :return:
+ """
+ count = self.redis.count()
+ logger.debug(f'{count} proxies to test')
+ cursor = 0
+ connector = aiohttp.TCPConnector(ssl=False, limit=TEST_BATCH)
+ async with aiohttp.ClientSession(connector=connector) as session:
+ while True:
+ logger.debug(
+ f'testing proxies use cursor {cursor}, count {TEST_BATCH}')
+ cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH)
+ if proxies:
+ tasks = [self.test(proxy, session) for proxy in proxies]
+ await asyncio.gather(*tasks, return_exceptions=True)
+ if not cursor:
+ break
+
+ @logger.catch
+ def run(self):
+ """
+ test main method
+ :return:
+ """
+ # event loop of aiohttp
+ logger.info('stating tester...')
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ try:
+ loop.run_until_complete(self.run_tests())
+ finally:
+ loop.close()
+
+
+def run_tester():
+ host = '96.113.165.182'
+ port = '3128'
+ tester = Tester()
+
+ async def _test():
+ async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
+ await tester.test(Proxy(host=host, port=port), session)
+
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ try:
+ loop.run_until_complete(_test())
+ finally:
+ loop.close()
+
+
+if __name__ == '__main__':
+ tester = Tester()
+ tester.run()
+ # run_tester()
diff --git a/proxypool/scheduler.py b/proxypool/scheduler.py
index 4ca55eed..d0582268 100644
--- a/proxypool/scheduler.py
+++ b/proxypool/scheduler.py
@@ -1,50 +1,146 @@
import time
-from multiprocessing import Process
-from proxypool.api import app
-from proxypool.getter import Getter
-from proxypool.tester import Tester
-from proxypool.db import RedisClient
-from proxypool.setting import *
+import multiprocessing
+from proxypool.processors.server import app
+from proxypool.processors.getter import Getter
+from proxypool.processors.tester import Tester
+from proxypool.setting import APP_PROD_METHOD_GEVENT, APP_PROD_METHOD_MEINHELD, APP_PROD_METHOD_TORNADO, CYCLE_GETTER, CYCLE_TESTER, API_HOST, \
+ API_THREADED, API_PORT, ENABLE_SERVER, IS_PROD, APP_PROD_METHOD, \
+ ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS
+from loguru import logger
+
+
+if IS_WINDOWS:
+ multiprocessing.freeze_support()
+
+tester_process, getter_process, server_process = None, None, None
class Scheduler():
- def schedule_tester(self, cycle=TESTER_CYCLE):
+ """
+ scheduler
+ """
+
+ def run_tester(self, cycle=CYCLE_TESTER):
"""
- 定时测试代理
+ run tester
"""
+ if not ENABLE_TESTER:
+ logger.info('tester not enabled, exit')
+ return
tester = Tester()
+ loop = 0
while True:
- print('测试器开始运行')
+ logger.debug(f'tester loop {loop} start...')
tester.run()
+ loop += 1
time.sleep(cycle)
-
- def schedule_getter(self, cycle=GETTER_CYCLE):
+
+ def run_getter(self, cycle=CYCLE_GETTER):
"""
- 定时获取代理
+ run getter
"""
+ if not ENABLE_GETTER:
+ logger.info('getter not enabled, exit')
+ return
getter = Getter()
+ loop = 0
while True:
- print('开始抓取代理')
+ logger.debug(f'getter loop {loop} start...')
getter.run()
+ loop += 1
time.sleep(cycle)
-
- def schedule_api(self):
+
+ def run_server(self):
"""
- 开启API
+ run server for api
"""
- app.run(API_HOST, API_PORT)
-
+ if not ENABLE_SERVER:
+ logger.info('server not enabled, exit')
+ return
+ if IS_PROD:
+ if APP_PROD_METHOD == APP_PROD_METHOD_GEVENT:
+ try:
+ from gevent.pywsgi import WSGIServer
+ except ImportError as e:
+ logger.exception(e)
+ else:
+ http_server = WSGIServer((API_HOST, API_PORT), app)
+ http_server.serve_forever()
+
+ elif APP_PROD_METHOD == APP_PROD_METHOD_TORNADO:
+ try:
+ from tornado.wsgi import WSGIContainer
+ from tornado.httpserver import HTTPServer
+ from tornado.ioloop import IOLoop
+ except ImportError as e:
+ logger.exception(e)
+ else:
+ http_server = HTTPServer(WSGIContainer(app))
+ http_server.listen(API_PORT)
+ IOLoop.instance().start()
+
+ elif APP_PROD_METHOD == APP_PROD_METHOD_MEINHELD:
+ try:
+ import meinheld
+ except ImportError as e:
+ logger.exception(e)
+ else:
+ meinheld.listen((API_HOST, API_PORT))
+ meinheld.run(app)
+
+ else:
+ logger.error("unsupported APP_PROD_METHOD")
+ return
+ else:
+ app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED, use_reloader=False)
+
def run(self):
- print('代理池开始运行')
-
- if TESTER_ENABLED:
- tester_process = Process(target=self.schedule_tester)
- tester_process.start()
-
- if GETTER_ENABLED:
- getter_process = Process(target=self.schedule_getter)
- getter_process.start()
-
- if API_ENABLED:
- api_process = Process(target=self.schedule_api)
- api_process.start()
+ global tester_process, getter_process, server_process
+ try:
+ logger.info('starting proxypool...')
+ if ENABLE_TESTER:
+ tester_process = multiprocessing.Process(
+ target=self.run_tester)
+ logger.info(f'starting tester, pid {tester_process.pid}...')
+ tester_process.start()
+
+ if ENABLE_GETTER:
+ getter_process = multiprocessing.Process(
+ target=self.run_getter)
+ logger.info(f'starting getter, pid {getter_process.pid}...')
+ getter_process.start()
+
+ if ENABLE_SERVER:
+ server_process = multiprocessing.Process(
+ target=self.run_server)
+ logger.info(f'starting server, pid {server_process.pid}...')
+ server_process.start()
+
+ tester_process and tester_process.join()
+ getter_process and getter_process.join()
+ server_process and server_process.join()
+ except KeyboardInterrupt:
+ logger.info('received keyboard interrupt signal')
+ tester_process and tester_process.terminate()
+ getter_process and getter_process.terminate()
+ server_process and server_process.terminate()
+ finally:
+ # must call join method before calling is_alive
+ tester_process and tester_process.join()
+ getter_process and getter_process.join()
+ server_process and server_process.join()
+ if tester_process:
+ logger.info(
+ f'tester is {"alive" if tester_process.is_alive() else "dead"}')
+ if getter_process:
+ logger.info(
+ f'getter is {"alive" if getter_process.is_alive() else "dead"}')
+ if server_process:
+ logger.info(
+ f'server is {"alive" if server_process.is_alive() else "dead"}')
+ logger.info('proxy terminated')
+
+
+if __name__ == '__main__':
+ scheduler = Scheduler()
+ scheduler.run()
diff --git a/proxypool/schemas/__init__.py b/proxypool/schemas/__init__.py
new file mode 100644
index 00000000..699f6dc2
--- /dev/null
+++ b/proxypool/schemas/__init__.py
@@ -0,0 +1 @@
+from .proxy import Proxy
\ No newline at end of file
diff --git a/proxypool/schemas/proxy.py b/proxypool/schemas/proxy.py
new file mode 100644
index 00000000..8be3fb34
--- /dev/null
+++ b/proxypool/schemas/proxy.py
@@ -0,0 +1,30 @@
+from attr import attrs, attr
+
+
+@attrs
+class Proxy(object):
+ """
+ proxy schema
+ """
+ host = attr(type=str, default=None)
+ port = attr(type=int, default=None)
+
+ def __str__(self):
+ """
+ to string, for print
+ :return:
+ """
+ return f'{self.host}:{self.port}'
+
+ def string(self):
+ """
+ to string
+ :return: :
+ """
+ return self.__str__()
+
+
+if __name__ == '__main__':
+ proxy = Proxy(host='8.8.8.8', port=8888)
+ print('proxy', proxy)
+ print('proxy', proxy.string())
diff --git a/proxypool/setting.py b/proxypool/setting.py
index 3e4ed901..4d1cec56 100644
--- a/proxypool/setting.py
+++ b/proxypool/setting.py
@@ -1,40 +1,127 @@
-# Redis数据库地址
-REDIS_HOST = '127.0.0.1'
+import platform
+from os.path import dirname, abspath, join
+from environs import Env
+from loguru import logger
+import shutil
-# Redis端口
-REDIS_PORT = 6379
-# Redis密码，如无填None
-REDIS_PASSWORD = None
+env = Env()
+env.read_env()
-REDIS_KEY = 'proxies'
+# definition of flags
+IS_WINDOWS = platform.system().lower() == 'windows'
-# 代理分数
-MAX_SCORE = 100
-MIN_SCORE = 0
-INITIAL_SCORE = 10
+# definition of dirs
+ROOT_DIR = dirname(dirname(abspath(__file__)))
+LOG_DIR = join(ROOT_DIR, env.str('LOG_DIR', 'logs'))
-VALID_STATUS_CODES = [200, 302]
+# definition of environments
+DEV_MODE, TEST_MODE, PROD_MODE = 'dev', 'test', 'prod'
+APP_ENV = env.str('APP_ENV', DEV_MODE).lower()
+APP_DEBUG = env.bool('APP_DEBUG', True if APP_ENV == DEV_MODE else False)
+APP_DEV = IS_DEV = APP_ENV == DEV_MODE
+APP_PROD = IS_PROD = APP_ENV == PROD_MODE
+APP_TEST = IS_TEST = APP_ENV == TEST_MODE
-# 代理池数量界限
-POOL_UPPER_THRESHOLD = 50000
-# 检查周期
-TESTER_CYCLE = 20
-# 获取周期
-GETTER_CYCLE = 300
+# Which WSGI container is used to run applications
+# - gevent: pip install gevent
+# - tornado: pip install tornado
+# - meinheld: pip install meinheld
+APP_PROD_METHOD_GEVENT = 'gevent'
+APP_PROD_METHOD_TORNADO = 'tornado'
+APP_PROD_METHOD_MEINHELD = 'meinheld'
+APP_PROD_METHOD = env.str('APP_PROD_METHOD', APP_PROD_METHOD_GEVENT).lower()
-# 测试API，建议抓哪个网站测哪个
-TEST_URL = 'http://www.baidu.com'
+# redis host
+REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST',
+ env.str('REDIS_HOST', '127.0.0.1'))
+# redis port
+REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379))
+# redis password, if no password, set it to None
+REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD',
+ env.str('REDIS_PASSWORD', None))
+# redis db, if no choice, set it to 0
+REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0))
+# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0,
+# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
+REDIS_CONNECTION_STRING = env.str(
+ 'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None))
-# API配置
-API_HOST = '0.0.0.0'
-API_PORT = 5555
+# redis hash table key name
+REDIS_KEY = env.str('PROXYPOOL_REDIS_KEY', env.str(
+ 'REDIS_KEY', 'proxies:universal'))
-# 开关
-TESTER_ENABLED = True
-GETTER_ENABLED = True
-API_ENABLED = True
+# definition of proxy scores
+PROXY_SCORE_MAX = env.int('PROXY_SCORE_MAX', 100)
+PROXY_SCORE_MIN = env.int('PROXY_SCORE_MIN', 0)
+PROXY_SCORE_INIT = env.int('PROXY_SCORE_INIT', 10)
+# whether to get a universal random proxy if no proxy exists in the sub-pool identified by a specific key
+PROXY_RAND_KEY_DEGRADED = env.bool('TEST_ANONYMOUS', True)
-# 最大批测试量
-BATCH_TEST_SIZE = 10
+# definition of proxy number
+PROXY_NUMBER_MAX = 50000
+PROXY_NUMBER_MIN = 0
+
+# definition of tester cycle, it will test every CYCLE_TESTER second
+CYCLE_TESTER = env.int('CYCLE_TESTER', 20)
+# definition of getter cycle, it will get proxy every CYCLE_GETTER second
+CYCLE_GETTER = env.int('CYCLE_GETTER', 100)
+GET_TIMEOUT = env.int('GET_TIMEOUT', 10)
+
+# definition of tester
+TEST_URL = env.str('TEST_URL', 'http://www.baidu.com')
+TEST_TIMEOUT = env.int('TEST_TIMEOUT', 10)
+TEST_BATCH = env.int('TEST_BATCH', 20)
+# only save anonymous proxy
+TEST_ANONYMOUS = env.bool('TEST_ANONYMOUS', True)
+# the url used to check the proxy anonymity and its exit ip;
+# must return json like httpbin.org/ip ({"origin": "1.2.3.4"});
+# point this to a self-hosted httpbin to avoid public rate limits
+TEST_ANONYMOUS_URL = env.str('TEST_ANONYMOUS_URL', 'https://httpbin.org/ip')
+# TEST_HEADERS = env.json('TEST_HEADERS', {
+# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
+# })
+TEST_VALID_STATUS = env.list('TEST_VALID_STATUS', [200, 206, 302])
+# whether to set max score when one proxy is tested valid
+TEST_DONT_SET_MAX_SCORE = env.bool('TEST_DONT_SET_MAX_SCORE', False)
+
+# definition of api
+API_HOST = env.str('API_HOST', '0.0.0.0')
+API_PORT = env.int('API_PORT', 5555)
+API_THREADED = env.bool('API_THREADED', True)
+# add an api key to get proxy
+# need a header of `API-KEY` in get request to pass the authenticate
+# API_KEY='', do not need `API-KEY` header
+API_KEY = env.str('API_KEY', '')
+
+# flags of enable
+ENABLE_TESTER = env.bool('ENABLE_TESTER', True)
+ENABLE_GETTER = env.bool('ENABLE_GETTER', True)
+ENABLE_SERVER = env.bool('ENABLE_SERVER', True)
+
+
+ENABLE_LOG_FILE = env.bool('ENABLE_LOG_FILE', True)
+ENABLE_LOG_RUNTIME_FILE = env.bool('ENABLE_LOG_RUNTIME_FILE', True)
+ENABLE_LOG_ERROR_FILE = env.bool('ENABLE_LOG_ERROR_FILE', True)
+
+
+LOG_LEVEL_MAP = {
+ DEV_MODE: "DEBUG",
+ TEST_MODE: "INFO",
+ PROD_MODE: "ERROR"
+}
+
+LOG_LEVEL = LOG_LEVEL_MAP.get(APP_ENV)
+LOG_ROTATION = env.str('LOG_ROTATION', '500MB')
+LOG_RETENTION = env.str('LOG_RETENTION', '1 week')
+
+if ENABLE_LOG_FILE:
+ if ENABLE_LOG_RUNTIME_FILE:
+ logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')),
+ level=LOG_LEVEL, rotation=LOG_ROTATION, retention=LOG_RETENTION)
+ if ENABLE_LOG_ERROR_FILE:
+ logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')),
+ level='ERROR', rotation=LOG_ROTATION)
+else:
+ shutil.rmtree(LOG_DIR, ignore_errors=True)
diff --git a/proxypool/storages/__init__.py b/proxypool/storages/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/proxypool/storages/redis.py b/proxypool/storages/redis.py
new file mode 100644
index 00000000..aed45029
--- /dev/null
+++ b/proxypool/storages/redis.py
@@ -0,0 +1,157 @@
+import redis
+from proxypool.exceptions import PoolEmptyException
+from proxypool.schemas.proxy import Proxy
+from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
+ PROXY_SCORE_INIT
+from random import choice, sample
+from typing import List
+from loguru import logger
+from proxypool.utils.proxy import is_valid_proxy, convert_proxy_or_proxies
+
+
+REDIS_CLIENT_VERSION = redis.__version__
+IS_REDIS_VERSION_2 = REDIS_CLIENT_VERSION.startswith('2.')
+
+
+class RedisClient(object):
+ """
+ redis connection client of proxypool
+ """
+
+ def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB,
+ connection_string=REDIS_CONNECTION_STRING, **kwargs):
+ """
+ init redis client
+ :param host: redis host
+ :param port: redis port
+ :param password: redis password
+ :param connection_string: redis connection_string
+ """
+ # if set connection_string, just use it
+ if connection_string:
+ self.db = redis.StrictRedis.from_url(connection_string, decode_responses=True, **kwargs)
+ else:
+ self.db = redis.StrictRedis(
+ host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
+
+ def add(self, proxy: Proxy, score=PROXY_SCORE_INIT, redis_key=REDIS_KEY) -> int:
+ """
+ add proxy and set it to init score
+ :param proxy: proxy, ip:port, like 8.8.8.8:88
+ :param score: int score
+ :return: result
+ """
+ if not is_valid_proxy(f'{proxy.host}:{proxy.port}'):
+ logger.info(f'invalid proxy {proxy}, throw it')
+ return
+ if not self.exists(proxy, redis_key):
+ if IS_REDIS_VERSION_2:
+ return self.db.zadd(redis_key, score, proxy.string())
+ return self.db.zadd(redis_key, {proxy.string(): score})
+
+ def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> Proxy:
+ """
+ get random proxy
+ firstly try to get proxy with max score
+ if not exists, try to get proxy by rank
+ if not exists, raise error
+ :return: proxy, like 8.8.8.8:8
+ """
+ # try to get proxy with max score
+ proxies = self.db.zrangebyscore(
+ redis_key, proxy_score_max, proxy_score_max)
+ if len(proxies):
+ return convert_proxy_or_proxies(choice(proxies))
+ # else get proxy by rank
+ proxies = self.db.zrevrange(
+ redis_key, proxy_score_min, proxy_score_max)
+ if len(proxies):
+ return convert_proxy_or_proxies(choice(proxies))
+ # else raise error
+ raise PoolEmptyException
+
+ def randoms(self, count, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]:
+ """
+ get a batch of random proxies
+ firstly try to get proxies with max score,
+ if not enough, get proxies by rank (score from high to low)
+ if none exists, raise error
+ :param count: number of proxies to return
+ :return: list of proxies
+ """
+ # try to get proxies with max score first
+ proxies = self.db.zrangebyscore(
+ redis_key, proxy_score_max, proxy_score_max)
+ if len(proxies) < count:
+ # not enough max-score proxies, fall back to all proxies by rank
+ proxies = self.db.zrevrangebyscore(
+ redis_key, proxy_score_max, proxy_score_min)
+ if not proxies:
+ raise PoolEmptyException
+ count = min(count, len(proxies))
+ return convert_proxy_or_proxies(sample(proxies, count))
+
+ def decrease(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN) -> int:
+ """
+ decrease score of proxy, if small than PROXY_SCORE_MIN, delete it
+ :param proxy: proxy
+ :return: new score
+ """
+ if IS_REDIS_VERSION_2:
+ self.db.zincrby(redis_key, proxy.string(), -1)
+ else:
+ self.db.zincrby(redis_key, -1, proxy.string())
+ score = self.db.zscore(redis_key, proxy.string())
+ logger.info(f'{proxy.string()} score decrease 1, current {score}')
+ if score <= proxy_score_min:
+ logger.info(f'{proxy.string()} current score {score}, remove')
+ self.db.zrem(redis_key, proxy.string())
+
+ def exists(self, proxy: Proxy, redis_key=REDIS_KEY) -> bool:
+ """
+ if proxy exists
+ :param proxy: proxy
+ :return: if exists, bool
+ """
+ return not self.db.zscore(redis_key, proxy.string()) is None
+
+ def max(self, proxy: Proxy, redis_key=REDIS_KEY, proxy_score_max=PROXY_SCORE_MAX) -> int:
+ """
+ set proxy to max score
+ :param proxy: proxy
+ :return: new score
+ """
+ logger.info(f'{proxy.string()} is valid, set to {proxy_score_max}')
+ if IS_REDIS_VERSION_2:
+ return self.db.zadd(redis_key, proxy_score_max, proxy.string())
+ return self.db.zadd(redis_key, {proxy.string(): proxy_score_max})
+
+ def count(self, redis_key=REDIS_KEY) -> int:
+ """
+ get count of proxies
+ :return: count, int
+ """
+ return self.db.zcard(redis_key)
+
+ def all(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]:
+ """
+ get all proxies
+ :return: list of proxies
+ """
+ return convert_proxy_or_proxies(self.db.zrangebyscore(redis_key, proxy_score_min, proxy_score_max))
+
+ def batch(self, cursor, count, redis_key=REDIS_KEY) -> List[Proxy]:
+ """
+ get batch of proxies
+ :param cursor: scan cursor
+ :param count: scan count
+ :return: list of proxies
+ """
+ cursor, proxies = self.db.zscan(redis_key, cursor, count=count)
+ return cursor, convert_proxy_or_proxies([i[0] for i in proxies])
+
+
+if __name__ == '__main__':
+ conn = RedisClient()
+ result = conn.random()
+ print(result)
diff --git a/proxypool/tester.py b/proxypool/tester.py
deleted file mode 100644
index b43b59f9..00000000
--- a/proxypool/tester.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import asyncio
-import aiohttp
-import time
-import sys
-try:
- from aiohttp import ClientError
-except:
- from aiohttp import ClientProxyConnectionError as ProxyConnectionError
-from proxypool.db import RedisClient
-from proxypool.setting import *
-
-
-class Tester(object):
- def __init__(self):
- self.redis = RedisClient()
-
- async def test_single_proxy(self, proxy):
- """
- 测试单个代理
- :param proxy:
- :return:
- """
- conn = aiohttp.TCPConnector(verify_ssl=False)
- async with aiohttp.ClientSession(connector=conn) as session:
- try:
- if isinstance(proxy, bytes):
- proxy = proxy.decode('utf-8')
- real_proxy = 'http://' + proxy
- print('正在测试', proxy)
- async with session.get(TEST_URL, proxy=real_proxy, timeout=15, allow_redirects=False) as response:
- if response.status in VALID_STATUS_CODES:
- self.redis.max(proxy)
- print('代理可用', proxy)
- else:
- self.redis.decrease(proxy)
- print('请求响应码不合法 ', response.status, 'IP', proxy)
- except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
- self.redis.decrease(proxy)
- print('代理请求失败', proxy)
-
- def run(self):
- """
- 测试主函数
- :return:
- """
- print('测试器开始运行')
- try:
- count = self.redis.count()
- print('当前剩余', count, '个代理')
- for i in range(0, count, BATCH_TEST_SIZE):
- start = i
- stop = min(i + BATCH_TEST_SIZE, count)
- print('正在测试第', start + 1, '-', stop, '个代理')
- test_proxies = self.redis.batch(start, stop)
- loop = asyncio.get_event_loop()
- tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
- loop.run_until_complete(asyncio.wait(tasks))
- sys.stdout.flush()
- time.sleep(5)
- except Exception as e:
- print('测试器发生错误', e.args)
diff --git a/proxypool/testers/__init__.py b/proxypool/testers/__init__.py
new file mode 100644
index 00000000..4e4df95e
--- /dev/null
+++ b/proxypool/testers/__init__.py
@@ -0,0 +1,16 @@
+import pkgutil
+from .base import BaseTester
+import inspect
+
+
+# load classes subclass of BaseCrawler
+classes = []
+for loader, name, is_pkg in pkgutil.walk_packages(__path__):
+ module = loader.find_module(name).load_module(name)
+ for name, value in inspect.getmembers(module):
+ globals()[name] = value
+ if inspect.isclass(value) and issubclass(value, BaseTester) and value is not BaseTester \
+ and not getattr(value, 'ignore', False):
+ classes.append(value)
+__all__ = __ALL__ = classes
+
diff --git a/proxypool/testers/base.py b/proxypool/testers/base.py
new file mode 100644
index 00000000..796b7cfc
--- /dev/null
+++ b/proxypool/testers/base.py
@@ -0,0 +1,19 @@
+from proxypool.setting import TEST_DONT_SET_MAX_SCORE, PROXY_SCORE_INIT, PROXY_SCORE_MAX, PROXY_SCORE_MIN
+
+
+class BaseTester(object):
+ test_url = ""
+ key = ""
+ test_dont_set_max_score = TEST_DONT_SET_MAX_SCORE
+ proxy_score_init = PROXY_SCORE_INIT
+ proxy_score_max = PROXY_SCORE_MAX
+ proxy_score_min = PROXY_SCORE_MIN
+
+ def headers(self):
+ return None
+
+ def cookies(self):
+ return None
+
+ async def parse(self, html, url, proxy, expr='{"code":0'):
+ return True if expr in html else False
diff --git a/proxypool/utils.py b/proxypool/utils.py
deleted file mode 100644
index 16fa2c5a..00000000
--- a/proxypool/utils.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import ConnectionError
-
-base_headers = {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
- 'Accept-Encoding': 'gzip, deflate, sdch',
- 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7'
-}
-
-
-def get_page(url, options={}):
- """
- 抓取代理
- :param url:
- :param options:
- :return:
- """
- headers = dict(base_headers, **options)
- print('正在抓取', url)
- try:
- response = requests.get(url, headers=headers)
- print('抓取成功', url, response.status_code)
- if response.status_code == 200:
- return response.text
- except ConnectionError:
- print('抓取失败', url)
- return None
diff --git a/proxypool/utils/__init__.py b/proxypool/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/proxypool/utils/geo.py b/proxypool/utils/geo.py
new file mode 100644
index 00000000..82a105a0
--- /dev/null
+++ b/proxypool/utils/geo.py
@@ -0,0 +1,35 @@
+from loguru import logger
+
+# geolite2 provides an offline IP -> country database (bundled with the
+# maxminddb_geolite2 dependency). loading it can fail if the optional
+# dependency is missing, so degrade gracefully and disable area filtering.
+try:
+ from geolite2 import geolite2
+
+ _reader = geolite2.reader()
+except Exception as e: # pragma: no cover
+ _reader = None
+ logger.warning(f'geolite2 is unavailable, area filtering disabled: {e}')
+
+
+def get_country_iso(ip):
+ """
+ look up the ISO country code (e.g. 'CN', 'US') for an ip address
+ :param ip: ip address string
+ :return: uppercase iso code, or None if unknown/unavailable
+ """
+ if _reader is None:
+ return None
+ try:
+ record = _reader.get(ip)
+ except Exception:
+ return None
+ if not record:
+ return None
+ country = record.get('country') or record.get('registered_country') or {}
+ return country.get('iso_code')
+
+
+if __name__ == '__main__':
+ print('8.8.8.8', get_country_iso('8.8.8.8'))
+ print('114.114.114.114', get_country_iso('114.114.114.114'))
diff --git a/proxypool/utils/proxy.py b/proxypool/utils/proxy.py
new file mode 100644
index 00000000..ed81ef8b
--- /dev/null
+++ b/proxypool/utils/proxy.py
@@ -0,0 +1,88 @@
+from proxypool.schemas import Proxy
+
+
+def is_valid_proxy(data):
+ """
+ check this string is within proxy format
+ """
+ if is_auth_proxy(data):
+ host, port = extract_auth_proxy(data)
+ return is_ip_valid(host) and is_port_valid(port)
+ elif data.__contains__(':'):
+ ip = data.split(':')[0]
+ port = data.split(':')[1]
+ return is_ip_valid(ip) and is_port_valid(port)
+ else:
+ return is_ip_valid(data)
+
+
+def is_ip_valid(ip):
+ """
+ check this string is within ip format
+ """
+ if is_auth_proxy(ip):
+ ip = ip.split('@')[1]
+ a = ip.split('.')
+ if len(a) != 4:
+ return False
+ for x in a:
+ if not x.isdigit():
+ return False
+ i = int(x)
+ if i < 0 or i > 255:
+ return False
+ return True
+
+
+def is_port_valid(port):
+ return port.isdigit()
+
+
+def convert_proxy_or_proxies(data):
+ """
+ convert list of str to valid proxies or proxy
+ :param data:
+ :return:
+ """
+ if not data:
+ return None
+ # if list of proxies
+ if isinstance(data, list):
+ result = []
+ for item in data:
+ # skip invalid item
+ item = item.strip()
+ if not is_valid_proxy(item): continue
+ if is_auth_proxy(item):
+ host, port = extract_auth_proxy(item)
+ else:
+ host, port, *_ = item.split(':')
+ result.append(Proxy(host=host, port=int(port)))
+ return result
+ if isinstance(data, str) and is_valid_proxy(data):
+ if is_auth_proxy(data):
+ host, port = extract_auth_proxy(data)
+ else:
+ host, port, *_ = data.split(':')
+ return Proxy(host=host, port=int(port))
+
+
+def is_auth_proxy(data: str) -> bool:
+ return '@' in data
+
+
+def extract_auth_proxy(data: str) -> (str, str):
+ """
+ extract host and port from a proxy with authentication
+ """
+ auth = data.split('@')[0]
+ ip_port = data.split('@')[1]
+ ip = ip_port.split(':')[0]
+ port = ip_port.split(':')[1]
+ host = auth + '@' + ip
+ return host, port
+
+
+if __name__ == '__main__':
+ proxy = 'test1234:test5678.@117.68.216.212:32425'
+ print(extract_auth_proxy(proxy))
diff --git a/release.sh b/release.sh
new file mode 100755
index 00000000..342cd06f
--- /dev/null
+++ b/release.sh
@@ -0,0 +1,2 @@
+git tag -a "`date +'%Y%m%d'`" -m "Release `date +'%Y%m%d'`"
+git push origin --tags
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 42e86f0f..49828461 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,16 @@
-aiohttp>=1.3.3
-Flask>=0.11.1
-redis>=2.10.5
-requests>=2.13.0
-pyquery>=1.2.17
+environs>=9.3.0,<12.0.0
+Flask>=2.0.0,<3.0.0
+Werkzeug>=2.0.0,<3.0.0
+attrs>=20.3.0,<24.0.0
+retrying>=1.3.3,<2.0.0
+aiohttp>=3.9.0,<4.0.0
+requests>=2.25.1,<3.0.0
+loguru>=0.5.3,<1.0.0
+pyquery>=1.4.3,<2.0.0
+supervisor>=4.2.1,<5.0.0
+redis>=4.3.0,<6.0.0
+lxml>=4.6.5,<6.0.0
+fake_headers>=1.0.2,<2.0.0
+maxminddb_geolite2==2018.703
+gevent>=22.10.2,<25.0.0
+tornado>=6.0,<7.0
diff --git a/run.py b/run.py
index 50f89f0c..e858da90 100644
--- a/run.py
+++ b/run.py
@@ -1,17 +1,14 @@
from proxypool.scheduler import Scheduler
-import sys
-import io
+import argparse
-sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
-
-
-def main():
- try:
- s = Scheduler()
- s.run()
- except:
- main()
+parser = argparse.ArgumentParser(description='ProxyPool')
+parser.add_argument('--processor', type=str, help='processor to run')
+args = parser.parse_args()
if __name__ == '__main__':
- main()
+ # if processor set, just run it
+ if args.processor:
+ getattr(Scheduler(), f'run_{args.processor}')()
+ else:
+ Scheduler().run()
diff --git a/supervisord.conf b/supervisord.conf
new file mode 100644
index 00000000..aff2cd64
--- /dev/null
+++ b/supervisord.conf
@@ -0,0 +1,40 @@
+[unix_http_server]
+file=/run/supervisor.sock
+chmod=0700
+
+[supervisord]
+pidfile=/run/supervisord.pid
+nodaemon=true
+
+[supervisorctl]
+serverurl=unix:///run/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
+
+[program:tester]
+process_name=tester
+command=python3 run.py --processor tester
+directory=/app
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:getter]
+process_name=getter
+command=python3 run.py --processor getter
+directory=/app
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:server]
+process_name=server
+command=python3 run.py --processor server
+directory=/app
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
\ No newline at end of file