diff --git a/PROXIES/db.conf b/PROXIES/db.conf new file mode 100644 index 0000000..1066d95 --- /dev/null +++ b/PROXIES/db.conf @@ -0,0 +1,31 @@ +[mysql] + +HOST = 172.20.6.100 +PORT = 3306 +USER = root +PASSWD = mysqladmin +DB = pydb +TABLE = pytab +CHARSET = utf8 + + +[redis] + +HOST = 172.20.6.100 +PORT = 6379 +PASSWD = redisadmin + + +[memcache] + +HOST = 172.20.6.100 +PORT = 11211 + + +[mongodb] + +HOST = 172.20.6.100 +PORT = 27017 +DB = db1 +USER = mongoadmin +PASSWD = mongopwd diff --git a/PROXIES/save_memcache.py b/PROXIES/save_memcache.py new file mode 100644 index 0000000..8edf889 --- /dev/null +++ b/PROXIES/save_memcache.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import memcache +import random +import configparser +import spider + +class MemcacheOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['memcache']['HOST'] + self.port = config['memcache']['PORT'] + self.mcoper = memcache.Client([self.host+':'+self.port], debug = True) + + def memcache_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.mcoper.set(str(num),cont) + + def memcache_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.mcoper.get(str(num)) + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MemcacheOper() + dbhelper.memcache_save(proxy_ip) + ip = dbhelper.memcache_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mongodb.py b/PROXIES/save_mongodb.py new file mode 100644 index 0000000..0f76994 --- /dev/null +++ b/PROXIES/save_mongodb.py @@ -0,0 +1,59 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + + + +import configparser +import spider +from pymongo import MongoClient + +class MongodbOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mongodb']['HOST'] + self.port = config['mongodb']['PORT'] + self.db = config['mongodb']['DB'] + self.user = config['mongodb']['USER'] + self.pwd = config['mongodb']['PASSWD'] + self.client = MongoClient(self.host, int(self.port)) + self.db_auth = self.client.admin + self.db_auth.authenticate(self.user,self.pwd) + self.DB = self.client[self.db] + self.collection = self.DB.myset + + def mongodb_save(self,result_list): + """ + save data + :return:None + """ + + for values in result_list: + self.collection.insert(values) + + def mongodb_gain(self): + """ + gain data + :return: proxies + """ + ip = self.collection.find_one() + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MongodbOper() + dbhelper.mongodb_save(proxy_ip) + ip = dbhelper.mongodb_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mysql.py b/PROXIES/save_mysql.py new file mode 100644 index 0000000..f7c40c7 --- /dev/null +++ b/PROXIES/save_mysql.py @@ -0,0 +1,64 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import pymysql +import configparser +import spider + + +class MysqlOper: + # initial database information + def __init__(self, result_list): + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mysql']['HOST'] + self.port = int(config['mysql']['PORT']) + self.user = config['mysql']['USER'] + self.passwd = config['mysql']['PASSWD'] + self.db = config['mysql']['DB'] + self.table = config['mysql']['TABLE'] + self.charset = config['mysql']['CHARSET'] + self.result_list = result_list + + def mysql_save(self): + + # create db cursor + try: + DB = pymysql.connect(self.host, self.user, self.passwd, self.db, port=self.port, charset=self.charset) + cursor = DB.cursor() + except Exception as e: + print("connect dbserver fail,Please see information:") + print(e) + exit(1) + + # check and create tables + cursor.execute('show tables in pydb') + tables = cursor.fetchall() + flag = True + for tab in tables: + if self.table in tab: + flag = False + print('%s is exist' % self.table) + print(flag) + if flag: + cursor.execute( + '''create table pytab (id int unsigned not null primary key auto_increment, protocol varchar(10),content varchar(50))''') + else: + return 0 + + # write database + for values in self.result_list: + for prot, cont in values.items(): + try: + cursor.execute("insert into pytab (protocol,content) value (%s,%s);", [prot, cont]) + except Exception as e: + print("insert db occer error", e) + + +if __name__ == "__main__": + proxyhelper = spider.GetProxyIP(3) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MysqlOper(proxy_ip) + dbhelper.mysql_save() diff --git a/PROXIES/save_redis.py b/PROXIES/save_redis.py new file mode 100644 index 0000000..5c7bb91 --- /dev/null +++ b/PROXIES/save_redis.py @@ -0,0 +1,52 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import redis +import random +import configparser +import spider + +class RedisOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['redis']['HOST'] + self.port = config['redis']['PORT'] + self.passwd = config['redis']['PASSWD'] + self.pool = redis.ConnectionPool(host=self.host,port=self.port,password=self.passwd) + self.redis_helper = redis.Redis(connection_pool=self.pool) + self.pipe = self.redis_helper.pipeline(transaction=True) + + def redis_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.redis_helper.set(num,cont) + self.pipe.execute() + + def redis_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.redis_helper.get(num) + self.pipe.execute() + return ip + +if __name__ == '__main__': + # proxyhelper = spider.GetProxyIP(2) + # res_pool = proxyhelper.get_ip() + # proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = RedisOper() + # dbhelper.redis_save(proxy_ip) + ip = dbhelper.redis_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/spider.py b/PROXIES/spider.py new file mode 100644 index 0000000..9b538c6 --- /dev/null +++ b/PROXIES/spider.py @@ -0,0 +1,61 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import requests +from bs4 import BeautifulSoup +import random + +class GetProxyIP: + + def __init__(self,page=10): + self._page = page + self.url_head = 'http://www.xicidaili.com/wt/' + + def get_ip(self): + """ + get resouce proxy ip pool + :return: res_pool list + """ + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"} + res_pool = [] + for pagenum in range(1,self._page): + url = self.url_head + str(pagenum) + response = requests.get(url, headers=headers) + soup = BeautifulSoup(response.text, "html.parser") + soup_tr = soup.find_all('tr') + for item in soup_tr: + try: + soup_td = item.find_all('td') + res_pool.append(soup_td[5].text.lower() + '://' + soup_td[1].text + ':' + soup_td[2].text) + except IndexError: + pass + return res_pool + + def right_proxies(self,res_pool): + """ + check available ip + :param res_pool: + :return:right_pool list + """ + right_pool = [] + for ip in res_pool: + if 'https' in ip: + proxies = {'http': ip} + else: + proxies = {"http": ip} + check_urllist = ['http://www.baidu.com', 'http://www.taobao.com', 'https://cloud.tencent.com/'] + try: + response = requests.get(random.choice(check_urllist), proxies=proxies, timeout = 1) + if response.status_code: + right_pool.append(proxies) + print('add ip %s' % proxies) + except Exception as e: + continue + return right_pool + +if __name__ == '__main__': + proxyhelper = GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip =proxyhelper.right_proxies(res_pool) + print(proxy_ip) diff --git a/RDSBAK/Rds_backup.py b/RDSBAK/Rds_backup.py new file mode 100644 index 0000000..d01cddc --- /dev/null +++ b/RDSBAK/Rds_backup.py @@ -0,0 +1,69 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from aliyunsdkcore import client +from aliyunsdkrds.request.v20140815 import CreateBackupRequest +import time +import os +import logging + +class rdsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('LTAIhfXlcjyl****','GwfAM344K2ELm345184356TVgRfAso','cn-shanghai') + self.logger = logger + self.logger.info("------------------------start exec rds backup API log-------------") + def backup_instance(self): + # 设置参数 + request = CreateBackupRequest.CreateBackupRequest() + request.set_accept_format('json') + request.add_query_param('DBInstanceId', 'rm-uf6x**5u1x842y61y') + + #如果为单库备份，可以添加DBName + # request.add_query_param('DBName', 'mydb') + + #BackupMethod为备份方式：Logical：逻辑备份，Physical：物理备份 + request.add_query_param('BackupMethod', 'Physical') + #BackupType为备份类型： Auto：自动计算是全量备份还是增量备份；FullBackup：全量备份。默认值为Auto。 + request.add_query_param('BackupType', 'Auto') + + response = self.clentoper.do_action_with_exception(request) + self.logger.info("rdsbackup mission submission successful!") + self.logger.info(response) + print(response) + + +class Rds_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'rdsbackuplog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Rds_Log('rdsbackup.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = rdsOper(logger) + app.backup_instance() + + diff --git a/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log new file mode 100644 index 0000000..02ab8ae --- /dev/null +++ b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log @@ -0,0 +1,3 @@ +2018-06-04 15:40:41,364 - root - INFO - ------------------------start exec rds backup API log------------- +2018-06-04 15:40:41,767 - root - INFO - rdsbackup mission submission successful! +2018-06-04 15:40:41,767 - root - INFO - b'{"RequestId":"2AA53A09-33D3-4A26-94D3-A745301E3A02"}' diff --git a/README.md b/README.md index 082ea2b..4d76d00 100644 --- a/README.md +++ b/README.md @@ -1 +1,37 @@ # my-python-code +## 重启深信服(SANGFOROPER) +> Python深信服重启 + +blog：[Python重启深信服设备](http://blog.51cto.com/kaliarch/2095178) + +## 翻译小工具(FANYI) +> Python实现翻译小工具 + +blog：[Python实现翻译小工具](http://blog.51cto.com/kaliarch/2072150) + +## 汽车票查询(XAGLKP) +> python搜索汽车票 + +blog：[python搜索汽车票](http://blog.51cto.com/kaliarch/2071288) + +## 电影爬取(DYTT8) +> 爬取电影并存储到excel + +blog：[爬取搜索出来的电影的下载地址并保存到excel](http://blog.51cto.com/kaliarch/2069544) + +## 爬取推荐博客(51BLOG) +> 爬取推进博客 + +blog：[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto.com/kaliarch/2067103) + +## 构建自己的代理库(PROXIES) +> Python构建自己的代理库 + +blog：[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) + + +## Python自定义阿里云RDS备份策略(RDSBAK) +> Python自定义阿里云RDS备份策略 + +blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/kaliarch/2124609) + diff --git a/SANGFOROPER/Check_scripts.sh b/SANGFOROPER/Check_scripts.sh new file mode 100644 index 0000000..a31edf8 --- /dev/null +++ b/SANGFOROPER/Check_scripts.sh @@ -0,0 +1,8 @@ +#!/bin/bash +num=$(ps -ef |grep pdc.sh|wc -l) +cmd="/usr/bin/nohup /bin/bash /sangfor/Shscripts/pdc/Ping_check.sh &" + +if [ ${num} -lt 2 ];then +${cmd} +fi + diff --git a/SANGFOROPER/Ecs_apireboot.py b/SANGFOROPER/Ecs_apireboot.py new file mode 100644 index 0000000..a6610c2 --- /dev/null +++ b/SANGFOROPER/Ecs_apireboot.py @@ -0,0 +1,81 @@ +#!/bin/env python3 +# -*- coding:UTF-8 -*- +# _author:kaliarch +from aliyunsdkcore import client +from aliyunsdkecs.request.v20140526 import RebootInstanceRequest,StartInstanceRequest,StopInstanceRequest +import time +import os +import logging +class ecsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('', '', 'cn-hangzhou') + self.logger = logger + self.logger.info("------------------------start reboot vpn ecs of API log-------------") + def reboot_instance(self): + # 设置参数 + request = RebootInstanceRequest.RebootInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response) + print(response) + + def start_instance(self): + request = StartInstanceRequest.StartInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn start successful!") + self.logger.info(response) + print(response) + + def stop_instance(self): + request = StopInstanceRequest.StopInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + request.add_query_param('ForceStop', 'false') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info(response) + print(response) + + def testlog(self): + self.logger.info("public test log") + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = ecsOper(logger) + app.reboot_instance() + diff --git a/SANGFOROPER/Ping_check.sh b/SANGFOROPER/Ping_check.sh new file mode 100644 index 0000000..7ce9231 --- /dev/null +++ b/SANGFOROPER/Ping_check.sh @@ -0,0 +1,42 @@ +#!/bin/bash +IP=10.10.10.2 +dir="/sangfor/Shscripts/pdc/" +if [ ! -d ${dir} ];then + mkdir -p ${dir} +fi +echo 1 > ${dir}pdcping.lock +while true +do + Time=`date +%F` + TIME="${Time} 23:59" + if [ "${data}" == "${TIME}" ];then + mkdir ${dir}${Time} && mv ${dir}pdcping.log ${dir}${Time}-pingpdc.log + mv ${dir}${Time}-pingpdc.log ${dir}${Time} + fi + find ${dir} -mtime +7 -type d -exec rm -rf {} \; + + find ${dir} -mtime +7 -name "*-pingpdc.log" -exec rm -rf {} \; + data=`date +%F' '%H:%M` + data1=`date +%F' '%H:%M:%S` + echo "------------${data1}---------------">>${dir}pingpdc.log + ping -c 10 ${IP} >>${dir}pingpdc.log + if [ $? -eq 1 ];then + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 1 ];then + /usr/local/python34/bin/python3 /sangfor/Pysangfor/SangFor_check.py + echo 0 > ${dir}pdcping.lock + else + continue + fi + else + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 0 ];then + echo 1 > ${dir}pdcping.lock + else + continue + fi + fi + + +done + diff --git a/SANGFOROPER/SangFor_oper.py b/SANGFOROPER/SangFor_oper.py new file mode 100644 index 0000000..cbfb72a --- /dev/null +++ b/SANGFOROPER/SangFor_oper.py @@ -0,0 +1,90 @@ +#!/bin/env python3 +from pyvirtualdisplay import Display +from selenium import webdriver +import time +import os +import logging + +class Glp_SangFor: + def __init__(self,logger): + self.logger = logger + self.logger.info("--------------start log----------------") + self.display = Display(visible=0, size=(800, 600)) + self.display.start() + self.browser = webdriver.Firefox() + self.logger.info("start browser successfuly") + self.sangfor_url = "深信服url" + self.username = '深信服用户名' + self.password = '深信服密码' + + def login(self): + self.browser.get(self.sangfor_url) + self.browser.implicitly_wait(5) + self.browser.find_element_by_name('user').send_keys(self.username) + self.browser.find_element_by_name('password').send_keys(self.password) + self.browser.find_element_by_class_name('buttons').click() + self.browser.implicitly_wait(5) + self.logger.info("loggin sangfor successfuly") + + def client_reboot(self): + self.browser.find_element_by_id("ext-gen111").click() + print(self.browser.find_element_by_id("ext-gen111").text) + self.browser.implicitly_wait(15) + time.sleep(60) + self.logger.info("switch mainiframe start") + try: + print(self.browser.find_element_by_link_text("重启/重启服务/关机").text) + self.browser.find_element_by_link_text("重启/重启服务/关机").click() + self.browser.implicitly_wait(3) + self.browser.switch_to_frame("mainiframe") + self.browser.implicitly_wait(8) + time.sleep(10) + self.browser.find_element_by_xpath("//button[@id='ext-gen19']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen19']").text) + self.browser.implicitly_wait(10) + #self.browser.find_element_by_xpath("//button[@id='ext-gen42']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen42']").text) + except Exception as e: + self.logger.exception("reboot successful") + return 1 + self.browser.close() + self.logger.info("browser close successful") + self.logger.info("--------------end log----------------") + return 0 + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == '__main__': + os.system("pkill firefox") + os.system("pkill Xvfb") + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + sangfor_oper = Glp_SangFor(logger) + sangfor_oper.login() + sangfor_oper.client_reboot()