diff --git "a/Python \351\273\221\351\255\224\346\263\225/Python \350\277\234\347\250\213\345\274\200\346\234\272.py" "b/Python \351\273\221\351\255\224\346\263\225/Python \350\277\234\347\250\213\345\274\200\346\234\272.py" new file mode 100644 index 00000000..1b23c160 --- /dev/null +++ "b/Python \351\273\221\351\255\224\346\263\225/Python \350\277\234\347\250\213\345\274\200\346\234\272.py" @@ -0,0 +1,28 @@ +def wake_up(request, mac='DC-4A-3E-78-3E-0A'): + MAC = mac + BROADCAST = "192.168.0.255" + if len(MAC) != 17: + raise ValueError("MAC address should be set as form 'XX-XX-XX-XX-XX-XX'") + mac_address = MAC.replace("-", '') + data = ''.join(['FFFFFFFFFFFF', mac_address * 20]) # 构造原始数据格式 + send_data = b'' + + # 把原始数据转换为16进制字节数组， + for i in range(0, len(data), 2): + send_data = b''.join([send_data, struct.pack('B', int(data[i: i + 2], 16))]) + print(send_data) + + # 通过socket广播出去，为避免失败，间隔广播三次 + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + sock.sendto(send_data, (BROADCAST, 7)) + time.sleep(1) + sock.sendto(send_data, (BROADCAST, 7)) + time.sleep(1) + sock.sendto(send_data, (BROADCAST, 7)) + return HttpResponse() + print("Done") + except Exception as e: + return HttpResponse() + print(e) \ No newline at end of file diff --git "a/Python \351\273\221\351\255\224\346\263\225/README.MD" "b/Python \351\273\221\351\255\224\346\263\225/README.MD" new file mode 100644 index 00000000..248602a7 --- /dev/null +++ "b/Python \351\273\221\351\255\224\346\263\225/README.MD" @@ -0,0 +1,4 @@ +# 代码详细说明请看文章 + +[Python 远程关机](https://mp.weixin.qq.com/s/RSod4XWxyzL32eNcrXLjUQ) + diff --git a/README.md b/README.md index ec80c6cd..a95b5974 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,78 @@ -### 这是我日常遇到的一些小问题的解决办法，全部是基于Python3 -1.[获取当前CPU状态，存储到Influxdb](https://github.com/injetlee/demo/blob/master/CpuToInfluxdb.py) +# 欢迎关注我的微信公众号【流浪土土】, 用 AI 探索自由生长的可能。 -2.[模拟登录知乎](https://github.com/injetlee/demo/blob/master/login_zhihu.py) +## 专注于 AI，Vibe coding , 独立开发等相关内容分享 -3.[对目录下所有文件计数](https://github.com/injetlee/demo/blob/master/countFile.py) +## 可以通过微信公众号加我好友 +二维码 -4.[爬取豆瓣电影top250](https://github.com/injetlee/demo/blob/master/douban_movie.py) +✨━━━━━━━ 赞助内容 ━━━━━━━✨ -5.[Excel文件读入数据库](https://github.com/injetlee/demo/blob/master/excelToDatabase.py) +### IPWO爬虫代理资源为采集、跨境与测试项目提供支持(免费试用，爬虫使用强烈推荐!!! 学习者务必遵守法律) +**IPWO提供的真实住宅 IP，大幅降低被封禁风险。195 + 国家/地区精准定位，轻松应对大型爬虫任务。让爬虫更简单，让数据更安全。** +#### 官网地址 +#### 👉 [**访问IPWO官网**](https://www.ipwo.net/?ref=githubaerjie) -6.[爬取拉勾网职位信息](https://github.com/injetlee/demo/blob/master/lagouSpider.py) +#### 产品简介： +- 9000万+真实住宅IP，覆盖195+国家和地区 +- 支持动态住宅代理、静态住宅代理(ISP) +- 适用于数据抓取、电商、广告验证、SEO监控等场景 +- 支持HTTP/HTTPS/SOCKS5协议，兼容性强 +- 纯净IP池，实时更新，99.9%连接成功率 +- 支持指定国家城市地区访问，保护隐私 -7.[批量修改文件名](https://github.com/injetlee/demo/blob/master/ModifyFilename.py) +✨━━━━━━━━━━━━━━━━━━━━━✨ -8.[读写excel](https://github.com/injetlee/demo/blob/master/readExcel.py) +# 内容列表 -9.[下载必应首页图片,只下载当天的，一张。](https://github.com/injetlee/Python/blob/master/biyingSpider.py) +## [Python微信公众号开发](https://github.com/injetlee/Python/tree/master/wechat) + +- ### Python 微信公众号开发—小白篇(一) + +- ### Python 公众号开发—颜值检测 + +## [Python 爬虫入门合集](https://github.com/injetlee/Python/tree/master/%E7%88%AC%E8%99%AB%E9%9B%86%E5%90%88) + +- ### Python 爬虫入门(一)——爬取糗事百科 + +- ### Python 爬虫入门(二)——爬取妹子图 + +- ### Python 爬虫——Python 岗位分析报告 + +- ### Python 爬虫利器——Selenium介绍 + +- ### Python 爬虫—— 抖音 App 视频抓包爬取 + +## [Python 黑魔法](https://github.com/injetlee/Python/tree/master/Python%20%E9%BB%91%E9%AD%94%E6%B3%95) + +- ### Python 远程关机 + +## SQL 数据库 + +- [1 小时 SQL 极速入门（一）](https://mp.weixin.qq.com/s/Lx4B349OlD49ihJPnB6YiA) +- [1 小时 SQL 极速入门（二）](https://mp.weixin.qq.com/s/D-CEtGYomne5kV_Ji4lodA) +- [1 小时 SQL 极速入门（三）](https://mp.weixin.qq.com/s/7aJqrhCNcvnt2gO3p5P50Q) +- [SQL 高级查询——（层次化查询，递归）](https://mp.weixin.qq.com/s/R9Yldd-5AK4ObRA9Lfbz-Q) +- [GROUP BY高级查询,ROLLUP，CUBE，GROUPPING详解](https://mp.weixin.qq.com/s/_OK6dtHGhp7ukC2pe1ginQ) +- [SQL 行转列，列转行](https://mp.weixin.qq.com/s/xOFIg42FQhNpyg94ajhtqQ) + +## 其他 + +- 1.[获取当前CPU状态，存储到Influxdb](https://github.com/injetlee/demo/blob/master/CpuToInfluxdb.py) + +- 2.[模拟登录知乎](https://github.com/injetlee/demo/blob/master/login_zhihu.py) + +- 3.[对目录下所有文件计数](https://github.com/injetlee/demo/blob/master/countFile.py) + +- 4.[爬取豆瓣电影top250](https://github.com/injetlee/demo/blob/master/douban_movie.py) + +- 5.[Excel文件读入数据库](https://github.com/injetlee/demo/blob/master/excelToDatabase.py) + +- 6.[爬取拉勾网职位信息](https://github.com/injetlee/demo/blob/master/lagouSpider.py) + +- 7.[批量修改文件名](https://github.com/injetlee/demo/blob/master/ModifyFilename.py) + +- 8.[读写excel](https://github.com/injetlee/demo/blob/master/readExcel.py) + +- 9.[下载必应首页图片,只下载当天的，一张。](https://github.com/injetlee/Python/blob/master/biyingSpider.py) diff --git a/biyingSpider.py b/biyingSpider.py index 86bf645b..b5fae1ed 100644 --- a/biyingSpider.py +++ b/biyingSpider.py @@ -5,10 +5,11 @@ url = 'http://cn.bing.com/' con = requests.get(url) content = con.text -reg = r"(http://s.cn.bing.net/az/hprichbg/rb/.*?.jpg)" +reg = r"(az/hprichbg/rb/.*?.jpg)" a = re.findall(reg, content, re.S)[0] print(a) -read = requests.get(a) +picUrl = url + a +read = requests.get(picUrl) f = open('%s.jpg' % local, 'wb') f.write(read.content) f.close() diff --git a/qiubai_crawer.py b/qiubai_crawer.py new file mode 100644 index 00000000..e37e7e7d --- /dev/null +++ b/qiubai_crawer.py @@ -0,0 +1,54 @@ +import requests +from bs4 import BeautifulSoup + + +def download_page(url): + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"} + r = requests.get(url, headers=headers) + return r.text + + +def get_content(html, page): + output = """第{}页 作者：{} 性别：{} 年龄：{} 点赞：{} 评论：{}\n{}\n------------\n""" + soup = BeautifulSoup(html, 'html.parser') + con = soup.find(id='content-left') + con_list = con.find_all('div', class_="article") + for i in con_list: + author = i.find('h2').string # 获取作者名字 + content = i.find('div', class_='content').find('span').get_text() # 获取内容 + stats = i.find('div', class_='stats') + vote = stats.find('span', class_='stats-vote').find('i', class_='number').string + comment = stats.find('span', class_='stats-comments').find('i', class_='number').string + author_info = i.find('div', class_='articleGender') # 获取作者 年龄，性别 + if author_info is not None: # 非匿名用户 + class_list = author_info['class'] + if "womenIcon" in class_list: + gender = '女' + elif "manIcon" in class_list: + gender = '男' + else: + gender = '' + age = author_info.string # 获取年龄 + else: # 匿名用户 + gender = '' + age = '' + + save_txt(output.format(page, author, gender, age, vote, comment, content)) + + +def save_txt(*args): + for i in args: + with open('qiubai.txt', 'a', encoding='utf-8') as f: + f.write(i) + + +def main(): + # 我们点击下面链接，在页面下方可以看到共有13页，可以构造如下 url， + # 当然我们最好是用 Beautiful Soup找到页面底部有多少页。 + for i in range(1, 14): + url = 'https://qiushibaike.com/text/page/{}'.format(i) + html = download_page(url) + get_content(html, i) + +if __name__ == '__main__': + main() diff --git a/qrcode.jpg b/qrcode.jpg new file mode 100644 index 00000000..f3a11691 Binary files /dev/null and b/qrcode.jpg differ diff --git a/wechat/README.MD b/wechat/README.MD new file mode 100644 index 00000000..f4f5386c --- /dev/null +++ b/wechat/README.MD @@ -0,0 +1,5 @@ +# 详细使用请看文章 + +[Python微信公众号开发—小白篇(一)](https://mp.weixin.qq.com/s/iMPUC0yxI-zuf4AjtyAu6g) + +[Python公众号开发—颜值检测](https://mp.weixin.qq.com/s/I0DxhIHkeqhc2LeQ2ICHeA) \ No newline at end of file diff --git a/wechat/connect.py b/wechat/connect.py new file mode 100644 index 00000000..3e0ca008 --- /dev/null +++ b/wechat/connect.py @@ -0,0 +1,52 @@ +# -*-coding:utf-8 -*- +import falcon +from falcon import uri +from wechatpy.utils import check_signature +from wechatpy.exceptions import InvalidSignatureException +from wechatpy import parse_message +from wechatpy.replies import TextReply, ImageReply + +from utils import img_download, img_upload +from face_id import access_api + + +class Connect(object): + + def on_get(self, req, resp): + query_string = req.query_string + query_list = query_string.split('&') + b = {} + for i in query_list: + b[i.split('=')[0]] = i.split('=')[1] + + try: + check_signature(token='lengxiao', signature=b['signature'], timestamp=b['timestamp'], nonce=b['nonce']) + resp.body = (b['echostr']) + except InvalidSignatureException: + pass + resp.status = falcon.HTTP_200 + + def on_post(self, req, resp): + xml = req.stream.read() + msg = parse_message(xml) + if msg.type == 'text': + reply = TextReply(content=msg.content, message=msg) + xml = reply.render() + resp.body = (xml) + resp.status = falcon.HTTP_200 + elif msg.type == 'image': + name = img_download(msg.image, msg.source) + print(name) + r = access_api('images/' + name) + if r == 'success': + media_id = img_upload('image', 'faces/' + name) + reply = ImageReply(media_id=media_id, message=msg) + else: + reply = TextReply(content='人脸检测失败，请上传1M以下人脸清晰的照片', message=msg) + xml = reply.render() + resp.body = (xml) + resp.status = falcon.HTTP_200 + +app = falcon.API() +connect = Connect() +app.add_route('/connect', connect) diff --git a/wechat/face_id.py b/wechat/face_id.py new file mode 100644 index 00000000..6726f153 --- /dev/null +++ b/wechat/face_id.py @@ -0,0 +1,137 @@ +# -*-coding:utf-8 -*- +import time +import random +import base64 +import hashlib +import requests +from urllib.parse import urlencode +import cv2 +import numpy as np +from PIL import Image, ImageDraw, ImageFont +import os + + +# 一.计算接口鉴权，构造请求参数 + +def random_str(): + '''得到随机字符串nonce_str''' + str = 'abcdefghijklmnopqrstuvwxyz' + r = '' + for i in range(15): + index = random.randint(0,25) + r += str[index] + return r + + +def image(name): + with open(name, 'rb') as f: + content = f.read() + return base64.b64encode(content) + + +def get_params(img): + '''组织接口请求的参数形式，并且计算sign接口鉴权信息， + 最终返回接口请求所需要的参数字典''' + params = { + 'app_id': '1106860829', + 'time_stamp': str(int(time.time())), + 'nonce_str': random_str(), + 'image': img, + 'mode': '0' + + } + + sort_dict = sorted(params.items(), key=lambda item: item[0], reverse=False) # 排序 + sort_dict.append(('app_key', 'P8Gt8nxi6k8vLKbS')) # 添加app_key + rawtext = urlencode(sort_dict).encode() # URL编码 + sha = hashlib.md5() + sha.update(rawtext) + md5text = sha.hexdigest().upper() # 计算出sign，接口鉴权 + params['sign'] = md5text # 添加到请求参数列表中 + return params + +# 二.请求接口URL + + +def access_api(img): + frame = cv2.imread(img) + nparry_encode = cv2.imencode('.jpg', frame)[1] + data_encode = np.array(nparry_encode) + img_encode = base64.b64encode(data_encode) # 图片转为base64编码格式 + url = 'https://api.ai.qq.com/fcgi-bin/face/face_detectface' + res = requests.post(url, get_params(img_encode)).json() # 请求URL,得到json信息 + # 把信息显示到图片上 + if res['ret'] == 0: # 0代表请求成功 + pil_img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) # 把opencv格式转换为PIL格式，方便写汉字 + draw = ImageDraw.Draw(pil_img) + for obj in res['data']['face_list']: + img_width = res['data']['image_width'] # 图像宽度 + img_height = res['data']['image_height'] # 图像高度 + # print(obj) + x = obj['x'] # 人脸框左上角x坐标 + y = obj['y'] # 人脸框左上角y坐标 + w = obj['width'] # 人脸框宽度 + h = obj['height'] # 人脸框高度 + # 根据返回的值，自定义一下显示的文字内容 + if obj['glass'] == 1: # 眼镜 + glass = '有' + else: + glass = '无' + if obj['gender'] >= 70: # 性别值从0-100表示从女性到男性 + gender = '男' + elif 50 <= obj['gender'] < 70: + gender = "娘" + elif obj['gender'] < 30: + gender = '女' + else: + gender = '女汉子' + if 90 < obj['expression'] <= 100: # 表情从0-100，表示笑的程度 + expression = '一笑倾城' + elif 80 < obj['expression'] <= 90: + expression = '心花怒放' + elif 70 < obj['expression'] <= 80: + expression = '兴高采烈' + elif 60 < obj['expression'] <= 70: + expression = '眉开眼笑' + elif 50 < obj['expression'] <= 60: + expression = '喜上眉梢' + elif 40 < obj['expression'] <= 50: + expression = '喜气洋洋' + elif 30 < obj['expression'] <= 40: + expression = '笑逐颜开' + elif 20 < obj['expression'] <= 30: + expression = '似笑非笑' + elif 10 < obj['expression'] <= 20: + expression = '半嗔半喜' + elif 0 <= obj['expression'] <= 10: + expression = '黯然伤神' + delt = h // 5 # 确定文字垂直距离 + # 写入图片 + if len(res['data']['face_list']) > 1: # 检测到多个人脸，就把信息写入人脸框内 + font = ImageFont.truetype('yahei.ttf', w // 8, encoding='utf-8') # 提前把字体文件下载好 + draw.text((x + 10, y + 10), '性别 :' + gender, (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 1), '年龄 :' + str(obj['age']), (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 2), '表情 :' + expression, (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 3), '魅力 :' + str(obj['beauty']), (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 4), '眼镜 :' + glass, (76, 176, 80), font=font) + elif img_width - x - w < 170: # 避免图片太窄，导致文字显示不完全 + font = ImageFont.truetype('yahei.ttf', w // 8, encoding='utf-8') + draw.text((x + 10, y + 10), '性别 :' + gender, (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 1), '年龄 :' + str(obj['age']), (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 2), '表情 :' + expression, (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 3), '魅力 :' + str(obj['beauty']), (76, 176, 80), font=font) + draw.text((x + 10, y + 10 + delt * 4), '眼镜 :' + glass, (76, 176, 80), font=font) + else: + font = ImageFont.truetype('yahei.ttf', 20, encoding='utf-8') + draw.text((x + w + 10, y + 10), '性别 :' + gender, (76, 176, 80), font=font) + draw.text((x + w + 10, y + 10 + delt * 1), '年龄 :' + str(obj['age']), (76, 176, 80), font=font) + draw.text((x + w + 10, y + 10 + delt * 2), '表情 :' + expression, (76, 176, 80), font=font) + draw.text((x + w + 10, y + 10 + delt * 3), '魅力 :' + str(obj['beauty']), (76, 176, 80), font=font) + draw.text((x + w + 10, y + 10 + delt * 4), '眼镜 :' + glass, (76, 176, 80), font=font) + + draw.rectangle((x, y, x + w, y + h), outline="#4CB050") # 画出人脸方框 + cv2img = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) # 把 pil 格式转换为 cv + cv2.imwrite('faces/{}'.format(os.path.basename(img)), cv2img) # 保存图片到 face 文件夹下 + return 'success' + else: + return 'fail' \ No newline at end of file diff --git a/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg b/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg new file mode 100644 index 00000000..1b06d215 Binary files /dev/null and b/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg differ diff --git a/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg b/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg new file mode 100644 index 00000000..4b2b43c9 Binary files /dev/null and b/wechat/faces/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg differ diff --git a/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg b/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg new file mode 100644 index 00000000..47a1d9b9 Binary files /dev/null and b/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1618_37_43.jpg differ diff --git a/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg b/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg new file mode 100644 index 00000000..a45619de Binary files /dev/null and b/wechat/images/oWFPW1ay1L-73ZFrNkCnq4siIgtQ-2018_06_1619_33_56.jpg differ diff --git a/wechat/requirements.txt b/wechat/requirements.txt new file mode 100644 index 00000000..9d289028 --- /dev/null +++ b/wechat/requirements.txt @@ -0,0 +1,9 @@ +falcon +numpy +opencv-python +Pillow +pycryptodome +requests +waitress +wechatpy +xmltodict diff --git a/wechat/utils.py b/wechat/utils.py new file mode 100644 index 00000000..c082b6f2 --- /dev/null +++ b/wechat/utils.py @@ -0,0 +1,44 @@ +# -*-coding:utf-8 -*- +import requests +import json +import threading +import time +import os + +token = '' +app_id = '开发者ID(AppID)' +secret = '开发者密码(AppSecret)' + + +def img_download(url, name): + r = requests.get(url) + with open('images/{}-{}.jpg'.format(name, time.strftime("%Y_%m_%d%H_%M_%S", time.localtime())), 'wb') as fd: + fd.write(r.content) + if os.path.getsize(fd.name) >= 1048576: + return 'large' + # print('namename', os.path.basename(fd.name)) + return os.path.basename(fd.name) + + +def get_access_token(appid, secret): + '''获取access_token,100分钟刷新一次''' + + url = 'https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid={}&secret={}'.format(appid, secret) + r = requests.get(url) + parse_json = json.loads(r.text) + global token + token = parse_json['access_token'] + global timer + timer = threading.Timer(6000, get_access_token) + timer.start() + + +def img_upload(mediaType, name): + global token + url = "https://api.weixin.qq.com/cgi-bin/media/upload?access_token=%s&type=%s" % (token, mediaType) + files = {'media': open('{}'.format(name), 'rb')} + r = requests.post(url, files=files) + parse_json = json.loads(r.text) + return parse_json['media_id'] + +get_access_token(app_id, secret) \ No newline at end of file diff --git a/wechat/yahei.ttf b/wechat/yahei.ttf new file mode 100644 index 00000000..66baa6c1 Binary files /dev/null and b/wechat/yahei.ttf differ diff --git "a/\347\210\254\350\231\253\351\233\206\345\220\210/README.MD" "b/\347\210\254\350\231\253\351\233\206\345\220\210/README.MD" new file mode 100644 index 00000000..4a64f572 --- /dev/null +++ "b/\347\210\254\350\231\253\351\233\206\345\220\210/README.MD" @@ -0,0 +1,11 @@ +# 代码详细说明请看文章 + +[Python 爬虫入门(一)——爬取糗事百科](https://mp.weixin.qq.com/s/ApnEy6NWS2f-DqIIrhHzGw) + +[Python 爬虫入门(二)——爬取妹子图](https://mp.weixin.qq.com/s/4TZHgoE_yqeDha17f3Tbew) + +[Python 爬虫——Python 岗位分析报告](https://mp.weixin.qq.com/s/8wAHBPnQMbcrP9La7WZiJA) + +[Python 爬虫利器——Selenium介绍](https://mp.weixin.qq.com/s/YJGjZkUejEos_yJ1ukp5kw) + +[Python 爬虫——抖音App视频抓包](https://mp.weixin.qq.com/s/a8Tky_u1u0A4vbssnAK2_g) \ No newline at end of file diff --git "a/\347\210\254\350\231\253\351\233\206\345\220\210/lagou.py" "b/\347\210\254\350\231\253\351\233\206\345\220\210/lagou.py" new file mode 100644 index 00000000..e0982308 --- /dev/null +++ "b/\347\210\254\350\231\253\351\233\206\345\220\210/lagou.py" @@ -0,0 +1,83 @@ +import random +import time + +import requests +from openpyxl import Workbook +import pymysql.cursors + + +def get_conn(): + '''建立数据库连接''' + conn = pymysql.connect(host='localhost', + user='root', + password='root', + db='python', + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + return conn + + +def insert(conn, info): + '''数据写入数据库''' + with conn.cursor() as cursor: + sql = "INSERT INTO `python` (`shortname`, `fullname`, `industryfield`, `companySize`, `salary`, `city`, `education`) VALUES (%s, %s, %s, %s, %s, %s, %s)" + cursor.execute(sql, info) + conn.commit() + + +def get_json(url, page, lang_name): + '''返回当前页面的信息列表''' + headers = { + 'Host': 'www.lagou.com', + 'Connection': 'keep-alive', + 'Content-Length': '23', + 'Origin': 'https://www.lagou.com', + 'X-Anit-Forge-Code': '0', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Accept': 'application/json, text/javascript, */*; q=0.01', + 'X-Requested-With': 'XMLHttpRequest', + 'X-Anit-Forge-Token': 'None', + 'Referer': 'https://www.lagou.com/jobs/list_python?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput=', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7' + } + data = {'first': 'false', 'pn': page, 'kd': lang_name} + json = requests.post(url, data, headers=headers).json() + list_con = json['content']['positionResult']['result'] + info_list = [] + for i in list_con: + info = [] + info.append(i.get('companyShortName', '无')) + info.append(i.get('companyFullName', '无')) + info.append(i.get('industryField', '无')) + info.append(i.get('companySize', '无')) + info.append(i.get('salary', '无')) + info.append(i.get('city', '无')) + info.append(i.get('education', '无')) + info_list.append(info) + return info_list + + +def main(): + lang_name = 'python' + wb = Workbook() # 打开 excel 工作簿 + conn = get_conn() # 建立数据库连接 不存数据库 注释此行 + for i in ['北京', '上海', '广州', '深圳', '杭州']: # 五个城市 + page = 1 + ws1 = wb.active + ws1.title = lang_name + url = 'https://www.lagou.com/jobs/positionAjax.json?city={}&needAddtionalResult=false'.format(i) + while page < 31: # 每个城市30页信息 + info = get_json(url, page, lang_name) + page += 1 + print(i, 'page', page) + time.sleep(random.randint(10, 20)) + for row in info: + insert(conn, tuple(row)) # 插入数据库，若不想存入 注释此行 + ws1.append(row) + conn.close() # 关闭数据库连接，不存数据库 注释此行 + wb.save('{}职位信息.xlsx'.format(lang_name)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/\347\210\254\350\231\253\351\233\206\345\220\210/meizitu.py" "b/\347\210\254\350\231\253\351\233\206\345\220\210/meizitu.py" new file mode 100644 index 00000000..4f6932db --- /dev/null +++ "b/\347\210\254\350\231\253\351\233\206\345\220\210/meizitu.py" @@ -0,0 +1,77 @@ +import requests +import os +import time +import threading +from bs4 import BeautifulSoup + + +def download_page(url): + ''' + 用于下载页面 + ''' + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"} + r = requests.get(url, headers=headers) + r.encoding = 'gb2312' + return r.text + + +def get_pic_list(html): + ''' + 获取每个页面的套图列表,之后循环调用get_pic函数获取图片 + ''' + soup = BeautifulSoup(html, 'html.parser') + pic_list = soup.find_all('li', class_='wp-item') + for i in pic_list: + a_tag = i.find('h3', class_='tit').find('a') + link = a_tag.get('href') + text = a_tag.get_text() + get_pic(link, text) + + +def get_pic(link, text): + ''' + 获取当前页面的图片,并保存 + ''' + html = download_page(link) # 下载界面 + soup = BeautifulSoup(html, 'html.parser') + pic_list = soup.find('div', id="picture").find_all('img') # 找到界面所有图片 + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"} + create_dir('pic/{}'.format(text)) + for i in pic_list: + pic_link = i.get('src') # 拿到图片的具体 url + r = requests.get(pic_link, headers=headers) # 下载图片，之后保存到文件 + with open('pic/{}/{}'.format(text, pic_link.split('/')[-1]), 'wb') as f: + f.write(r.content) + time.sleep(1) # 休息一下，不要给网站太大压力，避免被封 + + +def create_dir(name): + if not os.path.exists(name): + os.makedirs(name) + + +def execute(url): + page_html = download_page(url) + get_pic_list(page_html) + + +def main(): + create_dir('pic') + queue = [i for i in range(1, 72)] # 构造 url 链接 页码。 + threads = [] + while len(queue) > 0: + for thread in threads: + if not thread.is_alive(): + threads.remove(thread) + while len(threads) < 5 and len(queue) > 0: # 最大线程数设置为 5 + cur_page = queue.pop(0) + url = 'http://meizitu.com/a/more_{}.html'.format(cur_page) + thread = threading.Thread(target=execute, args=(url,)) + thread.setDaemon(True) + thread.start() + print('{}正在下载{}页'.format(threading.current_thread().name, cur_page)) + threads.append(thread) + + +if __name__ == '__main__': + main() diff --git "a/\347\210\254\350\231\253\351\233\206\345\220\210/qiubai_crawer.py" "b/\347\210\254\350\231\253\351\233\206\345\220\210/qiubai_crawer.py" new file mode 100644 index 00000000..e37e7e7d --- /dev/null +++ "b/\347\210\254\350\231\253\351\233\206\345\220\210/qiubai_crawer.py" @@ -0,0 +1,54 @@ +import requests +from bs4 import BeautifulSoup + + +def download_page(url): + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"} + r = requests.get(url, headers=headers) + return r.text + + +def get_content(html, page): + output = """第{}页 作者：{} 性别：{} 年龄：{} 点赞：{} 评论：{}\n{}\n------------\n""" + soup = BeautifulSoup(html, 'html.parser') + con = soup.find(id='content-left') + con_list = con.find_all('div', class_="article") + for i in con_list: + author = i.find('h2').string # 获取作者名字 + content = i.find('div', class_='content').find('span').get_text() # 获取内容 + stats = i.find('div', class_='stats') + vote = stats.find('span', class_='stats-vote').find('i', class_='number').string + comment = stats.find('span', class_='stats-comments').find('i', class_='number').string + author_info = i.find('div', class_='articleGender') # 获取作者 年龄，性别 + if author_info is not None: # 非匿名用户 + class_list = author_info['class'] + if "womenIcon" in class_list: + gender = '女' + elif "manIcon" in class_list: + gender = '男' + else: + gender = '' + age = author_info.string # 获取年龄 + else: # 匿名用户 + gender = '' + age = '' + + save_txt(output.format(page, author, gender, age, vote, comment, content)) + + +def save_txt(*args): + for i in args: + with open('qiubai.txt', 'a', encoding='utf-8') as f: + f.write(i) + + +def main(): + # 我们点击下面链接，在页面下方可以看到共有13页，可以构造如下 url， + # 当然我们最好是用 Beautiful Soup找到页面底部有多少页。 + for i in range(1, 14): + url = 'https://qiushibaike.com/text/page/{}'.format(i) + html = download_page(url) + get_content(html, i) + +if __name__ == '__main__': + main()