From 0c3056c5c2536f69275c586431f6bf20f00a3364 Mon Sep 17 00:00:00 2001 From: huangweiqi Date: Tue, 12 Jun 2018 16:34:34 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=95=B4=E7=90=86=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- {baiwan => bai_wan}/app.js | 0 {baiwan => bai_wan}/baiwan.py | 0 {baiwan => bai_wan}/file.txt | 0 {baiwan => bai_wan}/index.html | 0 {baiwan => bai_wan}/question.txt | 0 12306.py => common_util/12306.py | 0 common_util/__init__.py | 0 baiduwenku.py => common_util/baiduwenku.py | 0 .../baiduwenku_pro_1.py | 0 biqukan.py => common_util/biqukan.py | 0 daili.py => common_util/daili.py | 0 douyin.py => common_util/douyin.py | 0 douyin_pro.py => common_util/douyin_pro.py | 0 .../douyin_pro_2.py | 0 downloader.py => common_util/downloader.py | 0 financical.py => common_util/financical.py | 0 geetest.py => common_util/geetest.py | 0 hero.py => common_util/hero.py | 0 shuaia.py => common_util/shuaia.py | 0 {Netease => net_ease}/Netease.py | 470 +++++++++--------- {Netease => net_ease}/music_list.txt | 0 21 files changed, 235 insertions(+), 235 deletions(-) rename {baiwan => bai_wan}/app.js (100%) rename {baiwan => bai_wan}/baiwan.py (100%) rename {baiwan => bai_wan}/file.txt (100%) rename {baiwan => bai_wan}/index.html (100%) rename {baiwan => bai_wan}/question.txt (100%) rename 12306.py => common_util/12306.py (100%) create mode 100644 common_util/__init__.py rename baiduwenku.py => common_util/baiduwenku.py (100%) rename baiduwenku_pro_1.py => common_util/baiduwenku_pro_1.py (100%) rename biqukan.py => common_util/biqukan.py (100%) rename daili.py => common_util/daili.py (100%) rename douyin.py => common_util/douyin.py (100%) rename douyin_pro.py => common_util/douyin_pro.py (100%) rename douyin_pro_2.py => common_util/douyin_pro_2.py (100%) rename downloader.py => common_util/downloader.py (100%) rename financical.py => common_util/financical.py (100%) rename geetest.py => common_util/geetest.py (100%) rename hero.py => common_util/hero.py (100%) rename shuaia.py => common_util/shuaia.py (100%) rename {Netease => net_ease}/Netease.py (96%) rename {Netease => net_ease}/music_list.txt (100%) diff --git a/baiwan/app.js b/bai_wan/app.js similarity index 100% rename from baiwan/app.js rename to bai_wan/app.js diff --git a/baiwan/baiwan.py b/bai_wan/baiwan.py similarity index 100% rename from baiwan/baiwan.py rename to bai_wan/baiwan.py diff --git a/baiwan/file.txt b/bai_wan/file.txt similarity index 100% rename from baiwan/file.txt rename to bai_wan/file.txt diff --git a/baiwan/index.html b/bai_wan/index.html similarity index 100% rename from baiwan/index.html rename to bai_wan/index.html diff --git a/baiwan/question.txt b/bai_wan/question.txt similarity index 100% rename from baiwan/question.txt rename to bai_wan/question.txt diff --git a/12306.py b/common_util/12306.py similarity index 100% rename from 12306.py rename to common_util/12306.py diff --git a/common_util/__init__.py b/common_util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baiduwenku.py b/common_util/baiduwenku.py similarity index 100% rename from baiduwenku.py rename to common_util/baiduwenku.py diff --git a/baiduwenku_pro_1.py b/common_util/baiduwenku_pro_1.py similarity index 100% rename from baiduwenku_pro_1.py rename to common_util/baiduwenku_pro_1.py diff --git a/biqukan.py b/common_util/biqukan.py similarity index 100% rename from biqukan.py rename to common_util/biqukan.py diff --git a/daili.py b/common_util/daili.py similarity index 100% rename from daili.py rename to common_util/daili.py diff --git a/douyin.py b/common_util/douyin.py similarity index 100% rename from douyin.py rename to common_util/douyin.py diff --git a/douyin_pro.py b/common_util/douyin_pro.py similarity index 100% rename from douyin_pro.py rename to common_util/douyin_pro.py diff --git a/douyin_pro_2.py b/common_util/douyin_pro_2.py similarity index 100% rename from douyin_pro_2.py rename to common_util/douyin_pro_2.py diff --git a/downloader.py b/common_util/downloader.py similarity index 100% rename from downloader.py rename to common_util/downloader.py diff --git a/financical.py b/common_util/financical.py similarity index 100% rename from financical.py rename to common_util/financical.py diff --git a/geetest.py b/common_util/geetest.py similarity index 100% rename from geetest.py rename to common_util/geetest.py diff --git a/hero.py b/common_util/hero.py similarity index 100% rename from hero.py rename to common_util/hero.py diff --git a/shuaia.py b/common_util/shuaia.py similarity index 100% rename from shuaia.py rename to common_util/shuaia.py diff --git a/Netease/Netease.py b/net_ease/Netease.py similarity index 96% rename from Netease/Netease.py rename to net_ease/Netease.py index 1c5aacee..855acba6 100644 --- a/Netease/Netease.py +++ b/net_ease/Netease.py @@ -1,236 +1,236 @@ -# -*- coding:utf-8 -*- -import requests, hashlib, sys, click, re, base64, binascii, json, os -from Crypto.Cipher import AES -from http import cookiejar - -""" -Website:http://cuijiahua.com -Author:Jack Cui -Refer:https://github.com/darknessomi/musicbox -""" - -class Encrypyed(): - """ - 解密算法 - """ - def __init__(self): - self.modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7' - self.nonce = '0CoJUm6Qyw8W8jud' - self.pub_key = '010001' - - # 登录加密算法, 基于https://github.com/stkevintan/nw_musicbox脚本实现 - def encrypted_request(self, text): - text = json.dumps(text) - sec_key = self.create_secret_key(16) - enc_text = self.aes_encrypt(self.aes_encrypt(text, self.nonce), sec_key.decode('utf-8')) - enc_sec_key = self.rsa_encrpt(sec_key, self.pub_key, self.modulus) - data = {'params': enc_text, 'encSecKey': enc_sec_key} - return data - - def aes_encrypt(self, text, secKey): - pad = 16 - len(text) % 16 - text = text + chr(pad) * pad - encryptor = AES.new(secKey.encode('utf-8'), AES.MODE_CBC, b'0102030405060708') - ciphertext = encryptor.encrypt(text.encode('utf-8')) - ciphertext = base64.b64encode(ciphertext).decode('utf-8') - return ciphertext - - def rsa_encrpt(self, text, pubKey, modulus): - text = text[::-1] - rs = pow(int(binascii.hexlify(text), 16), int(pubKey, 16), int(modulus, 16)) - return format(rs, 'x').zfill(256) - - def create_secret_key(self, size): - return binascii.hexlify(os.urandom(size))[:16] - - -class Song(): - """ - 歌曲对象，用于存储歌曲的信息 - """ - def __init__(self, song_id, song_name, song_num, song_url=None): - self.song_id = song_id - self.song_name = song_name - self.song_num = song_num - self.song_url = '' if song_url is None else song_url - -class Crawler(): - """ - 网易云爬取API - """ - def __init__(self, timeout=60, cookie_path='.'): - self.headers = { - 'Accept': '*/*', - 'Accept-Encoding': 'gzip,deflate,sdch', - 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', - 'Connection': 'keep-alive', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Host': 'music.163.com', - 'Referer': 'http://music.163.com/search/', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' - } - self.session = requests.Session() - self.session.headers.update(self.headers) - self.session.cookies = cookiejar.LWPCookieJar(cookie_path) - self.download_session = requests.Session() - self.timeout = timeout - self.ep = Encrypyed() - - def post_request(self, url, params): - """ - Post请求 - :return: 字典 - """ - - data = self.ep.encrypted_request(params) - resp = self.session.post(url, data=data, timeout=self.timeout) - result = resp.json() - if result['code'] != 200: - click.echo('post_request error') - else: - return result - - def search(self, search_content, search_type, limit=9): - """ - 搜索API - :params search_content: 搜索内容 - :params search_type: 搜索类型 - :params limit: 返回结果数量 - :return: 字典. - """ - - url = 'http://music.163.com/weapi/cloudsearch/get/web?csrf_token=' - params = {'s': search_content, 'type': search_type, 'offset': 0, 'sub': 'false', 'limit': limit} - result = self.post_request(url, params) - return result - - def search_song(self, song_name, song_num, quiet=True, limit=9): - """ - 根据音乐名搜索 - :params song_name: 音乐名 - :params song_num: 下载的歌曲数 - :params quiet: 自动选择匹配最优结果 - :params limit: 返回结果数量 - :return: Song独享 - """ - - result = self.search(song_name, search_type=1, limit=limit) - - if result['result']['songCount'] <= 0: - click.echo('Song {} not existed.'.format(song_name)) - else: - songs = result['result']['songs'] - if quiet: - song_id, song_name = songs[0]['id'], songs[0]['name'] - song = Song(song_id=song_id, song_name=song_name, song_num=song_num) - return song - - def get_song_url(self, song_id, bit_rate=320000): - """ - 获得歌曲的下载地址 - :params song_id: 音乐ID. - :params bit_rate: {'MD 128k': 128000, 'HD 320k': 320000} - :return: 歌曲下载地址 - """ - - url = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token=' - csrf = '' - params = {'ids': [song_id], 'br': bit_rate, 'csrf_token': csrf} - result = self.post_request(url, params) - # 歌曲下载地址 - song_url = result['data'][0]['url'] - - # 歌曲不存在 - if song_url is None: - click.echo('Song {} is not available due to copyright issue.'.format(song_id)) - else: - return song_url - - def get_song_by_url(self, song_url, song_name, song_num, folder): - """ - 下载歌曲到本地 - :params song_url: 歌曲下载地址 - :params song_name: 歌曲名字 - :params song_num: 下载的歌曲数 - :params folder: 保存路径 - """ - if not os.path.exists(folder): - os.makedirs(folder) - fpath = os.path.join(folder, str(song_num) + '_' + song_name + '.mp3') - if sys.platform == 'win32' or sys.platform == 'cygwin': - valid_name = re.sub(r'[<>:"/\\|?*]', '', song_name) - if valid_name != song_name: - click.echo('{} will be saved as: {}.mp3'.format(song_name, valid_name)) - fpath = os.path.join(folder, str(song_num) + '_' + valid_name + '.mp3') - - if not os.path.exists(fpath): - resp = self.download_session.get(song_url, timeout=self.timeout, stream=True) - length = int(resp.headers.get('content-length')) - label = 'Downloading {} {}kb'.format(song_name, int(length/1024)) - - with click.progressbar(length=length, label=label) as progressbar: - with open(fpath, 'wb') as song_file: - for chunk in resp.iter_content(chunk_size=1024): - if chunk: - song_file.write(chunk) - progressbar.update(1024) - - -class Netease(): - """ - 网易云音乐下载 - """ - def __init__(self, timeout, folder, quiet, cookie_path): - self.crawler = Crawler(timeout, cookie_path) - self.folder = '.' if folder is None else folder - self.quiet = quiet - - def download_song_by_search(self, song_name, song_num): - """ - 根据歌曲名进行搜索 - :params song_name: 歌曲名字 - :params song_num: 下载的歌曲数 - """ - - try: - song = self.crawler.search_song(song_name, song_num, self.quiet) - except: - click.echo('download_song_by_serach error') - # 如果找到了音乐, 则下载 - if song != None: - self.download_song_by_id(song.song_id, song.song_name, song.song_num, self.folder) - - def download_song_by_id(self, song_id, song_name, song_num, folder='.'): - """ - 通过歌曲的ID下载 - :params song_id: 歌曲ID - :params song_name: 歌曲名 - :params song_num: 下载的歌曲数 - :params folder: 保存地址 - """ - try: - url = self.crawler.get_song_url(song_id) - # 去掉非法字符 - song_name = song_name.replace('/', '') - song_name = song_name.replace('.', '') - self.crawler.get_song_by_url(url, song_name, song_num, folder) - - except: - click.echo('download_song_by_id error') - - -if __name__ == '__main__': - timeout = 60 - output = 'Musics' - quiet = True - cookie_path = 'Cookie' - netease = Netease(timeout, output, quiet, cookie_path) - music_list_name = 'music_list.txt' - # 如果music列表存在, 那么开始下载 - if os.path.exists(music_list_name): - with open(music_list_name, 'r') as f: - music_list = list(map(lambda x: x.strip(), f.readlines())) - for song_num, song_name in enumerate(music_list): - netease.download_song_by_search(song_name,song_num + 1) - else: +# -*- coding:utf-8 -*- +import requests, hashlib, sys, click, re, base64, binascii, json, os +from Crypto.Cipher import AES +from http import cookiejar + +""" +Website:http://cuijiahua.com +Author:Jack Cui +Refer:https://github.com/darknessomi/musicbox +""" + +class Encrypyed(): + """ + 解密算法 + """ + def __init__(self): + self.modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7' + self.nonce = '0CoJUm6Qyw8W8jud' + self.pub_key = '010001' + + # 登录加密算法, 基于https://github.com/stkevintan/nw_musicbox脚本实现 + def encrypted_request(self, text): + text = json.dumps(text) + sec_key = self.create_secret_key(16) + enc_text = self.aes_encrypt(self.aes_encrypt(text, self.nonce), sec_key.decode('utf-8')) + enc_sec_key = self.rsa_encrpt(sec_key, self.pub_key, self.modulus) + data = {'params': enc_text, 'encSecKey': enc_sec_key} + return data + + def aes_encrypt(self, text, secKey): + pad = 16 - len(text) % 16 + text = text + chr(pad) * pad + encryptor = AES.new(secKey.encode('utf-8'), AES.MODE_CBC, b'0102030405060708') + ciphertext = encryptor.encrypt(text.encode('utf-8')) + ciphertext = base64.b64encode(ciphertext).decode('utf-8') + return ciphertext + + def rsa_encrpt(self, text, pubKey, modulus): + text = text[::-1] + rs = pow(int(binascii.hexlify(text), 16), int(pubKey, 16), int(modulus, 16)) + return format(rs, 'x').zfill(256) + + def create_secret_key(self, size): + return binascii.hexlify(os.urandom(size))[:16] + + +class Song(): + """ + 歌曲对象，用于存储歌曲的信息 + """ + def __init__(self, song_id, song_name, song_num, song_url=None): + self.song_id = song_id + self.song_name = song_name + self.song_num = song_num + self.song_url = '' if song_url is None else song_url + +class Crawler(): + """ + 网易云爬取API + """ + def __init__(self, timeout=60, cookie_path='.'): + self.headers = { + 'Accept': '*/*', + 'Accept-Encoding': 'gzip,deflate,sdch', + 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', + 'Connection': 'keep-alive', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Host': 'music.163.com', + 'Referer': 'http://music.163.com/search/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' + } + self.session = requests.Session() + self.session.headers.update(self.headers) + self.session.cookies = cookiejar.LWPCookieJar(cookie_path) + self.download_session = requests.Session() + self.timeout = timeout + self.ep = Encrypyed() + + def post_request(self, url, params): + """ + Post请求 + :return: 字典 + """ + + data = self.ep.encrypted_request(params) + resp = self.session.post(url, data=data, timeout=self.timeout) + result = resp.json() + if result['code'] != 200: + click.echo('post_request error') + else: + return result + + def search(self, search_content, search_type, limit=9): + """ + 搜索API + :params search_content: 搜索内容 + :params search_type: 搜索类型 + :params limit: 返回结果数量 + :return: 字典. + """ + + url = 'http://music.163.com/weapi/cloudsearch/get/web?csrf_token=' + params = {'s': search_content, 'type': search_type, 'offset': 0, 'sub': 'false', 'limit': limit} + result = self.post_request(url, params) + return result + + def search_song(self, song_name, song_num, quiet=True, limit=9): + """ + 根据音乐名搜索 + :params song_name: 音乐名 + :params song_num: 下载的歌曲数 + :params quiet: 自动选择匹配最优结果 + :params limit: 返回结果数量 + :return: Song独享 + """ + + result = self.search(song_name, search_type=1, limit=limit) + + if result['result']['songCount'] <= 0: + click.echo('Song {} not existed.'.format(song_name)) + else: + songs = result['result']['songs'] + if quiet: + song_id, song_name = songs[0]['id'], songs[0]['name'] + song = Song(song_id=song_id, song_name=song_name, song_num=song_num) + return song + + def get_song_url(self, song_id, bit_rate=320000): + """ + 获得歌曲的下载地址 + :params song_id: 音乐ID. + :params bit_rate: {'MD 128k': 128000, 'HD 320k': 320000} + :return: 歌曲下载地址 + """ + + url = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token=' + csrf = '' + params = {'ids': [song_id], 'br': bit_rate, 'csrf_token': csrf} + result = self.post_request(url, params) + # 歌曲下载地址 + song_url = result['data'][0]['url'] + + # 歌曲不存在 + if song_url is None: + click.echo('Song {} is not available due to copyright issue.'.format(song_id)) + else: + return song_url + + def get_song_by_url(self, song_url, song_name, song_num, folder): + """ + 下载歌曲到本地 + :params song_url: 歌曲下载地址 + :params song_name: 歌曲名字 + :params song_num: 下载的歌曲数 + :params folder: 保存路径 + """ + if not os.path.exists(folder): + os.makedirs(folder) + fpath = os.path.join(folder, str(song_num) + '_' + song_name + '.mp3') + if sys.platform == 'win32' or sys.platform == 'cygwin': + valid_name = re.sub(r'[<>:"/\\|?*]', '', song_name) + if valid_name != song_name: + click.echo('{} will be saved as: {}.mp3'.format(song_name, valid_name)) + fpath = os.path.join(folder, str(song_num) + '_' + valid_name + '.mp3') + + if not os.path.exists(fpath): + resp = self.download_session.get(song_url, timeout=self.timeout, stream=True) + length = int(resp.headers.get('content-length')) + label = 'Downloading {} {}kb'.format(song_name, int(length/1024)) + + with click.progressbar(length=length, label=label) as progressbar: + with open(fpath, 'wb') as song_file: + for chunk in resp.iter_content(chunk_size=1024): + if chunk: + song_file.write(chunk) + progressbar.update(1024) + + +class Netease(): + """ + 网易云音乐下载 + """ + def __init__(self, timeout, folder, quiet, cookie_path): + self.crawler = Crawler(timeout, cookie_path) + self.folder = '.' if folder is None else folder + self.quiet = quiet + + def download_song_by_search(self, song_name, song_num): + """ + 根据歌曲名进行搜索 + :params song_name: 歌曲名字 + :params song_num: 下载的歌曲数 + """ + + try: + song = self.crawler.search_song(song_name, song_num, self.quiet) + except: + click.echo('download_song_by_serach error') + # 如果找到了音乐, 则下载 + if song != None: + self.download_song_by_id(song.song_id, song.song_name, song.song_num, self.folder) + + def download_song_by_id(self, song_id, song_name, song_num, folder='.'): + """ + 通过歌曲的ID下载 + :params song_id: 歌曲ID + :params song_name: 歌曲名 + :params song_num: 下载的歌曲数 + :params folder: 保存地址 + """ + try: + url = self.crawler.get_song_url(song_id) + # 去掉非法字符 + song_name = song_name.replace('/', '') + song_name = song_name.replace('.', '') + self.crawler.get_song_by_url(url, song_name, song_num, folder) + + except: + click.echo('download_song_by_id error') + + +if __name__ == '__main__': + timeout = 60 + output = 'Musics' + quiet = True + cookie_path = 'Cookie' + netease = Netease(timeout, output, quiet, cookie_path) + music_list_name = 'music_list.txt' + # 如果music列表存在, 那么开始下载 + if os.path.exists(music_list_name): + with open(music_list_name, 'r') as f: + music_list = list(map(lambda x: x.strip(), f.readlines())) + for song_num, song_name in enumerate(music_list): + netease.download_song_by_search(song_name,song_num + 1) + else: click.echo('music_list.txt not exist.') \ No newline at end of file diff --git a/Netease/music_list.txt b/net_ease/music_list.txt similarity index 100% rename from Netease/music_list.txt rename to net_ease/music_list.txt From 2268fbe455a1fcddb942c6d65dd1cce4eecbb66e Mon Sep 17 00:00:00 2001 From: huangweiqi Date: Tue, 12 Jun 2018 16:34:53 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0abc=E7=9A=84readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abc/README.md | 10 ++++++++++ abc/__init__.py | 0 2 files changed, 10 insertions(+) create mode 100644 abc/README.md create mode 100644 abc/__init__.py diff --git a/abc/README.md b/abc/README.md new file mode 100644 index 00000000..04282bfe --- /dev/null +++ b/abc/README.md @@ -0,0 +1,10 @@ + +## Python爬虫入门 +1、[]() +2、[]() + +## 参考文献 +1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider) +2、[Python3 爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) +3、 + diff --git a/abc/__init__.py b/abc/__init__.py new file mode 100644 index 00000000..e69de29b From a11b9137ff6139139c46a1c2faf44b19397445fa Mon Sep 17 00:00:00 2001 From: huangweiqi Date: Tue, 12 Jun 2018 16:42:35 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0spiders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abc/SimpleCrawler.py | 39 +++++++++ abc/biying_spider.py | 20 +++++ abc/downloader.py | 65 ++++++++++++++ abc/dragon_ball.py | 107 +++++++++++++++++++++++ abc/get_photo.py | 126 +++++++++++++++++++++++++++ abc/lagou_spider.py | 42 +++++++++ video_downloader/video_downloader.py | 1 - 7 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 abc/SimpleCrawler.py create mode 100644 abc/biying_spider.py create mode 100644 abc/downloader.py create mode 100644 abc/dragon_ball.py create mode 100644 abc/get_photo.py create mode 100644 abc/lagou_spider.py diff --git a/abc/SimpleCrawler.py b/abc/SimpleCrawler.py new file mode 100644 index 00000000..99bb193a --- /dev/null +++ b/abc/SimpleCrawler.py @@ -0,0 +1,39 @@ +# -*- coding: UTF-8 -*- + +# Filename : block_chain.py +# author by : WeiQi + +import json + +class SimpleCrawler: + init_url = "https://zhuanlan.zhihu.com/api/columns/SVlaw/followers" + offset = 0 + + def crawl(self, params=None): + import requests + # 必须指定UA，否则知乎服务器会判定请求不合法 + headers = { + "Host": "zhuanlan.zhihu.com", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + } + response = requests.get(self.init_url, headers=headers, params=params) + print(response.url) + data = response.json() + # 分页加载更多，递归调用 这里为了演示只获取前100条数据 + while self.offset < 100: + self.parse(data) + self.offset += 20 + params = {"limit": 20, "offset": self.offset} + self.crawl(params) + + def parse(self, data): + # 以json格式存储到文件 + with open("followers.json", "a", encoding="utf-8") as f: + for item in data: + f.write(json.dumps(item)) + f.write('\n') + + +if __name__ == '__main__': + SimpleCrawler().crawl() \ No newline at end of file diff --git a/abc/biying_spider.py b/abc/biying_spider.py new file mode 100644 index 00000000..83ad9c1e --- /dev/null +++ b/abc/biying_spider.py @@ -0,0 +1,20 @@ +# -*- coding: UTF-8 -*- + +# Filename : biying_spider.py +# author by : WeiQi + +import requests +import re +import time + +local = time.strftime("%Y.%m.%d") +url = 'https://cn.bing.com/' +con = requests.get(url) +content = con.text +reg = r"(http://s.cn.bing.net/az/hprichbg/rb/.*?.jpg)" +a = re.findall(reg, content, re.S)[0] +print(a) +read = requests.get(a) +f = open('%s.jpg' % local, 'wb') +f.write(read.content) +f.close() \ No newline at end of file diff --git a/abc/downloader.py b/abc/downloader.py new file mode 100644 index 00000000..be883464 --- /dev/null +++ b/abc/downloader.py @@ -0,0 +1,65 @@ +# -*- coding: UTF-8 -*- + +# Filename : downloader.py +# author by : WeiQi + +import requests +from contextlib import closing + +class ProgressBar(object): + def __init__(self, title, count=0.0, run_status=None, fin_status=None, total=100.0, unit='', sep='/', + chunk_size=1.0): + super(ProgressBar, self).__init__() + self.info = "[%s] %s %.2f %s %s %.2f %s" + self.title = title + self.total = total + self.count = count + self.chunk_size = chunk_size + self.status = run_status or "" + self.fin_status = fin_status or " " * len(self.status) + self.unit = unit + self.seq = sep + + def __get_info(self): + # [名称] 状态 进度 单位 分割线 总数 单位 + _info = self.info % ( + self.title, self.status, self.count / self.chunk_size, self.unit, self.seq, self.total / self.chunk_size, + self.unit) + return _info + + def refresh(self, count=1, status=None): + self.count += count + self.status = status or self.status + end_str = "\r" + if self.count >= self.total: + end_str = '\n' + self.status = status or self.fin_status + print(self.__get_info(), end=end_str, ) + + +if __name__ == '__main__': + url = 'http://www.demongan.com/source/game/二十四点.zip' + filename = '二十四点.zip' + print('*' * 100) + print('\t\t\t\t欢迎使用文件下载小助手') + print('*' * 100) + url = input('请输入需要下载的文件链接:\n') + filename = url.split('/')[-1] + with closing(requests.get(url, stream=True)) as response: + chunk_size = 1024 + content_size = int(response.headers['content-length']) + if response.status_code == 200: + print('文件大小:%0.2f KB' % (content_size / chunk_size)) + progress = ProgressBar("%s下载进度" % filename + , total=content_size + , unit="KB" + , chunk_size=chunk_size + , run_status="正在下载" + , fin_status="下载完成") + + with open(filename, "wb") as file: + for data in response.iter_content(chunk_size=chunk_size): + file.write(data) + progress.refresh(count=len(data)) + else: + print('链接异常') \ No newline at end of file diff --git a/abc/dragon_ball.py b/abc/dragon_ball.py new file mode 100644 index 00000000..c3a99148 --- /dev/null +++ b/abc/dragon_ball.py @@ -0,0 +1,107 @@ +# -*- coding: UTF-8 -*- + +# Filename : dragon_ball.py +# author by : WeiQi + +from urllib.parse import urljoin +import urllib.request + +from bs4 import BeautifulSoup + +import os +import datetime +import re +import errno + +def mkdir_p(path): # 递归创建多级目录 + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5) + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def get_link(page): # 寻找链接的href + linkData = [] + for page in page.find_all('td'): + links = page.select("a") + for each in links: + # if str(each.get('href'))[:1] == '/': 过滤if代码 + data = each.get('href') + linkData.append(data) + return (linkData) + +def gain(url): # 获取网页指定内容 + page = urllib.request.urlopen(url).read() + soup = BeautifulSoup(page, 'lxml') # 利用soup获取网页内容 + links = get_link(soup) # 获取末页') + szresult = re.findall(szurlre, html) + if len(szresult) == 0: + page_num = 0 + else: + page_num = int(szresult[0]) + print (page_num) + return page_num + +def get_page_num2(html): + html = html.decode('gbk') + szurlre = re.compile(r'共(\d+)页') + szresult = re.findall(szurlre, html) + if len(szresult) == 0: + page_num = 0 + else: + page_num = int(szresult[0]) + return page_num + +#获得单页的相册 +def get_ablum_list(html): + html = html.decode('gbk') + szurlre = re.compile(r'(http://www.5442.com/meinv/2\d+/\d+.html)" target=') + ablum_list = re.findall(szurlre, html); + return ablum_list + +#获得相册的名称 +def get_ablum_name(html): + szurlre = re.compile(r'(\S+)') + ablum_name = re.findall(szurlre, html) + return ablum_name[0] + +#获得单页的图片 +def get_photo(html, dir, photo_num): + html = html.decode('gbk') + imgre = re.compile(r'点击图片进入下一页\' >) + imglist = re.findall(imgre, html) + for imgurl in imglist: + try: + socket.setdefaulttimeout(2) + urllib.urlretrieve(imgurl, Date: Tue, 12 Jun 2018 17:04:57 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abc/README.md | 9 +++++---- abc/{SimpleCrawler.py => simple_crawler.py} | 0 2 files changed, 5 insertions(+), 4 deletions(-) rename abc/{SimpleCrawler.py => simple_crawler.py} (100%) diff --git a/abc/README.md b/abc/README.md index 04282bfe..7c295bd5 100644 --- a/abc/README.md +++ b/abc/README.md @@ -1,10 +1,11 @@ ## Python爬虫入门 -1、[]() -2、[]() +1、[]() ## 参考文献 -1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider) +1、[Python爬虫原理](https://www.cnblogs.com/sss4/p/7809821.html) 2、[Python3 爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) 3、 - + +## 参考项目 +1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider) diff --git a/abc/SimpleCrawler.py b/abc/simple_crawler.py similarity index 100% rename from abc/SimpleCrawler.py rename to abc/simple_crawler.py From c74e3520dc7ccbbb048c50f2342bda8f8fbc25f1 Mon Sep 17 00:00:00 2001 From: huangweiqi Date: Tue, 12 Jun 2018 17:27:33 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abc/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/abc/README.md b/abc/README.md index 7c295bd5..9c8697a5 100644 --- a/abc/README.md +++ b/abc/README.md @@ -1,11 +1,16 @@ ## Python爬虫入门 -1、[]() - +1、你需要学习的是怎么样做一个比较大的东西， +2、在这个过程中，你会很快地学会需要学会的东西。 +3、不要问怎么入门，直接上路就好了。 + ## 参考文献 1、[Python爬虫原理](https://www.cnblogs.com/sss4/p/7809821.html) -2、[Python3 爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) +2、[Python3爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) 3、 ## 参考项目 1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider) + +## TODO +[Python爬虫(1～12)](http://www.cnblogs.com/miqi1992/p/7967367.html) From c291f6c21135647ad9c6f6b8a0ffd4369aa6d0b7 Mon Sep 17 00:00:00 2001 From: huangweiqi Date: Tue, 12 Jun 2018 17:30:54 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abc/README.md b/abc/README.md index 9c8697a5..7611d425 100644 --- a/abc/README.md +++ b/abc/README.md @@ -7,7 +7,7 @@ ## 参考文献 1、[Python爬虫原理](https://www.cnblogs.com/sss4/p/7809821.html) 2、[Python3爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) -3、 +3、[Python集群爬下整个豆瓣的经验](https://www.zhihu.com/question/20899988) ## 参考项目 1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider)