diff --git a/abc/README.md b/abc/README.md new file mode 100644 index 00000000..7611d425 --- /dev/null +++ b/abc/README.md @@ -0,0 +1,16 @@ + +## Python爬虫入门 +1、你需要学习的是怎么样做一个比较大的东西, +2、在这个过程中,你会很快地学会需要学会的东西。 +3、不要问怎么入门,直接上路就好了。 + +## 参考文献 +1、[Python爬虫原理](https://www.cnblogs.com/sss4/p/7809821.html) +2、[Python3爬虫的基本原理](https://blog.csdn.net/yulianlin/article/details/79453499) +3、[Python集群爬下整个豆瓣的经验](https://www.zhihu.com/question/20899988) + +## 参考项目 +1、[爬虫基础项目](https://github.com/Jack-Cherish/python-spider) + +## TODO +[Python爬虫(1~12)](http://www.cnblogs.com/miqi1992/p/7967367.html) diff --git a/abc/__init__.py b/abc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/abc/biying_spider.py b/abc/biying_spider.py new file mode 100644 index 00000000..83ad9c1e --- /dev/null +++ b/abc/biying_spider.py @@ -0,0 +1,20 @@ +# -*- coding: UTF-8 -*- + +# Filename : biying_spider.py +# author by : WeiQi + +import requests +import re +import time + +local = time.strftime("%Y.%m.%d") +url = 'https://cn.bing.com/' +con = requests.get(url) +content = con.text +reg = r"(http://s.cn.bing.net/az/hprichbg/rb/.*?.jpg)" +a = re.findall(reg, content, re.S)[0] +print(a) +read = requests.get(a) +f = open('%s.jpg' % local, 'wb') +f.write(read.content) +f.close() \ No newline at end of file diff --git a/abc/downloader.py b/abc/downloader.py new file mode 100644 index 00000000..be883464 --- /dev/null +++ b/abc/downloader.py @@ -0,0 +1,65 @@ +# -*- coding: UTF-8 -*- + +# Filename : downloader.py +# author by : WeiQi + +import requests +from contextlib import closing + +class ProgressBar(object): + def __init__(self, title, count=0.0, run_status=None, fin_status=None, total=100.0, unit='', sep='/', + chunk_size=1.0): + super(ProgressBar, self).__init__() + self.info = "[%s] %s %.2f %s %s %.2f %s" + self.title = title + self.total = total + self.count = count + self.chunk_size = chunk_size + self.status = run_status or "" + self.fin_status = fin_status or " " * len(self.status) + self.unit = unit + self.seq = sep + + def __get_info(self): + # [名称] 状态 进度 单位 分割线 总数 单位 + _info = self.info % ( + self.title, self.status, self.count / self.chunk_size, self.unit, self.seq, self.total / self.chunk_size, + self.unit) + return _info + + def refresh(self, count=1, status=None): + self.count += count + self.status = status or self.status + end_str = "\r" + if self.count >= self.total: + end_str = '\n' + self.status = status or self.fin_status + print(self.__get_info(), end=end_str, ) + + +if __name__ == '__main__': + url = 'http://www.demongan.com/source/game/二十四点.zip' + filename = '二十四点.zip' + print('*' * 100) + print('\t\t\t\t欢迎使用文件下载小助手') + print('*' * 100) + url = input('请输入需要下载的文件链接:\n') + filename = url.split('/')[-1] + with closing(requests.get(url, stream=True)) as response: + chunk_size = 1024 + content_size = int(response.headers['content-length']) + if response.status_code == 200: + print('文件大小:%0.2f KB' % (content_size / chunk_size)) + progress = ProgressBar("%s下载进度" % filename + , total=content_size + , unit="KB" + , chunk_size=chunk_size + , run_status="正在下载" + , fin_status="下载完成") + + with open(filename, "wb") as file: + for data in response.iter_content(chunk_size=chunk_size): + file.write(data) + progress.refresh(count=len(data)) + else: + print('链接异常') \ No newline at end of file diff --git a/abc/dragon_ball.py b/abc/dragon_ball.py new file mode 100644 index 00000000..c3a99148 --- /dev/null +++ b/abc/dragon_ball.py @@ -0,0 +1,107 @@ +# -*- coding: UTF-8 -*- + +# Filename : dragon_ball.py +# author by : WeiQi + +from urllib.parse import urljoin +import urllib.request + +from bs4 import BeautifulSoup + +import os +import datetime +import re +import errno + +def mkdir_p(path): # 递归创建多级目录 + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5) + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def get_link(page): # 寻找链接的href + linkData = [] + for page in page.find_all('td'): + links = page.select("a") + for each in links: + # if str(each.get('href'))[:1] == '/': 过滤if代码 + data = each.get('href') + linkData.append(data) + return (linkData) + +def gain(url): # 获取网页指定内容 + page = urllib.request.urlopen(url).read() + soup = BeautifulSoup(page, 'lxml') # 利用soup获取网页内容 + links = get_link(soup) # 获取末页') + szresult = re.findall(szurlre, html) + if len(szresult) == 0: + page_num = 0 + else: + page_num = int(szresult[0]) + print (page_num) + return page_num + +def get_page_num2(html): + html = html.decode('gbk') + szurlre = re.compile(r'共(\d+)页') + szresult = re.findall(szurlre, html) + if len(szresult) == 0: + page_num = 0 + else: + page_num = int(szresult[0]) + return page_num + +#获得单页的相册 +def get_ablum_list(html): + html = html.decode('gbk') + szurlre = re.compile(r'(http://www.5442.com/meinv/2\d+/\d+.html)" target=') + ablum_list = re.findall(szurlre, html); + return ablum_list + +#获得相册的名称 +def get_ablum_name(html): + szurlre = re.compile(r'(\S+)') + ablum_name = re.findall(szurlre, html) + return ablum_name[0] + +#获得单页的图片 +def get_photo(html, dir, photo_num): + html = html.decode('gbk') + imgre = re.compile(r'点击图片进入下一页\' >)
+    imglist = re.findall(imgre, html)
+    for imgurl in imglist:
+        try:
+            socket.setdefaulttimeout(2)
+            urllib.urlretrieve(imgurl, . - :params bit_rate: {'MD 128k': 128000, 'HD 320k': 320000} - :return: 歌曲下载地址 - """ - - url = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token=' - csrf = '' - params = {'ids': [song_id], 'br': bit_rate, 'csrf_token': csrf} - result = self.post_request(url, params) - # 歌曲下载地址 - song_url = result['data'][0]['url'] - - # 歌曲不存在 - if song_url is None: - click.echo('Song {} is not available due to copyright issue.'.format(song_id)) - else: - return song_url - - def get_song_by_url(self, song_url, song_name, song_num, folder): - """ - 下载歌曲到本地 - :params song_url: 歌曲下载地址 - :params song_name: 歌曲名字 - :params song_num: 下载的歌曲数 - :params folder: 保存路径 - """ - if not os.path.exists(folder): - os.makedirs(folder) - fpath = os.path.join(folder, str(song_num) + '_' + song_name + '.mp3') - if sys.platform == 'win32' or sys.platform == 'cygwin': - valid_name = re.sub(r'[<>:"/\\|?*]', '', song_name) - if valid_name != song_name: - click.echo('{} will be saved as: {}.mp3'.format(song_name, valid_name)) - fpath = os.path.join(folder, str(song_num) + '_' + valid_name + '.mp3') - - if not os.path.exists(fpath): - resp = self.download_session.get(song_url, timeout=self.timeout, stream=True) - length = int(resp.headers.get('content-length')) - label = 'Downloading {} {}kb'.format(song_name, int(length/1024)) - - with click.progressbar(length=length, label=label) as progressbar: - with open(fpath, 'wb') as song_file: - for chunk in resp.iter_content(chunk_size=1024): - if chunk: - song_file.write(chunk) - progressbar.update(1024) - - -class Netease(): - """ - 网易云音乐下载 - """ - def __init__(self, timeout, folder, quiet, cookie_path): - self.crawler = Crawler(timeout, cookie_path) - self.folder = '.' if folder is None else folder - self.quiet = quiet - - def download_song_by_search(self, song_name, song_num): - """ - 根据歌曲名进行搜索 - :params song_name: 歌曲名字 - :params song_num: 下载的歌曲数 - """ - - try: - song = self.crawler.search_song(song_name, song_num, self.quiet) - except: - click.echo('download_song_by_serach error') - # 如果找到了音乐, 则下载 - if song != None: - self.download_song_by_id(song.song_id, song.song_name, song.song_num, self.folder) - - def download_song_by_id(self, song_id, song_name, song_num, folder='.'): - """ - 通过歌曲的ID下载 - :params song_id: 歌曲ID - :params song_name: 歌曲名 - :params song_num: 下载的歌曲数 - :params folder: 保存地址 - """ - try: - url = self.crawler.get_song_url(song_id) - # 去掉非法字符 - song_name = song_name.replace('/', '') - song_name = song_name.replace('.', '') - self.crawler.get_song_by_url(url, song_name, song_num, folder) - - except: - click.echo('download_song_by_id error') - - -if __name__ == '__main__': - timeout = 60 - output = 'Musics' - quiet = True - cookie_path = 'Cookie' - netease = Netease(timeout, output, quiet, cookie_path) - music_list_name = 'music_list.txt' - # 如果music列表存在, 那么开始下载 - if os.path.exists(music_list_name): - with open(music_list_name, 'r') as f: - music_list = list(map(lambda x: x.strip(), f.readlines())) - for song_num, song_name in enumerate(music_list): - netease.download_song_by_search(song_name,song_num + 1) - else: +# -*- coding:utf-8 -*- +import requests, hashlib, sys, click, re, base64, binascii, json, os +from Crypto.Cipher import AES +from http import cookiejar + +""" +Website:http://cuijiahua.com +Author:Jack Cui +Refer:https://github.com/darknessomi/musicbox +""" + +class Encrypyed(): + """ + 解密算法 + """ + def __init__(self): + self.modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7' + self.nonce = '0CoJUm6Qyw8W8jud' + self.pub_key = '010001' + + # 登录加密算法, 基于https://github.com/stkevintan/nw_musicbox脚本实现 + def encrypted_request(self, text): + text = json.dumps(text) + sec_key = self.create_secret_key(16) + enc_text = self.aes_encrypt(self.aes_encrypt(text, self.nonce), sec_key.decode('utf-8')) + enc_sec_key = self.rsa_encrpt(sec_key, self.pub_key, self.modulus) + data = {'params': enc_text, 'encSecKey': enc_sec_key} + return data + + def aes_encrypt(self, text, secKey): + pad = 16 - len(text) % 16 + text = text + chr(pad) * pad + encryptor = AES.new(secKey.encode('utf-8'), AES.MODE_CBC, b'0102030405060708') + ciphertext = encryptor.encrypt(text.encode('utf-8')) + ciphertext = base64.b64encode(ciphertext).decode('utf-8') + return ciphertext + + def rsa_encrpt(self, text, pubKey, modulus): + text = text[::-1] + rs = pow(int(binascii.hexlify(text), 16), int(pubKey, 16), int(modulus, 16)) + return format(rs, 'x').zfill(256) + + def create_secret_key(self, size): + return binascii.hexlify(os.urandom(size))[:16] + + +class Song(): + """ + 歌曲对象,用于存储歌曲的信息 + """ + def __init__(self, song_id, song_name, song_num, song_url=None): + self.song_id = song_id + self.song_name = song_name + self.song_num = song_num + self.song_url = '' if song_url is None else song_url + +class Crawler(): + """ + 网易云爬取API + """ + def __init__(self, timeout=60, cookie_path='.'): + self.headers = { + 'Accept': '*/*', + 'Accept-Encoding': 'gzip,deflate,sdch', + 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', + 'Connection': 'keep-alive', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Host': 'music.163.com', + 'Referer': 'http://music.163.com/search/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' + } + self.session = requests.Session() + self.session.headers.update(self.headers) + self.session.cookies = cookiejar.LWPCookieJar(cookie_path) + self.download_session = requests.Session() + self.timeout = timeout + self.ep = Encrypyed() + + def post_request(self, url, params): + """ + Post请求 + :return: 字典 + """ + + data = self.ep.encrypted_request(params) + resp = self.session.post(url, data=data, timeout=self.timeout) + result = resp.json() + if result['code'] != 200: + click.echo('post_request error') + else: + return result + + def search(self, search_content, search_type, limit=9): + """ + 搜索API + :params search_content: 搜索内容 + :params search_type: 搜索类型 + :params limit: 返回结果数量 + :return: 字典. + """ + + url = 'http://music.163.com/weapi/cloudsearch/get/web?csrf_token=' + params = {'s': search_content, 'type': search_type, 'offset': 0, 'sub': 'false', 'limit': limit} + result = self.post_request(url, params) + return result + + def search_song(self, song_name, song_num, quiet=True, limit=9): + """ + 根据音乐名搜索 + :params song_name: 音乐名 + :params song_num: 下载的歌曲数 + :params quiet: 自动选择匹配最优结果 + :params limit: 返回结果数量 + :return: Song独享 + """ + + result = self.search(song_name, search_type=1, limit=limit) + + if result['result']['songCount'] <= 0: + click.echo('Song {} not existed.'.format(song_name)) + else: + songs = result['result']['songs'] + if quiet: + song_id, song_name = songs[0]['id'], songs[0]['name'] + song = Song(song_id=song_id, song_name=song_name, song_num=song_num) + return song + + def get_song_url(self, song_id, bit_rate=320000): + """ + 获得歌曲的下载地址 + :params song_id: 音乐ID. + :params bit_rate: {'MD 128k': 128000, 'HD 320k': 320000} + :return: 歌曲下载地址 + """ + + url = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token=' + csrf = '' + params = {'ids': [song_id], 'br': bit_rate, 'csrf_token': csrf} + result = self.post_request(url, params) + # 歌曲下载地址 + song_url = result['data'][0]['url'] + + # 歌曲不存在 + if song_url is None: + click.echo('Song {} is not available due to copyright issue.'.format(song_id)) + else: + return song_url + + def get_song_by_url(self, song_url, song_name, song_num, folder): + """ + 下载歌曲到本地 + :params song_url: 歌曲下载地址 + :params song_name: 歌曲名字 + :params song_num: 下载的歌曲数 + :params folder: 保存路径 + """ + if not os.path.exists(folder): + os.makedirs(folder) + fpath = os.path.join(folder, str(song_num) + '_' + song_name + '.mp3') + if sys.platform == 'win32' or sys.platform == 'cygwin': + valid_name = re.sub(r'[<>:"/\\|?*]', '', song_name) + if valid_name != song_name: + click.echo('{} will be saved as: {}.mp3'.format(song_name, valid_name)) + fpath = os.path.join(folder, str(song_num) + '_' + valid_name + '.mp3') + + if not os.path.exists(fpath): + resp = self.download_session.get(song_url, timeout=self.timeout, stream=True) + length = int(resp.headers.get('content-length')) + label = 'Downloading {} {}kb'.format(song_name, int(length/1024)) + + with click.progressbar(length=length, label=label) as progressbar: + with open(fpath, 'wb') as song_file: + for chunk in resp.iter_content(chunk_size=1024): + if chunk: + song_file.write(chunk) + progressbar.update(1024) + + +class Netease(): + """ + 网易云音乐下载 + """ + def __init__(self, timeout, folder, quiet, cookie_path): + self.crawler = Crawler(timeout, cookie_path) + self.folder = '.' if folder is None else folder + self.quiet = quiet + + def download_song_by_search(self, song_name, song_num): + """ + 根据歌曲名进行搜索 + :params song_name: 歌曲名字 + :params song_num: 下载的歌曲数 + """ + + try: + song = self.crawler.search_song(song_name, song_num, self.quiet) + except: + click.echo('download_song_by_serach error') + # 如果找到了音乐, 则下载 + if song != None: + self.download_song_by_id(song.song_id, song.song_name, song.song_num, self.folder) + + def download_song_by_id(self, song_id, song_name, song_num, folder='.'): + """ + 通过歌曲的ID下载 + :params song_id: 歌曲ID + :params song_name: 歌曲名 + :params song_num: 下载的歌曲数 + :params folder: 保存地址 + """ + try: + url = self.crawler.get_song_url(song_id) + # 去掉非法字符 + song_name = song_name.replace('/', '') + song_name = song_name.replace('.', '') + self.crawler.get_song_by_url(url, song_name, song_num, folder) + + except: + click.echo('download_song_by_id error') + + +if __name__ == '__main__': + timeout = 60 + output = 'Musics' + quiet = True + cookie_path = 'Cookie' + netease = Netease(timeout, output, quiet, cookie_path) + music_list_name = 'music_list.txt' + # 如果music列表存在, 那么开始下载 + if os.path.exists(music_list_name): + with open(music_list_name, 'r') as f: + music_list = list(map(lambda x: x.strip(), f.readlines())) + for song_num, song_name in enumerate(music_list): + netease.download_song_by_search(song_name,song_num + 1) + else: click.echo('music_list.txt not exist.') \ No newline at end of file diff --git a/Netease/music_list.txt b/net_ease/music_list.txt similarity index 100% rename from Netease/music_list.txt rename to net_ease/music_list.txt diff --git a/video_downloader/video_downloader.py b/video_downloader/video_downloader.py index 9a4e63c1..61952a24 100644 --- a/video_downloader/video_downloader.py +++ b/video_downloader/video_downloader.py @@ -14,7 +14,6 @@ import requests import time - """ 类说明:爱奇艺、优酷等实现在线观看以及视频下载的类