Archived
重写下载器
This commit is contained in:
MarkusJoe
3 files changed
+59
-192
No files matched your search
+51
-117
@@ -7,136 +7,70 @@
|
||||
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import sys
|
||||
import requests
|
||||
import threading
|
||||
from requests import get
|
||||
from threading import Lock
|
||||
from bin.utils.logger import logger
|
||||
from bin.utils.settings import Settings
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
|
||||
lock = Lock()
|
||||
conf = Settings()
|
||||
|
||||
|
||||
class MulThreadDownload(threading.Thread):
|
||||
def __init__(self, url, startpos, endpos, f, name):
|
||||
super(MulThreadDownload, self).__init__()
|
||||
self.session = requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.url = url # 资源Url
|
||||
self.startpos = startpos
|
||||
self.endpos = endpos
|
||||
self.fd = f # 文件操作
|
||||
self.name = name # 线程名称
|
||||
|
||||
def download(self):
|
||||
class Downloader:
|
||||
def __init__(self, url, nums, file):
|
||||
"""
|
||||
多线程下载
|
||||
初始化
|
||||
:param url:
|
||||
:param nums:
|
||||
:param file:
|
||||
"""
|
||||
self.url = url
|
||||
self.num = nums
|
||||
self.name = file
|
||||
r = get(self.url)
|
||||
self.size = int(r.headers['Content-Length'])
|
||||
logger.info('文件大小为：{} Mb'.format(round(self.size / 1024 / 1024, 2)))
|
||||
|
||||
def down(self, start, end):
|
||||
"""
|
||||
下载
|
||||
:param start:
|
||||
:param end:
|
||||
:return:
|
||||
"""
|
||||
logger.info(f'线程: Thread-{self.name} 开始下载')
|
||||
headers = {"Range": "bytes=%s-%s" % (self.startpos, self.endpos)}
|
||||
res = self.session.get(self.url, headers=headers)
|
||||
self.fd.seek(self.startpos)
|
||||
self.fd.write(res.content)
|
||||
logger.info(f'线程: Thread-{self.name} 结束下载')
|
||||
headers = {'Range': 'bytes={}-{}'.format(start, end)}
|
||||
r = get(self.url, headers=headers, stream=True)
|
||||
lock.acquire()
|
||||
with open(self.name, "rb+") as fp:
|
||||
fp.seek(start)
|
||||
fp.write(r.content)
|
||||
lock.release()
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
此处启动
|
||||
运行
|
||||
:return:
|
||||
"""
|
||||
self.download()
|
||||
|
||||
|
||||
class Check:
|
||||
"""检查md5是否相同和下载数据库"""
|
||||
|
||||
def __init__(self):
|
||||
self.db_name = 'data.db'
|
||||
self.assets_url = 'https://themedatabases.vercel.app/assets'
|
||||
self.remote_md5 = 'https://themedatabases.vercel.app/md5'
|
||||
self.session = requests.Session()
|
||||
self.session.trust_env = False
|
||||
|
||||
def check_md5(self):
|
||||
"""
|
||||
检验本地文件md5是否和远程md5相同
|
||||
:return:
|
||||
"""
|
||||
with open(f'./bin/db/{self.db_name}', 'rb') as fp:
|
||||
data = fp.read()
|
||||
local_md5 = hashlib.md5(data).hexdigest()
|
||||
remote_md5 = self.session.get(self.remote_md5).json()['data'][0]
|
||||
logger.info(f'本地数据库md5: {local_md5}')
|
||||
logger.info(f'远程数据库md5: {remote_md5}')
|
||||
if local_md5 != remote_md5:
|
||||
logger.error('下载错误: 本地数据库md5和远程数据库md5检验不通过, 即将开始重新下载\nI: 本次下载将使用单线程下载')
|
||||
self.single_download()
|
||||
else:
|
||||
logger.info('md5检验已通过')
|
||||
|
||||
def download(self):
|
||||
"""
|
||||
开始下载
|
||||
:return:
|
||||
"""
|
||||
filesize = int(self.session.get(self.assets_url).headers['Content-Length'])
|
||||
threaded_count = 3
|
||||
logger.info(f'数据库大小: {round(filesize / 1024 / 1024, 2)}Mb. 下载线程: {threaded_count}')
|
||||
threading.BoundedSemaphore(threaded_count)
|
||||
step = filesize // threaded_count
|
||||
mtd_list = []
|
||||
start = 0
|
||||
end = -1
|
||||
with open(f'./bin/db/{self.db_name}', 'w') as initial_file:
|
||||
initial_file.close()
|
||||
with open(f'./bin/db/{self.db_name}', 'rb+') as f:
|
||||
name = 1
|
||||
fileno = f.fileno()
|
||||
while end < filesize - 1:
|
||||
start = end + 1
|
||||
end = start + step - 1
|
||||
if end > filesize:
|
||||
end = filesize
|
||||
dup = os.dup(fileno)
|
||||
fd = os.fdopen(dup, 'rb+', -1)
|
||||
t = MulThreadDownload(self.assets_url, start, end, fd, name)
|
||||
name += 1
|
||||
t.start()
|
||||
mtd_list.append(t)
|
||||
|
||||
for i in mtd_list:
|
||||
i.join()
|
||||
|
||||
self.check_md5()
|
||||
|
||||
def single_download(self):
|
||||
"""
|
||||
单线程进行下载
|
||||
:return:
|
||||
"""
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
logger.info(f'正在使用单线程下载中')
|
||||
resp = session.get(self.assets_url)
|
||||
with open(f'./bin/db/{self.db_name}', 'wb') as fp:
|
||||
fp.write(resp.content)
|
||||
logger.info('下载完成 正在检验文件md5')
|
||||
with open(f'./bin/db/{self.db_name}', 'rb') as fp:
|
||||
data = fp.read()
|
||||
local_md5 = hashlib.md5(data).hexdigest()
|
||||
remote_md5 = session.get('https://themedatabases.vercel.app/md5').json()['data'][0]
|
||||
logger.info(f'本地数据库md5: {local_md5}')
|
||||
logger.info(f'远程数据库md5: {remote_md5}')
|
||||
if local_md5 != remote_md5:
|
||||
logger.error('md5检验未通过请手动前往 https://themedatabases.vercel.app/assets 下载文件并放入./bin/db文件夹内')
|
||||
sys.exit(-1)
|
||||
else:
|
||||
logger.info('md5检验已通过')
|
||||
fp = open(self.name, "wb")
|
||||
fp.truncate(self.size)
|
||||
fp.close()
|
||||
part = self.size // self.num
|
||||
pool = ThreadPoolExecutor(max_workers=self.num)
|
||||
futures = []
|
||||
for i in range(self.num):
|
||||
start = part * i
|
||||
if i == self.num - 1:
|
||||
end = self.size
|
||||
else:
|
||||
end = start + part - 1
|
||||
futures.append(pool.submit(self.down, start, end))
|
||||
wait(futures)
|
||||
logger.info('数据库: %s 下载完成' % self.name.split('/')[-1])
|
||||
|
||||
|
||||
if __name__ != '__main__':
|
||||
conf = Settings()
|
||||
if not os.path.exists('./bin/log'):
|
||||
os.mkdir('./bin/log')
|
||||
if not os.path.exists(f'./bin/db/data.db'):
|
||||
logger.error('没有检测到本地主题数据库即将开始下载')
|
||||
Check().download()
|
||||
if not os.path.exists('./bin/db/data.db'):
|
||||
logger.error('没有检测到数据库文件, 即将开始下载data.db')
|
||||
Downloader('https://themedatabase.vercel.app/assets', 4, './bin/db/data.db').run()
|
||||
Reference in New Issue
Block a user