diff --git a/bin/__init__.py b/bin/__init__.py index e743e69..48711ed 100644 --- a/bin/__init__.py +++ b/bin/__init__.py @@ -7,136 +7,70 @@ import os -import hashlib import sys -import requests -import threading +from requests import get +from threading import Lock from bin.utils.logger import logger from bin.utils.settings import Settings +from concurrent.futures import ThreadPoolExecutor, wait + +lock = Lock() +conf = Settings() -class MulThreadDownload(threading.Thread): - def __init__(self, url, startpos, endpos, f, name): - super(MulThreadDownload, self).__init__() - self.session = requests.Session() - self.session.trust_env = False - self.url = url # 资源Url - self.startpos = startpos - self.endpos = endpos - self.fd = f # 文件操作 - self.name = name # 线程名称 - - def download(self): +class Downloader: + def __init__(self, url, nums, file): """ - 多线程下载 + 初始化 + :param url: + :param nums: + :param file: + """ + self.url = url + self.num = nums + self.name = file + r = get(self.url) + self.size = int(r.headers['Content-Length']) + logger.info('文件大小为：{} Mb'.format(round(self.size / 1024 / 1024, 2))) + + def down(self, start, end): + """ + 下载 + :param start: + :param end: :return: """ - logger.info(f'线程: Thread-{self.name} 开始下载') - headers = {"Range": "bytes=%s-%s" % (self.startpos, self.endpos)} - res = self.session.get(self.url, headers=headers) - self.fd.seek(self.startpos) - self.fd.write(res.content) - logger.info(f'线程: Thread-{self.name} 结束下载') + headers = {'Range': 'bytes={}-{}'.format(start, end)} + r = get(self.url, headers=headers, stream=True) + lock.acquire() + with open(self.name, "rb+") as fp: + fp.seek(start) + fp.write(r.content) + lock.release() def run(self): """ - 此处启动 + 运行 :return: """ - self.download() - - -class Check: - """检查md5是否相同和下载数据库""" - - def __init__(self): - self.db_name = 'data.db' - self.assets_url = 'https://themedatabases.vercel.app/assets' - self.remote_md5 = 'https://themedatabases.vercel.app/md5' - self.session = requests.Session() - self.session.trust_env = False - - def check_md5(self): - """ - 检验本地文件md5是否和远程md5相同 - :return: - """ - with open(f'./bin/db/{self.db_name}', 'rb') as fp: - data = fp.read() - local_md5 = hashlib.md5(data).hexdigest() - remote_md5 = self.session.get(self.remote_md5).json()['data'][0] - logger.info(f'本地数据库md5: {local_md5}') - logger.info(f'远程数据库md5: {remote_md5}') - if local_md5 != remote_md5: - logger.error('下载错误: 本地数据库md5和远程数据库md5检验不通过, 即将开始重新下载\nI: 本次下载将使用单线程下载') - self.single_download() - else: - logger.info('md5检验已通过') - - def download(self): - """ - 开始下载 - :return: - """ - filesize = int(self.session.get(self.assets_url).headers['Content-Length']) - threaded_count = 3 - logger.info(f'数据库大小: {round(filesize / 1024 / 1024, 2)}Mb. 下载线程: {threaded_count}') - threading.BoundedSemaphore(threaded_count) - step = filesize // threaded_count - mtd_list = [] - start = 0 - end = -1 - with open(f'./bin/db/{self.db_name}', 'w') as initial_file: - initial_file.close() - with open(f'./bin/db/{self.db_name}', 'rb+') as f: - name = 1 - fileno = f.fileno() - while end < filesize - 1: - start = end + 1 - end = start + step - 1 - if end > filesize: - end = filesize - dup = os.dup(fileno) - fd = os.fdopen(dup, 'rb+', -1) - t = MulThreadDownload(self.assets_url, start, end, fd, name) - name += 1 - t.start() - mtd_list.append(t) - - for i in mtd_list: - i.join() - - self.check_md5() - - def single_download(self): - """ - 单线程进行下载 - :return: - """ - session = requests.Session() - session.trust_env = False - logger.info(f'正在使用单线程下载中') - resp = session.get(self.assets_url) - with open(f'./bin/db/{self.db_name}', 'wb') as fp: - fp.write(resp.content) - logger.info('下载完成 正在检验文件md5') - with open(f'./bin/db/{self.db_name}', 'rb') as fp: - data = fp.read() - local_md5 = hashlib.md5(data).hexdigest() - remote_md5 = session.get('https://themedatabases.vercel.app/md5').json()['data'][0] - logger.info(f'本地数据库md5: {local_md5}') - logger.info(f'远程数据库md5: {remote_md5}') - if local_md5 != remote_md5: - logger.error('md5检验未通过请手动前往 https://themedatabases.vercel.app/assets 下载文件并放入./bin/db文件夹内') - sys.exit(-1) - else: - logger.info('md5检验已通过') + fp = open(self.name, "wb") + fp.truncate(self.size) + fp.close() + part = self.size // self.num + pool = ThreadPoolExecutor(max_workers=self.num) + futures = [] + for i in range(self.num): + start = part * i + if i == self.num - 1: + end = self.size + else: + end = start + part - 1 + futures.append(pool.submit(self.down, start, end)) + wait(futures) + logger.info('数据库: %s 下载完成' % self.name.split('/')[-1]) if __name__ != '__main__': - conf = Settings() - if not os.path.exists('./bin/log'): - os.mkdir('./bin/log') - if not os.path.exists(f'./bin/db/data.db'): - logger.error('没有检测到本地主题数据库即将开始下载') - Check().download() + if not os.path.exists('./bin/db/data.db'): + logger.error('没有检测到数据库文件, 即将开始下载data.db') + Downloader('https://themedatabase.vercel.app/assets', 4, './bin/db/data.db').run() diff --git a/bin/tests/__init__.py b/bin/tests/__init__.py index 3950941..3d22f75 100644 --- a/bin/tests/__init__.py +++ b/bin/tests/__init__.py @@ -5,76 +5,8 @@ # @Create Time: 2022/2/3 # @File Name: __init__.py.py + import sys -sys.stdout.write('\033[0;34mMySQL数据库支持在./bin/tests/MySQLSET/MySQL.py\n\033[0m' - '\033[0;34m由原本pymysql库更换为了SQLAlchemy库, 已经将增删查改四个功能写好\n\033[0m' - '\033[0;34m由于一些原因无法使用\033[0m' - '\033[0;34m数据库导入文件请前往 https://themedatabase.vercel.app/source/sql 下载\n\033[0m' - '\033[0;34msql文件来源: 使用SQLiteStudio直接导出为.sql文件\n\033[0m' - '\033[0;34m使用 source db.sql 导入数据库时出现了一些错误:\n\033[0m' - '\033[0;34m有几个base64编码的值无法插入到表内\n\033[0m' - '\033[0;34m如果你有能力贡献代码请毫不犹豫地提交 pull request 吧!\n\n\033[0m') -sys.stdout.write('\033[1;31m----------以下为报错信息----------\033[0m') -sys.stdout.write( - r""" - Traceback (most recent call last): - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 523, in cmd_query - self._cmysql.query(query, - _mysql_connector.MySQLInterfaceError: Table 'data.reqcount' doesn't exist - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context - self.dialect.do_execute( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 732, in do_execute - cursor.execute(statement, parameters) - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\cursor_cext.py", line 269, in execute - result = self._cnx.cmd_query(stmt, raw=self._raw, - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 528, in cmd_query - raise errors.get_mysql_exception(exc.errno, msg=exc.msg, - mysql.connector.errors.ProgrammingError: 1146 (42S02): Table 'data.reqcount' doesn't exist - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "C:\Users\Tapso\PycharmProjects\RequestCounter\bin\tests\MySQL.py", line 133, in - print(mysql.fetch('AAA')) - File "C:\Users\Tapso\PycharmProjects\RequestCounter\bin\tests\MySQL.py", line 102, in fetch - data = self.session.query(ReqCount).filter(ReqCount.name == name).all() - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2759, in all - return self._iter().all() - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2894, in _iter - result = self.session.execute( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\session.py", line 1692, in execute - result = conn._execute_20(statement, params or {}, execution_options) - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1614, in _execute_20 - return meth(self, args_10style, kwargs_10style, execution_options) - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\sql\elements.py", line 325, in _execute_on_connection - return connection._execute_clauseelement( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1481, in _execute_clauseelement - ret = self._execute_context( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1845, in _execute_context - self._handle_dbapi_exception( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 2026, in _handle_dbapi_exception - util.raise_( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_ - raise exception - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context - self.dialect.do_execute( - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 732, in do_execute - cursor.execute(statement, parameters) - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\cursor_cext.py", line 269, in execute - result = self._cnx.cmd_query(stmt, raw=self._raw, - File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 528, in cmd_query - raise errors.get_mysql_exception(exc.errno, msg=exc.msg, - sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError) 1146 (42S02): Table 'data.reqcount' doesn't exist - [SQL: SELECT reqcount.name AS reqcount_name, reqcount.times AS reqcount_times - FROM reqcount - WHERE reqcount.name = %(name_1)s] - [parameters: {'name_1': 'AAA'}] - (Background on this error at: https://sqlalche.me/e/14/f405)""" -) - -input() \ No newline at end of file +sys.stdout.write('禁止调用') +sys.exit(-1) diff --git a/bin/utils/__init__.py b/bin/utils/__init__.py index 02eda15..7ea9087 100644 --- a/bin/utils/__init__.py +++ b/bin/utils/__init__.py @@ -11,7 +11,8 @@ from bin.utils.logger import logger from bin.utils.settings import Settings if __name__ != '__main__': - if Settings().type == 'MySQL' and not os.path.exists('./bin/db/origin.sql'): - logger.info('当前使用的数据库为MySQL请前往 https://themedatabase.vercel.app/source/sql 下载sql文件') - logger.info('并使用 source 命令来导入MySQL数据库') - logger.info('忽略此消息请在./bin/db内创建一个名为origin.sql的文件') + if Settings().type.lower() == 'mysql' and not os.path.exists('./static/origin.sql'): + logger.warning('当前使用的数据库为MySQL请前往 https://themedatabase.vercel.app/source/sql 下载sql文件') + logger.warning('并使用 source 命令来导入MySQL数据库') + logger.warning('此消息只显示一次, 下次启动不显示') + open('./static/origin.sql', 'w').close()