重写下载器

bin/__init__.py

@@ -7,136 +7,70 @@
 
 
 import os
-import hashlib
 import sys
-import requests
+from requests import get
-import threading
+from threading import Lock
 from bin.utils.logger import logger
 from bin.utils.settings import Settings
+from concurrent.futures import ThreadPoolExecutor, wait
+
+lock = Lock()
+conf = Settings()
 
 
-class MulThreadDownload(threading.Thread):
+class Downloader:
-    def __init__(self, url, startpos, endpos, f, name):
+    def __init__(self, url, nums, file):
-        super(MulThreadDownload, self).__init__()
-        self.session = requests.Session()
-        self.session.trust_env = False
-        self.url = url  # 资源Url
-        self.startpos = startpos
-        self.endpos = endpos
-        self.fd = f  # 文件操作
-        self.name = name  # 线程名称
-
-    def download(self):
         """
-        多线程下载
+        初始化
+        :param url:
+        :param nums:
+        :param file:
+        """
+        self.url = url
+        self.num = nums
+        self.name = file
+        r = get(self.url)
+        self.size = int(r.headers['Content-Length'])
+        logger.info('文件大小为：{} Mb'.format(round(self.size / 1024 / 1024, 2)))
+
+    def down(self, start, end):
+        """
+        下载
+        :param start:
+        :param end:
         :return:
         """
-        logger.info(f'线程: Thread-{self.name} 开始下载')
+        headers = {'Range': 'bytes={}-{}'.format(start, end)}
-        headers = {"Range": "bytes=%s-%s" % (self.startpos, self.endpos)}
+        r = get(self.url, headers=headers, stream=True)
-        res = self.session.get(self.url, headers=headers)
+        lock.acquire()
-        self.fd.seek(self.startpos)
+        with open(self.name, "rb+") as fp:
-        self.fd.write(res.content)
+            fp.seek(start)
-        logger.info(f'线程: Thread-{self.name} 结束下载')
+            fp.write(r.content)
+            lock.release()
 
     def run(self):
         """
-        此处启动
+        运行
         :return:
         """
-        self.download()
+        fp = open(self.name, "wb")
-
+        fp.truncate(self.size)
-
+        fp.close()
-class Check:
+        part = self.size // self.num
-    """检查md5是否相同和下载数据库"""
+        pool = ThreadPoolExecutor(max_workers=self.num)
-
+        futures = []
-    def __init__(self):
+        for i in range(self.num):
-        self.db_name = 'data.db'
+            start = part * i
-        self.assets_url = 'https://themedatabases.vercel.app/assets'
+            if i == self.num - 1:
-        self.remote_md5 = 'https://themedatabases.vercel.app/md5'
+                end = self.size
-        self.session = requests.Session()
+            else:
-        self.session.trust_env = False
+                end = start + part - 1
-
+            futures.append(pool.submit(self.down, start, end))
-    def check_md5(self):
+        wait(futures)
-        """
+        logger.info('数据库: %s 下载完成' % self.name.split('/')[-1])
-        检验本地文件md5是否和远程md5相同
-        :return:
-        """
-        with open(f'./bin/db/{self.db_name}', 'rb') as fp:
-            data = fp.read()
-        local_md5 = hashlib.md5(data).hexdigest()
-        remote_md5 = self.session.get(self.remote_md5).json()['data'][0]
-        logger.info(f'本地数据库md5: {local_md5}')
-        logger.info(f'远程数据库md5: {remote_md5}')
-        if local_md5 != remote_md5:
-            logger.error('下载错误: 本地数据库md5和远程数据库md5检验不通过, 即将开始重新下载\nI: 本次下载将使用单线程下载')
-            self.single_download()
-        else:
-            logger.info('md5检验已通过')
-
-    def download(self):
-        """
-        开始下载
-        :return:
-        """
-        filesize = int(self.session.get(self.assets_url).headers['Content-Length'])
-        threaded_count = 3
-        logger.info(f'数据库大小: {round(filesize / 1024 / 1024, 2)}Mb. 下载线程: {threaded_count}')
-        threading.BoundedSemaphore(threaded_count)
-        step = filesize // threaded_count
-        mtd_list = []
-        start = 0
-        end = -1
-        with open(f'./bin/db/{self.db_name}', 'w') as initial_file:
-            initial_file.close()
-        with open(f'./bin/db/{self.db_name}', 'rb+') as f:
-            name = 1
-            fileno = f.fileno()
-            while end < filesize - 1:
-                start = end + 1
-                end = start + step - 1
-                if end > filesize:
-                    end = filesize
-                dup = os.dup(fileno)
-                fd = os.fdopen(dup, 'rb+', -1)
-                t = MulThreadDownload(self.assets_url, start, end, fd, name)
-                name += 1
-                t.start()
-                mtd_list.append(t)
-
-            for i in mtd_list:
-                i.join()
-
-        self.check_md5()
-
-    def single_download(self):
-        """
-        单线程进行下载
-        :return:
-        """
-        session = requests.Session()
-        session.trust_env = False
-        logger.info(f'正在使用单线程下载中')
-        resp = session.get(self.assets_url)
-        with open(f'./bin/db/{self.db_name}', 'wb') as fp:
-            fp.write(resp.content)
-        logger.info('下载完成 正在检验文件md5')
-        with open(f'./bin/db/{self.db_name}', 'rb') as fp:
-            data = fp.read()
-        local_md5 = hashlib.md5(data).hexdigest()
-        remote_md5 = session.get('https://themedatabases.vercel.app/md5').json()['data'][0]
-        logger.info(f'本地数据库md5: {local_md5}')
-        logger.info(f'远程数据库md5: {remote_md5}')
-        if local_md5 != remote_md5:
-            logger.error('md5检验未通过请手动前往 https://themedatabases.vercel.app/assets 下载文件并放入./bin/db文件夹内')
-            sys.exit(-1)
-        else:
-            logger.info('md5检验已通过')
 
 
 if __name__ != '__main__':
-    conf = Settings()
+    if not os.path.exists('./bin/db/data.db'):
-    if not os.path.exists('./bin/log'):
+        logger.error('没有检测到数据库文件, 即将开始下载data.db')
-        os.mkdir('./bin/log')
+        Downloader('https://themedatabase.vercel.app/assets', 4, './bin/db/data.db').run()
-    if not os.path.exists(f'./bin/db/data.db'):
-        logger.error('没有检测到本地主题数据库即将开始下载')
-        Check().download()

bin/tests/__init__.py

@@ -5,76 +5,8 @@
 # @Create Time: 2022/2/3
 # @File Name: __init__.py.py
 
+
 import sys
 
-sys.stdout.write('\033[0;34mMySQL数据库支持在./bin/tests/MySQLSET/MySQL.py\n\033[0m'
+sys.stdout.write('禁止调用')
-                 '\033[0;34m由原本pymysql库更换为了SQLAlchemy库, 已经将增删查改四个功能写好\n\033[0m'
+sys.exit(-1)
-                 '\033[0;34m由于一些原因无法使用\033[0m'
-                 '\033[0;34m数据库导入文件请前往 https://themedatabase.vercel.app/source/sql 下载\n\033[0m'
-                 '\033[0;34msql文件来源: 使用SQLiteStudio直接导出为.sql文件\n\033[0m'
-                 '\033[0;34m使用 source db.sql 导入数据库时出现了一些错误:\n\033[0m'
-                 '\033[0;34m有几个base64编码的值无法插入到表内\n\033[0m'
-                 '\033[0;34m如果你有能力贡献代码请毫不犹豫地提交 pull request 吧!\n\n\033[0m')
-sys.stdout.write('\033[1;31m----------以下为报错信息----------\033[0m')
-sys.stdout.write(
-    r"""
-    Traceback (most recent call last):
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 523, in cmd_query
-        self._cmysql.query(query,
-    _mysql_connector.MySQLInterfaceError: Table 'data.reqcount' doesn't exist
-
-    During handling of the above exception, another exception occurred:
-
-    Traceback (most recent call last):
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context
-        self.dialect.do_execute(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 732, in do_execute
-        cursor.execute(statement, parameters)
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\cursor_cext.py", line 269, in execute
-        result = self._cnx.cmd_query(stmt, raw=self._raw,
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 528, in cmd_query
-        raise errors.get_mysql_exception(exc.errno, msg=exc.msg,
-    mysql.connector.errors.ProgrammingError: 1146 (42S02): Table 'data.reqcount' doesn't exist
-
-    The above exception was the direct cause of the following exception:
-
-    Traceback (most recent call last):
-      File "C:\Users\Tapso\PycharmProjects\RequestCounter\bin\tests\MySQL.py", line 133, in <module>
-        print(mysql.fetch('AAA'))
-      File "C:\Users\Tapso\PycharmProjects\RequestCounter\bin\tests\MySQL.py", line 102, in fetch
-        data = self.session.query(ReqCount).filter(ReqCount.name == name).all()
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2759, in all
-        return self._iter().all()
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2894, in _iter
-        result = self.session.execute(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\session.py", line 1692, in execute
-        result = conn._execute_20(statement, params or {}, execution_options)
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1614, in _execute_20
-        return meth(self, args_10style, kwargs_10style, execution_options)
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\sql\elements.py", line 325, in _execute_on_connection
-        return connection._execute_clauseelement(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1481, in _execute_clauseelement
-        ret = self._execute_context(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1845, in _execute_context
-        self._handle_dbapi_exception(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 2026, in _handle_dbapi_exception
-        util.raise_(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
-        raise exception
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1802, in _execute_context
-        self.dialect.do_execute(
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 732, in do_execute
-        cursor.execute(statement, parameters)
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\cursor_cext.py", line 269, in execute
-        result = self._cnx.cmd_query(stmt, raw=self._raw,
-      File "C:\Users\Tapso\AppData\Local\Programs\Python\Python310\lib\site-packages\mysql\connector\connection_cext.py", line 528, in cmd_query
-        raise errors.get_mysql_exception(exc.errno, msg=exc.msg,
-    sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError) 1146 (42S02): Table 'data.reqcount' doesn't exist
-    [SQL: SELECT reqcount.name AS reqcount_name, reqcount.times AS reqcount_times
-    FROM reqcount
-    WHERE reqcount.name = %(name_1)s]
-    [parameters: {'name_1': 'AAA'}]
-    (Background on this error at: https://sqlalche.me/e/14/f405)"""
-)
-
-input()

bin/utils/__init__.py

@@ -11,7 +11,8 @@ from bin.utils.logger import logger
 from bin.utils.settings import Settings
 
 if __name__ != '__main__':
-    if Settings().type == 'MySQL' and not os.path.exists('./bin/db/origin.sql'):
+    if Settings().type.lower() == 'mysql' and not os.path.exists('./static/origin.sql'):
-        logger.info('当前使用的数据库为MySQL请前往 https://themedatabase.vercel.app/source/sql 下载sql文件')
+        logger.warning('当前使用的数据库为MySQL请前往 https://themedatabase.vercel.app/source/sql 下载sql文件')
-        logger.info('并使用 source 命令来导入MySQL数据库')
+        logger.warning('并使用 source 命令来导入MySQL数据库')
-        logger.info('忽略此消息请在./bin/db内创建一个名为origin.sql的文件')
+        logger.warning('此消息只显示一次, 下次启动不显示')
+        open('./static/origin.sql', 'w').close()