diff --git a/db.py b/db.py index d0a9073..5e59554 100644 --- a/db.py +++ b/db.py @@ -18,6 +18,9 @@ class DbAction: def get_db_data(self): query_sql = f"SELECT h.id, h.cate, h.date, h.`name`, h.save_link, h.`code`, h.unzip_pwd FROM scrapyh h WHERE h.date IN ({self.date_list})" + ext_filter = " AND save_link != '' AND code != '' AND file_name IS NULL " + query_sql = query_sql + ext_filter + print(query_sql) # 错误调试 # query_sql = query_sql + " AND id IN ('35334', '35335', '35336', '35337', '35338', '35339')" self.cursor.execute(query_sql) @@ -27,6 +30,12 @@ class DbAction: query_sql = f"UPDATE scrapyh SET file_name = %s WHERE id = %s" self.cursor.execute(query_sql, (file_name, data_id)) self.conn.commit() + print(f'{data_id}文件名为{file_name}') + + def mark_failed(self, data_id): + query_sql = f"UPDATE scrapyh SET file_name = '链接失效' WHERE id = %s" + self.cursor.execute(query_sql, (data_id,)) + self.conn.commit() def disconnect_db(self): self.cursor.close() diff --git a/main.py b/main.py index aa9d4e6..5625dcd 100644 --- a/main.py +++ b/main.py @@ -6,9 +6,10 @@ import random # config # get_end_date = 'today' -get_end_date = '2023-08-03' +get_end_date = '2023-08-09' +# 整理链接格式 def format_link(link): if '?pwd=' in link: link = link[:-9] @@ -17,6 +18,7 @@ def format_link(link): return link +# 封装通用方法 def req(_data, req_obj, db_obj): base_temp_path = r'/Temp/' date_str = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d') @@ -39,6 +41,7 @@ def main(): db_obj = DbAction(get_end_date) req_obj = ReqAction() try: + # 处理正常任务 data_list = db_obj.get_db_data() failed_list = [] req_obj.prepare() @@ -48,6 +51,7 @@ def main(): failed_list.append(result) sleep(random.randint(0, 3) + random.random()) i = 3 + # 重试失败任务 while len(failed_list) > 0 and i > 0: _temp_list = [] for _data in failed_list: @@ -57,9 +61,12 @@ def main(): sleep(random.randint(0, 3) + random.random()) failed_list = _temp_list i -= 1 - + # 标记失败任务 if len(failed_list): - failed_id = [_data['id'] for _data in failed_list] + failed_id = [] + for _data in failed_list: + failed_id.append(_data['id']) + db_obj.mark_failed(_data['id']) print('重试后依然失败:') print(failed_id) except Exception as e: diff --git a/req.py b/req.py index 79bb679..b3d9be4 100644 --- a/req.py +++ b/req.py @@ -138,7 +138,7 @@ class ReqAction: return [shareid_list[0], user_id_list[0], fs_id_list] # 转存文件函数 - @retry(stop_max_attempt_number=9, wait_fixed=random.randint(1000, 3000)) + @retry(stop_max_attempt_number=3, wait_fixed=random.randint(1000, 3000)) def transfer_files(self, verify_links_reason, target_directory_name): url = f'{BASE_URL}/share/transfer?shareid={verify_links_reason[0]}&from={verify_links_reason[1]}&bdstoken={self.bdstoken}&channel=chunlei&web=1&clienttype=0' post_data = {'fsidlist': f'[{",".join(i for i in verify_links_reason[2])}]', @@ -147,8 +147,11 @@ class ReqAction: allow_redirects=False, verify=False) print('transfer_files') print(response.text) - file_name = response.json()['extra']['list'][0]['from'] - return response.json()['errno'], file_name + result = response.json()['errno'] + file_name = None + if result == 0: + file_name = response.json()['extra']['list'][0]['from'] + return result, file_name def prepare(self): self.get_cookies() @@ -167,7 +170,7 @@ class ReqAction: verified_links = self.verify_links(save_link, save_code) if isinstance(verified_links, list): result, name = self.transfer_files(verified_links, save_path) - if result == 0: + if result == 0 or result == 4: return True, name return False, None