解决chrome quit的问题

This commit is contained in:
liuwei
2025-05-26 10:33:19 +08:00
parent ae3055ba63
commit 2475fb8d7f

View File

@@ -20,7 +20,8 @@ headers = {
seen_posts = set() seen_posts = set()
download_root = "/mnt/nfs_share" # 全局定义下载根目录 download_root = "/mnt/nfs_share" # 全局定义下载根目录
def fetch_posts(base_url,dl_path, posts_per_batch=10):
def fetch_posts(base_url, dl_path, posts_per_batch=10):
posts = [] posts = []
page = 1 page = 1
@@ -79,6 +80,8 @@ def get_total_pages(post_url):
def fetch_images(post_url): def fetch_images(post_url):
driver = None
try:
images = [] images = []
total_pages = get_total_pages(post_url) total_pages = get_total_pages(post_url)
logger.info(f"帖子 {post_url} 共有 {total_pages}") logger.info(f"帖子 {post_url} 共有 {total_pages}")
@@ -122,6 +125,11 @@ def fetch_images(post_url):
driver.quit() driver.quit()
return images return images
except Exception as e:
logger.info(f"爬取 {post_url} 失败: {e}")
return []
finally:
driver.quit()
def download_image(img_url, folder_path, img_index, max_retries=3): def download_image(img_url, folder_path, img_index, max_retries=3):
@@ -175,7 +183,7 @@ def meitu_dowload_pic(dl_path, dl_url):
os.makedirs(dl_path) os.makedirs(dl_path)
logger.info(f"开始爬取 {base_url} 的帖子...") logger.info(f"开始爬取 {base_url} 的帖子...")
posts = fetch_posts(base_url,dl_path, 10) posts = fetch_posts(base_url, dl_path, 10)
if not posts: if not posts:
logger.info("未获取到符合条件的帖子,请检查选择器或网络连接。") logger.info("未获取到符合条件的帖子,请检查选择器或网络连接。")
@@ -210,6 +218,5 @@ def meitu_dowload_pub_pic():
meitu_dowload_pic(download_root, "https://www.mntuce.com/") meitu_dowload_pic(download_root, "https://www.mntuce.com/")
if __name__ == "__main__": if __name__ == "__main__":
meitu_dowload_pub_pic() meitu_dowload_pub_pic()