解决chrome quit的问题
This commit is contained in:
@@ -20,7 +20,8 @@ headers = {
|
||||
seen_posts = set()
|
||||
download_root = "/mnt/nfs_share" # 全局定义下载根目录
|
||||
|
||||
def fetch_posts(base_url,dl_path, posts_per_batch=10):
|
||||
|
||||
def fetch_posts(base_url, dl_path, posts_per_batch=10):
|
||||
posts = []
|
||||
page = 1
|
||||
|
||||
@@ -79,6 +80,8 @@ def get_total_pages(post_url):
|
||||
|
||||
|
||||
def fetch_images(post_url):
|
||||
driver = None
|
||||
try:
|
||||
images = []
|
||||
total_pages = get_total_pages(post_url)
|
||||
logger.info(f"帖子 {post_url} 共有 {total_pages} 页")
|
||||
@@ -122,6 +125,11 @@ def fetch_images(post_url):
|
||||
|
||||
driver.quit()
|
||||
return images
|
||||
except Exception as e:
|
||||
logger.info(f"爬取 {post_url} 失败: {e}")
|
||||
return []
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
|
||||
def download_image(img_url, folder_path, img_index, max_retries=3):
|
||||
@@ -175,7 +183,7 @@ def meitu_dowload_pic(dl_path, dl_url):
|
||||
os.makedirs(dl_path)
|
||||
|
||||
logger.info(f"开始爬取 {base_url} 的帖子...")
|
||||
posts = fetch_posts(base_url,dl_path, 10)
|
||||
posts = fetch_posts(base_url, dl_path, 10)
|
||||
|
||||
if not posts:
|
||||
logger.info("未获取到符合条件的帖子,请检查选择器或网络连接。")
|
||||
@@ -210,6 +218,5 @@ def meitu_dowload_pub_pic():
|
||||
meitu_dowload_pic(download_root, "https://www.mntuce.com/")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
meitu_dowload_pub_pic()
|
||||
|
||||
Reference in New Issue
Block a user