解决chrome quit的问题
This commit is contained in:
@@ -20,7 +20,8 @@ headers = {
|
|||||||
seen_posts = set()
|
seen_posts = set()
|
||||||
download_root = "/mnt/nfs_share" # 全局定义下载根目录
|
download_root = "/mnt/nfs_share" # 全局定义下载根目录
|
||||||
|
|
||||||
def fetch_posts(base_url,dl_path, posts_per_batch=10):
|
|
||||||
|
def fetch_posts(base_url, dl_path, posts_per_batch=10):
|
||||||
posts = []
|
posts = []
|
||||||
page = 1
|
page = 1
|
||||||
|
|
||||||
@@ -79,6 +80,8 @@ def get_total_pages(post_url):
|
|||||||
|
|
||||||
|
|
||||||
def fetch_images(post_url):
|
def fetch_images(post_url):
|
||||||
|
driver = None
|
||||||
|
try:
|
||||||
images = []
|
images = []
|
||||||
total_pages = get_total_pages(post_url)
|
total_pages = get_total_pages(post_url)
|
||||||
logger.info(f"帖子 {post_url} 共有 {total_pages} 页")
|
logger.info(f"帖子 {post_url} 共有 {total_pages} 页")
|
||||||
@@ -122,6 +125,11 @@ def fetch_images(post_url):
|
|||||||
|
|
||||||
driver.quit()
|
driver.quit()
|
||||||
return images
|
return images
|
||||||
|
except Exception as e:
|
||||||
|
logger.info(f"爬取 {post_url} 失败: {e}")
|
||||||
|
return []
|
||||||
|
finally:
|
||||||
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
def download_image(img_url, folder_path, img_index, max_retries=3):
|
def download_image(img_url, folder_path, img_index, max_retries=3):
|
||||||
@@ -175,7 +183,7 @@ def meitu_dowload_pic(dl_path, dl_url):
|
|||||||
os.makedirs(dl_path)
|
os.makedirs(dl_path)
|
||||||
|
|
||||||
logger.info(f"开始爬取 {base_url} 的帖子...")
|
logger.info(f"开始爬取 {base_url} 的帖子...")
|
||||||
posts = fetch_posts(base_url,dl_path, 10)
|
posts = fetch_posts(base_url, dl_path, 10)
|
||||||
|
|
||||||
if not posts:
|
if not posts:
|
||||||
logger.info("未获取到符合条件的帖子,请检查选择器或网络连接。")
|
logger.info("未获取到符合条件的帖子,请检查选择器或网络连接。")
|
||||||
@@ -210,6 +218,5 @@ def meitu_dowload_pub_pic():
|
|||||||
meitu_dowload_pic(download_root, "https://www.mntuce.com/")
|
meitu_dowload_pic(download_root, "https://www.mntuce.com/")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
meitu_dowload_pub_pic()
|
meitu_dowload_pub_pic()
|
||||||
|
|||||||
Reference in New Issue
Block a user