ArticleReplace/get_web_content.py

from bs4 import BeautifulSoup
import time
import random
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests


def extract_images_from_html(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')

    # 匹配所有以 https://p3-sign.toutiaoi 开头的图片链接
    img_tags = soup.find_all('img')
    img_urls = []

    for img in img_tags:
        for attr in ['src', 'data-src']:
            url = img.get(attr)
            if url and url.startswith("https://p3-sign.toutiaoimg.com/tos-cn-i"):
                img_urls.append(url)

    # 去重处理
    img_urls = list(dict.fromkeys(img_urls))

    # 返回 JSON 格式
    return {"image": img_urls}



# ============================================================
def get_webpage_source(url):
    """
    获取网页源代码的通用函数
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }

    try:
        # 添加随机延迟，模拟人类行为
        time.sleep(random.uniform(1, 3))
        response = requests.get(url, headers=headers, timeout=10)
        response.encoding = 'utf-8'

        # 检查响应状态
        if response.status_code == 200:
            return response.text
        else:
            print(f"请求失败，状态码: {response.status_code}")
            return None
    except Exception as e:
        print(f"获取网页源代码时出错: {e}")
        return None


# def get_webpage_source_selenium(url):
#     """
#     使用Selenium获取网页源代码，适用于动态加载内容的网站
#     """
#     # 配置Chrome选项
#     chrome_options = Options()
#     chrome_options.add_argument('--headless')  # 无头模式
#     chrome_options.add_argument('--disable-gpu')
#     chrome_options.add_argument('--no-sandbox')
#     chrome_options.add_argument('--disable-dev-shm-usage')
#     chrome_options.add_argument('--disable-blink-features=AutomationControlled')
#     chrome_options.add_argument(
#         'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
#
#     # 初始化WebDriver
#     driver = webdriver.Chrome(options=chrome_options)
#
#     try:
#         # 访问URL
#         driver.get(url)
#
#         # 等待页面加载完成（可根据实际情况调整等待条件）
#         time.sleep(3)  # 简单等待3秒
#
#         # 尝试等待文章内容加载
#         try:
#             WebDriverWait(driver, 10).until(
#                 EC.presence_of_element_located((By.TAG_NAME, "article"))
#             )
#         except:
#             print("等待文章元素超时，将使用当前页面内容")
#
#         # 获取页面源代码
#         page_source = driver.page_source
#
#         # 保存源代码到文件
#         with open("toutiao_source_selenium.html", "w", encoding="utf-8") as f:
#             f.write(page_source)
#
#         return page_source
#     except Exception as e:
#         print(f"使用Selenium获取网页源代码时出错: {e}")
#         return None
#     finally:
#         # 关闭浏览器
#         driver.quit()


# =====================采集内容内容==================================
# def toutiao_w_extract_content(url):
#     """
#     使用requests和BeautifulSoup提取头条页面内容
#     """
#     html_content = get_webpage_source_selenium(url)
#
#     # 使用BeautifulSoup解析HTML
#     soup = BeautifulSoup(html_content, 'html.parser')
#
#     # 提取标题和文章内容
#     article_element = soup.select_one('article')
#
#     if not article_element:
#         # 尝试其他可能的选择器
#         article_element = soup.select_one('.article-content') or soup.select_one('.content')
#
#     title_element = soup.select_one('h1') or soup.select_one('.article-title')
#     title_text = title_element.get_text().strip() if title_element else ""
#     article_text = article_element.get_text().strip() if article_element else ""
#
#     # 提取图片URL
#     img_elements = article_element.select('img') if article_element else []
#     img_urls = [img.get('src') for img in img_elements if img.get('src')]
#
#     return title_text, article_text, img_urls


def toutiao_extract_content(url):

    """
    使用requests和BeautifulSoup提取头条页面内容
    """
    html_content = get_webpage_source_selenium(url)

    # 使用BeautifulSoup解析HTML
    soup = BeautifulSoup(html_content, 'html.parser')



    # 提取标题和文章内容
    title_selector = '#root > div.article-detail-container > div.main > div.show-monitor > div > div > div > div > h1'
    article_selector = '#root > div.article-detail-container > div.main > div.show-monitor > div > div > div > div > div > article'

    title_element = soup.select_one(title_selector)
    article_element = soup.select_one(article_selector)

    title_text = title_element.get_text().strip() if title_element else ""
    article_text = article_element.get_text().strip() if article_element else ""

    # 提取图片URL
    # img_selector = "#root > div.article-detail-container > div.main > div.show-monitor article img"
    # img_elements = soup.select(img_selector)
    # # img_elements = article_element.select('img') if article_element else []

    img_urls = extract_images_from_html(html_content)['image']
    # img_urls = [img.get('src') for img in img_elements if img.get('src').startswith("https://p3")]




    return title_text, article_text, img_urls







def wechat_extract_content(url):
    """
    使用requests和BeautifulSoup提取微信公众号页面内容
    """
    html_content = get_webpage_source_selenium(url)

    # 使用BeautifulSoup解析HTML
    soup = BeautifulSoup(html_content, 'html.parser')

    # 使用指定的选择器提取标题和文章内容
    title_element = soup.select_one('#activity-name')
    article_element = soup.select_one('#js_content')

    title_text = title_element.get_text().strip() if title_element else ""
    article_text = article_element.get_text().strip() if article_element else ""

    # 提取特定 section 中的图片 URL（仅保留以 https://mmbiz.qpic.cn 开头的）
    img_elements = article_element.select('img') if article_element else []
    img_urls = []
    for img in img_elements:
        src = img.get('src') or img.get('data-src')
        if src and src.startswith('https://mmbiz.qpic.cn'):
            img_urls.append(src)

    return title_text, article_text, img_urls



def wangyi_extract_content(url):

    """
    使用requests和BeautifulSoup提取头条页面内容
    """
    html_content = get_webpage_source_selenium(url)

    # 使用BeautifulSoup解析HTML
    soup = BeautifulSoup(html_content, 'html.parser')



    # 提取标题和文章内容
    title_selector = '#contain > div.post_main > h1'
    article_selector = '#content > div.post_body'
    # img_selector = "#content > div.post_body > p > img"

    title_element = soup.select_one(title_selector)
    article_element = soup.select_one(article_selector)

    title_text = title_element.get_text().strip() if title_element else ""
    article_text = article_element.get_text().strip() if article_element else ""

    # 提取图片URL
    img_selector = "#content > div.post_body > p > img"
    img_elements = soup.select(img_selector)
    img_elements = article_element.select('img') if article_element else []

    # img_urls = extract_images_from_html(html_content)['image']
    img_urls = [img.get('src') for img in img_elements if img.get('src')]

    return title_text, article_text, img_urls




def souhu_extract_content(url):

    """
    使用requests和BeautifulSoup提取头条页面内容
    """
    html_content = get_webpage_source_selenium(url)

    # 使用BeautifulSoup解析HTML
    soup = BeautifulSoup(html_content, 'html.parser')


    print(soup)
    # 提取标题和文章内容
    title_selector = '#article-container > div.left.main > div:nth-child(1) > div > div.text-title > h1'
    article_selector = '#mp-editor'
    # img_selector = "#content > div.post_body > p > img"

    title_element = soup.select_one(title_selector)
    article_element = soup.select_one(article_selector)

    title_text = title_element.get_text().strip() if title_element else ""
    article_text = article_element.get_text().strip() if article_element else ""

    # 提取图片URL
    # img_selector = "#mp-editor > p > img"
    # img_elements = soup.select(img_selector)
    img_elements = article_element.select('img') if article_element else []


    img_urls = [img.get('src') for img in img_elements if img.get('src')]

    return title_text, article_text, img_urls


def toutiao_w_extract_content(url):
    """
    优化后的头条页面内容提取函数
    专门获取文章内容中的图片链接
    """
    html_content = get_webpage_source_selenium(url)

    if not html_content:
        print("获取HTML内容失败")
        return "", "", []

    # 使用BeautifulSoup解析HTML
    soup = BeautifulSoup(html_content, 'html.parser')

    # 多种标题选择器，按优先级尝试
    title_selectors = [
        '#root > div.article-detail-container > div.main > div.show-monitor > div > div > div > div > h1',
        'h1.article-title',
        'h1[data-testid="headline"]',
        '.article-title h1',
        '.article-header h1',
        'article h1',
        'h1'
    ]

    title_text = ""
    for selector in title_selectors:
        title_element = soup.select_one(selector)
        if title_element:
            title_text = title_element.get_text().strip()
            break

    # 多种文章内容选择器，按优先级尝试
    article_selectors = [
        '#root > div.article-detail-container > div.main > div.show-monitor > div > div > div > div > div > article',
        'article',
        '.article-content',
        '.content',
        '#js_content',
        '.post_body',
        '[data-testid="article-content"]'
    ]

    article_text = ""
    article_element = None
    for selector in article_selectors:
        article_element = soup.select_one(selector)
        if article_element:
            article_text = article_element.get_text().strip()
            break

    # 只从文章内容中提取图片
    img_urls = []

    if article_element:
        # 查找文章内容中的所有图片元素
        img_elements = article_element.find_all('img')

        for img in img_elements:
            # 尝试多种可能的图片URL属性
            for attr in ['src', 'data-src', 'data-original', 'data-lazy-src']:
                url = img.get(attr)
                if url:
                    # 处理相对路径
                    if url.startswith('//'):
                        url = 'https:' + url
                    elif url.startswith('/'):
                        url = 'https://www.toutiao.com' + url

                    # 只收集头条相关的图片URL
                    if any(domain in url for domain in ['toutiaoimg.com', 'p3-sign.toutiaoimg.com', 'byteimg.com']):
                        img_urls.append(url)
                        break  # 找到一个有效URL就跳出内层循环

    # 如果上面没有找到图片，尝试使用现有的extract_images_from_html函数作为备选
    if not img_urls:
        extracted_imgs = extract_images_from_html(html_content)
        if extracted_imgs and 'image' in extracted_imgs:
            img_urls = extracted_imgs['image']

    # 去重处理
    img_urls = list(dict.fromkeys(img_urls))

    return title_text, article_text, img_urls


def get_webpage_source_selenium(url):
    """
    增强版的Selenium获取网页源代码函数
    专门针对头条网站的动态加载特性进行优化
    """
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-dev-shm-usage')
    chrome_options.add_argument('--disable-blink-features=AutomationControlled')
    chrome_options.add_argument('--disable-images')  # 禁用图片加载以提高速度
    chrome_options.add_argument('--disable-javascript')  # 如果不需要JS，可以禁用
    chrome_options.add_argument(
        'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')

    driver = webdriver.Chrome(options=chrome_options)

    try:
        driver.get(url)

        # 等待页面加载完成
        time.sleep(5)

        # 尝试等待关键元素加载
        wait = WebDriverWait(driver, 15)
        try:
            # 等待文章标题加载
            wait.until(EC.presence_of_element_located((By.TAG_NAME, "h1")))
            # 等待文章内容加载
            wait.until(EC.presence_of_element_located((By.TAG_NAME, "article")))
        except:
            print("等待关键元素超时，使用当前页面内容")

        # 滚动页面以触发懒加载
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        driver.execute_script("window.scrollTo(0, 0);")
        time.sleep(1)

        page_source = driver.page_source

        # # 保存源代码用于调试
        # with open("toutiao_source_enhanced.html", "w", encoding="utf-8") as f:
        #     f.write(page_source)

        return page_source

    except Exception as e:
        print(f"使用增强版Selenium获取网页源代码时出错: {e}")
        return None
    finally:
        driver.quit()