import time import random import json import threading from flask import Flask, request, jsonify from DrissionPage import ChromiumPage, ChromiumOptions from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime from sqlalchemy.orm import declarative_base, sessionmaker # =================== 配置部分 =================== # 浏览器路径（请根据本地实际路径修改） CHROME_PATH = r'C:\Program Files\Google\Chrome\Application\chrome.exe' # MySQL 配置 db_config = { "host": "192.168.8.88", "port": 3306, "user": "root", "password": "mysql_7sjTXH", # 修改为你的密码 "database": "jd" } # 初始化 Flask 应用 app = Flask(__name__) # 初始化锁 fetch_lock = threading.Lock() # 全局爬虫控制标志 crawler_running = False crawler_thread = None current_product_id = None # 当前“允许运行”的抓取任务 product_id（新请求会覆盖，旧线程检测到不匹配则退出） active_fetch_product_id = None # 初始化数据库连接 db_url = f"mysql+pymysql://{db_config['user']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['database']}?charset=utf8mb4" engine = create_engine(db_url, echo=False) Session = sessionmaker(bind=engine) Base = declarative_base() # 定义评论模型 class Comment(Base): __tablename__ = 'comments' id = Column(Integer, primary_key=True) product_id = Column(String(50), nullable=False) user_name = Column(String(100)) comment_text = Column(Text) comment_id = Column(String(100)) picture_urls = Column(Text) # 存储 JSON 字符串 created_at = Column(DateTime, default=time.strftime('%Y-%m-%d %H:%M:%S')) comment_date = Column(DateTime, default=time.strftime('%Y-%m-%d %H:%M:%S')) # 创建表（如果不存在） Base.metadata.create_all(engine) # =================== 核心爬虫函数 =================== # 全局浏览器实例（只初始化一次） global_page = None def get_global_browser(): global global_page if global_page is None: options = ChromiumOptions() options.set_browser_path(CHROME_PATH) global_page = ChromiumPage(options) return global_page def _is_fetch_cancelled(product_id): """当前任务是否已被新请求取消（只保留最新请求的 product_id）""" global active_fetch_product_id return active_fetch_product_id is not None and active_fetch_product_id != product_id def fetch_jd_comments(product_id): global active_fetch_product_id page = get_global_browser() # 使用全局浏览器 try: # 打开商品页面 page.get(f'https://item.jd.com/{product_id}.html#crumb-wrap') time.sleep(random.uniform(5, 8)) if _is_fetch_cancelled(product_id): return 0 # 向下滚动主页面 page.scroll.down(150) time.sleep(random.uniform(3, 5)) if _is_fetch_cancelled(product_id): return 0 # 点击“买家赞不绝口” element1 = page.ele('xpath=//div[contains(text(), "买家赞不绝口")]') if element1: element1.click() time.sleep(random.uniform(3, 5)) else: element1 = page.ele('xpath=//div[contains(text(), "好评率")]') if element1: element1.click() time.sleep(random.uniform(3, 5)) if _is_fetch_cancelled(product_id): return 0 # 点击“当前商品” element2 = page.ele('xpath=//div[contains(text(), "当前商品")]') if element2: element2.click() time.sleep(random.uniform(3, 5)) if _is_fetch_cancelled(product_id): return 0 # 定位弹窗区域 popup = page.ele('xpath=//*[@id="rateList"]/div/div[3]') if not popup: return 0 # 点击“视频” element3 = page.ele('xpath=//div[contains(text(), "视频")]') if element3: element3.click() time.sleep(random.uniform(3, 5)) if _is_fetch_cancelled(product_id): return 0 # 监听请求 page.listen.start('https://api.m.jd.com/client.action') retry_count = 0 new_comments = [] # 存储最终的新评论 seen_ids = set() # 已处理过的 comment_id total_comments_saved = 0 # 总共保存的评论数 # 持续获取评论，直到被新请求取消或手动停止 while True: if _is_fetch_cancelled(product_id): print(f"[fetch_jd_comments] 商品 {product_id} 已被新请求取消，退出") break scroll_amount = random.randint(10000, 100000) popup.scroll.down(scroll_amount) print(f"弹窗向下滚动了 {scroll_amount} 像素") time.sleep(random.uniform(3, 5)) if _is_fetch_cancelled(product_id): break resp = page.listen.wait(timeout=5) if resp and 'getCommentListPage' in resp.request.postData: json_data = resp.response.body if 'result' in json_data and 'floors' in json_data['result']: comment_floor = json_data['result']['floors'][2] if 'data' in comment_floor and isinstance(comment_floor['data'], list): batch_comments = comment_floor['data'] # 提取这批评论中的新评论 fresh_comments = [] for comment in batch_comments: comment_info = comment.get('commentInfo', {}) comment_id = comment_info.get('commentId', '') comment_score = comment_info.get('commentScore', '') # 获取评分字段 if not comment_id: continue # 只保留五星好评 if comment_score != '5': print(f"跳过非五星评论：{comment_id}，评分为 {comment_score}") continue # 如果该评论已存在数据库或本次已收集，则跳过 exists_in_db = False if comment_id in seen_ids: exists_in_db = True else: session = Session() exists_in_db = session.query(Comment).filter_by(comment_id=comment_id).first() is not None session.close() if exists_in_db: print(f"评论已存在：{comment_id}") continue seen_ids.add(comment_id) fresh_comments.append(comment) if fresh_comments: print(f"本次获取到 {len(fresh_comments)} 条新评论") new_comments.extend(fresh_comments) retry_count = 0 # 有新数据，重置重试计数器 # 立即保存这批评论到数据库 save_comments_to_db(product_id, fresh_comments) total_comments_saved += len(fresh_comments) print(f"已保存 {len(fresh_comments)} 条评论到数据库，总计保存 {total_comments_saved} 条评论") else: print("本次无新评论，继续滚动...") retry_count += 1 else: print("未找到有效的评论列表") retry_count += 1 else: print("返回数据结构异常") retry_count += 1 else: print("未捕获到新的评论数据，继续滚动...") retry_count += 1 if _is_fetch_cancelled(product_id): break print(f"爬虫已停止，共抓取到 {total_comments_saved} 条评论") return total_comments_saved except Exception as e: print("发生错误:", e) return 0 # =================== 持续爬虫后台运行函数 =================== def continuous_crawler(product_id): """持续爬取评论的后台函数""" global crawler_running try: print(f"开始持续爬取商品 {product_id} 的评论...") while crawler_running: result = fetch_jd_comments(product_id) if not crawler_running: break # 如果没有获取到数据，等待一段时间再继续 time.sleep(10) print(f"商品 {product_id} 的持续爬取已停止") except Exception as e: print(f"持续爬虫发生错误: {e}") crawler_running = False # =================== 提取评论并保存到数据库 =================== def save_comments_to_db(product_id, comments): session = Session() try: for comment in comments: comment_info = comment.get('commentInfo', {}) comment_id = comment_info.get('commentId', '') # 如果 comment_id 为空，跳过这条评论 if not comment_id: print("跳过无 comment_id 的评论") continue # 检查是否已存在该评论 exists = session.query(Comment).filter_by(comment_id=comment_id).first() if exists: print(f"评论已存在：{comment_id}") continue # 提取其他字段 user_name = comment_info.get('userNickName', '匿名用户') comment_text = comment_info.get('commentData', '无评论内容') product_id = comment_info.get('productId', product_id) picture_list = comment_info.get('pictureInfoList', []) comment_date = comment_info.get('commentDate', '') picture_urls = [pic.get('largePicURL') for pic in picture_list if pic.get('largePicURL')] new_comment = Comment( product_id=product_id, user_name=user_name, comment_text=comment_text, comment_id=comment_id, picture_urls=json.dumps(picture_urls, ensure_ascii=False), comment_date=comment_date ) session.add(new_comment) session.commit() except Exception as e: session.rollback() print("保存失败:", e) finally: session.close() # =================== Flask API 接口 =================== @app.route('/start_crawler', methods=['POST']) def start_crawler(): """启动持续爬虫""" global crawler_running, crawler_thread, current_product_id product_id = request.args.get('product_id') if not product_id: return jsonify({"error": "缺少 product_id"}), 400 if crawler_running: return jsonify({ "message": f"爬虫已在运行中，当前商品ID: {current_product_id}", "status": "already_running" }), 200 try: with fetch_lock: crawler_running = True current_product_id = product_id crawler_thread = threading.Thread(target=continuous_crawler, args=(product_id,)) crawler_thread.daemon = True crawler_thread.start() return jsonify({ "message": f"已启动持续爬虫，商品ID: {product_id}", "status": "started", "product_id": product_id }), 200 except Exception as e: crawler_running = False return jsonify({"error": str(e)}), 500 @app.route('/stop_crawler', methods=['POST']) def stop_crawler(): """停止持续爬虫""" global crawler_running, crawler_thread, current_product_id if not crawler_running: return jsonify({ "message": "爬虫未在运行", "status": "not_running" }), 200 try: with fetch_lock: crawler_running = False stopped_product_id = current_product_id current_product_id = None # 等待线程结束 if crawler_thread and crawler_thread.is_alive(): crawler_thread.join(timeout=10) return jsonify({ "message": f"已停止持续爬虫，商品ID: {stopped_product_id}", "status": "stopped", "product_id": stopped_product_id }), 200 except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/crawler_status', methods=['GET']) def crawler_status(): """获取爬虫状态""" global crawler_running, current_product_id return jsonify({ "running": crawler_running, "product_id": current_product_id, "status": "running" if crawler_running else "stopped" }), 200 @app.route('/test', methods=['GET']) def test(): """测试端点，验证服务器是否正常工作""" print("测试端点被访问") return jsonify({"message": "服务器运行正常", "status": "ok"}), 200 @app.route('/fetch_comments', methods=['GET', 'POST']) def fetch_comments(): """单次获取评论（在后台运行，立即返回）。新请求会中断所有历史请求线程，只执行本次请求。""" global crawler_running, active_fetch_product_id print(f"[fetch_comments] 收到请求，方法: {request.method}, 参数: {request.args}") product_id = request.args.get('product_id') if not product_id: print("[fetch_comments] 错误: 缺少 product_id") return jsonify({"error": "缺少 product_id"}), 400 print(f"[fetch_comments] 开始处理商品ID: {product_id}，将中断所有历史请求后执行") try: # 立刻中断所有历史：停止持续爬虫并标记“当前任务”为新 product_id，旧线程在循环中检测到会自行退出 with fetch_lock: crawler_running = False active_fetch_product_id = product_id def run_fetch(): try: print(f"[后台线程] 开始获取商品 {product_id} 的评论...") result = fetch_jd_comments(product_id) print(f"[后台线程] 获取完成，结果: {result}") except Exception as e: import traceback error_msg = f"后台获取评论时发生错误: {e}\n{traceback.format_exc()}" print(f"[后台线程] {error_msg}") fetch_thread = threading.Thread(target=run_fetch) fetch_thread.daemon = True fetch_thread.start() print(f"[fetch_comments] 后台线程已启动（历史请求已标记为取消）") response_data = { "message": f"已开始获取商品 {product_id} 的评论，正在后台运行中...（已中断之前的请求）", "status": "started", "product_id": product_id, "note": "评论获取在后台进行，请稍后查看数据库或使用 /crawler_status 查看状态" } print(f"[fetch_comments] 返回响应: {response_data}") return jsonify(response_data), 200 except Exception as e: import traceback error_msg = f"处理请求时发生错误: {e}\n{traceback.format_exc()}" print(f"[fetch_comments] {error_msg}") return jsonify({"error": str(e)}), 500 # =================== 启动服务 =================== if __name__ == '__main__': try: app.run(host='0.0.0.0', port=5008, debug=True) finally: if 'global_page' in globals() and global_page: global_page.quit() print("浏览器已关闭")