From a9f247bf62369f433205f725380a1b81ef49b9ec Mon Sep 17 00:00:00 2001 From: Van0313 <60689272+Van0313@users.noreply.github.com> Date: Fri, 2 May 2025 20:39:46 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E8=AF=84=E8=AE=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/van/business/model/pl/Comment.java | 50 ++++++ .../repository/CommentRepository.java | 20 +++ .../java/cn/van/business/util/JDUtil.java | 153 +++++++++++------- .../business/util/ds/pl/FetchEquivalent.java | 115 ------------- .../util/ds/pl/JDItemReviewCrawler.java | 78 --------- .../util/ds/pl/SimpleCommentScraper.java | 109 ------------- .../cn/van/business/util/ds/pl/cookie.txt | 1 - 7 files changed, 169 insertions(+), 357 deletions(-) create mode 100644 src/main/java/cn/van/business/model/pl/Comment.java create mode 100644 src/main/java/cn/van/business/repository/CommentRepository.java delete mode 100644 src/main/java/cn/van/business/util/ds/pl/FetchEquivalent.java delete mode 100644 src/main/java/cn/van/business/util/ds/pl/JDItemReviewCrawler.java delete mode 100644 src/main/java/cn/van/business/util/ds/pl/SimpleCommentScraper.java delete mode 100644 src/main/java/cn/van/business/util/ds/pl/cookie.txt diff --git a/src/main/java/cn/van/business/model/pl/Comment.java b/src/main/java/cn/van/business/model/pl/Comment.java new file mode 100644 index 0000000..26d0cae --- /dev/null +++ b/src/main/java/cn/van/business/model/pl/Comment.java @@ -0,0 +1,50 @@ +package cn.van.business.model.pl; + +import jakarta.persistence.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * @author Leo + * @version 1.0 + * @create 2025/6/4 + * @description：评论实体类 + */ +@Entity +@Table(name = "comments") +@Data +public class Comment { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "product_id", nullable = false, columnDefinition = "TEXT") + private String productId; + + @Column(name = "user_name", columnDefinition = "TEXT") + private String userName; + + @Lob + @Column(name = "comment_text") + private String commentText; + + @Column(name = "comment_id", length = 64, unique = true) + private String commentId; + + @Column(name = "picture_urls", columnDefinition = "TEXT") + private String pictureUrls; + + @Column(name = "created_at") + private LocalDateTime createdAt; + + @Column(name = "comment_date") + private LocalDateTime commentDate; + + @Column(name = "is_use") + private Integer isUse; + + // Getters and Setters + +} diff --git a/src/main/java/cn/van/business/repository/CommentRepository.java b/src/main/java/cn/van/business/repository/CommentRepository.java new file mode 100644 index 0000000..5cc1768 --- /dev/null +++ b/src/main/java/cn/van/business/repository/CommentRepository.java @@ -0,0 +1,20 @@ +package cn.van.business.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import cn.van.business.model.pl.Comment; + +import java.util.List; + +/** + * @author Leo + * @version 1.0 + * @create 2025/5/2 19:16 + * @description：评论数据访问层接口 + */ +@Repository +public interface CommentRepository extends JpaRepository { + + List findByProductIdAndIsUseNotAndPictureUrlsIsNotNull(String productId, Integer isUse); + +} diff --git a/src/main/java/cn/van/business/util/JDUtil.java b/src/main/java/cn/van/business/util/JDUtil.java index 35046c2..b17ff3e 100644 --- a/src/main/java/cn/van/business/util/JDUtil.java +++ b/src/main/java/cn/van/business/util/JDUtil.java @@ -1,12 +1,17 @@ package cn.van.business.util; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; import cn.van.business.model.jd.OrderRow; +import cn.van.business.model.pl.Comment; +import cn.van.business.repository.CommentRepository; import cn.van.business.repository.OrderRowRepository; import cn.van.business.util.ds.DeepSeekClientUtil; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; +import com.alibaba.fastjson2.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.jd.open.api.sdk.DefaultJdClient; import com.jd.open.api.sdk.JdClient; @@ -78,7 +83,8 @@ public class JDUtil { private static final String ACCESS_TOKEN = ""; private static final Logger logger = LoggerFactory.getLogger(JDUtil.class); private static final String INTERACTION_STATE_PREFIX = "interaction_state:"; - private static final String COMMENT_TEMPLATES = "comment_templates:"; + private static final String PRODUCT_TYPE_MAP_PREFIX = "product_type_map" ; + private static HashMap productTypeMap = new HashMap<>(); private static final String COMMENT_TEMPLATES_DS = "我需要为我的商品模拟一些商品评论。你协助我生成3条评价内容，京东商品评价的风格，每条评价100字即可，不需要太浮夸，也不要太像ai生成，尽量模拟真实客户评价，不要提到以旧换新和国家补贴。我会为你提供其他真实用户的评论："; private static final long TIMEOUT_MINUTES = 2; private static final String WENAN_FANAN_LQD = "提供方法自己下单\n" + "全程都是自己的账号下单\n" + "标价就是下单的到手价\n" + "本人有耐心，会一步一步提供教程\n" + "以后有什么质量问题都是用自己的账号走京东售后\n" + "\n" + "更新\n" + "\n" + "用你自己的账号下单\n" + "官方店铺 提供方法自己下单\n" + "——————————————————————————————————————————\n" + "同行可长久合作，可提供神级家电线报\n" + "\n" + "配合家电线报可以自己下单，不用找代购和代下，订单和利润都掌握在自己手中。\n" + "\n" + "一次入会永久使用，包含家电帮，雷神价，韭菜帮，河南&湖南帮，各种暗号帮后返等内部独家家电线报\n" + "\n" + "JD采销采购不定时发放独家优惠券\n" + "\n" + "基本上你能看到的京东家电低价都是从这些渠道里面出来。\n" + "\n" + "2025年家电项目新方向，配合家电线报下单，秒省1K+。"; @@ -105,6 +111,8 @@ public class JDUtil { final WXUtil wxUtil; private final StringRedisTemplate redisTemplate; private final OrderRowRepository orderRowRepository; + private final CommentRepository commentRepository; + private final OrderUtil orderUtil; private final DeepSeekClientUtil deepSeekClientUtil; // 添加ObjectMapper来序列化和反序列化UserInteractionState @@ -114,12 +122,14 @@ public class JDUtil { // 构造函数中注入StringRedisTemplate @Autowired - public JDUtil(StringRedisTemplate redisTemplate, OrderRowRepository orderRowRepository, WXUtil wxUtil, OrderUtil orderUtil, DeepSeekClientUtil deepSeekClientUtil) { + public JDUtil(StringRedisTemplate redisTemplate, OrderRowRepository orderRowRepository, WXUtil wxUtil, OrderUtil orderUtil, DeepSeekClientUtil deepSeekClientUtil, CommentRepository commentRepository) { + this.redisTemplate = redisTemplate; this.orderRowRepository = orderRowRepository; this.wxUtil = wxUtil; this.orderUtil = orderUtil; this.deepSeekClientUtil = deepSeekClientUtil; + this.commentRepository = commentRepository; } private List filterOrdersByDate(List orderRows, int daysBack) { @@ -1577,7 +1587,7 @@ public class JDUtil { return; } String remark = ""; - if (!split[0].trim().replace("单", "").isEmpty()){ + if (!split[0].trim().replace("单", "").isEmpty()) { remark = split[0].trim().replace("{单}", "").replace("\n", ""); } // 今天的日期 @@ -1612,10 +1622,8 @@ public class JDUtil { for (int i = 0; i < num; i++) { int currentCount = startCount + i; String orderID = today + String.format("%03d", currentCount); // 格式化为三位数 - // 每次使用原始模板进行替换，避免污染 String currentTemp = WENAN_D.replace("{单的备注}", remark).replace("{单号}", orderID).replace("{分销标记}", split[1]).replace("{链接}", split[3]).replace("{地址}", split[5]).replace("{型号}", split[2]).replaceAll("[|]", ""); - // 发送订单信息到微信 wxUtil.sendTextMessage(fromWxid, currentTemp, 1, fromWxid, true); } @@ -1658,17 +1666,21 @@ public class JDUtil { } } + //getProductTypeMap 从redis中获取PRODUCT_TYPE_MAP_PREFIX + public void getProductTypeMap() { + String json = redisTemplate.opsForValue().get(PRODUCT_TYPE_MAP_PREFIX); + productTypeMap = JSON.parseObject(json, new TypeReference>() { + }); + } /** * 处理生成评论流程中的用户交互 */ private void handleCommentInteraction(String fromWxid, String message, UserInteractionState state) { - String superOrder = ""; try { String[] split = message.replaceAll("\r", "").split("\n"); if (split.length != 0) { message = split[0].trim().replace("\n", ""); for (int i = 1; i < split.length; i++) { - superOrder += split[i] + " "; } } } catch (Exception e) { @@ -1681,23 +1693,11 @@ public class JDUtil { if (!"commentTypeSelection".equals(state.getCurrentField())) { return; } - - // 根据用户输入生成对应的评论 - switch (message) { - case "1": - generateComment(fromWxid, "1", superOrder); - break; - case "1+": - generateComment(fromWxid, "1+", superOrder.replace("+", "")); - break; - case "2": - generateComment(fromWxid, "2", superOrder); - break; - case "2+": - generateComment(fromWxid, "2+", superOrder.replace("+", "")); - break; - default: - wxUtil.sendTextMessage(fromWxid, "无效的选择，请回复 1 或 2", 1, fromWxid, false); + getProductTypeMap(); + if (productTypeMap.containsKey(message)) { + generateComment(fromWxid, message); + } else { + wxUtil.sendTextMessage(fromWxid, "无效的选择，请回复 1 或 2", 1, fromWxid, false); } } catch (Exception e) { @@ -1713,42 +1713,87 @@ public class JDUtil { /** * 生成评论内容 */ - private void generateComment(String fromWxid, String productType, String superOrder) { - if (productType.equals("1") || productType.equals("2")) { - wxUtil.sendTextMessage(fromWxid, "DS生成中，等候过程请勿重复输入", 1, fromWxid, false); - // 这里可以调用缓存或AI生成文案，目前先返回固定模板 - //用redis的 list存放评论模板， - List commentTemplates = redisTemplate.opsForList().range(COMMENT_TEMPLATES + productType, 0, -1); - if (commentTemplates == null || commentTemplates.isEmpty()) { - wxUtil.sendTextMessage(fromWxid, "暂无评论模板，请先添加评论模板", 1, fromWxid, false); + private void generateComment(String fromWxid, String productType) { + wxUtil.sendTextMessage(fromWxid, "已接收到评论生成指令，等候过程请勿重复输入", 1, fromWxid, false); + + // 获取产品ID + getProductTypeMap(); + String product_id = productTypeMap.get(productType); + if (product_id == null || product_id.isEmpty()) { + wxUtil.sendTextMessage(fromWxid, "缺失对应的SKUID", 1, fromWxid, false); + return; + } + + // 从数据库获取可用评论 + List availableComments = commentRepository.findByProductIdAndIsUseNotAndPictureUrlsIsNotNull(product_id, 1); + + Comment commentToUse = null; + + if (!availableComments.isEmpty()) { + // 随机选取一个未使用的评论 + Collections.shuffle(availableComments); + commentToUse = availableComments.get(0); + } else { + // 没有本地评论，调用外部接口抓取 + try { + String fetchUrl = "http://192.168.8.169:5000/fetch_comments?product_id=" + product_id; + // 用hutool发起post请求 + HttpResponse response = HttpRequest.post(fetchUrl).execute(); + + logger.info("fetchUrl: {}", fetchUrl); +// code = 200 表示成功，-200 表示失败 + if (response.getStatus() == 200) { + // ✅ 关键修改：重新从数据库中查询，而不是使用内存中的 fetchedComments + availableComments = commentRepository.findByProductIdAndIsUseNotAndPictureUrlsIsNotNull(product_id, 1); + if (!availableComments.isEmpty()) { + Collections.shuffle(availableComments); + commentToUse = availableComments.get(0); + } + } else if (response.getStatus() == -200) { + wxUtil.sendTextMessage(fromWxid, "暂时无法获取新的评论，请稍后再试", 1, fromWxid, false); + return; + } + } catch (Exception e) { + logger.error("调用外部接口获取评论失败", e); + wxUtil.sendTextMessage(fromWxid, "调用外部接口获取评论失败", 1, fromWxid, false); return; } - // 全部追加到成一个文本 - StringBuilder comment = new StringBuilder(); - int count = 0; - for (String commentTemplate : commentTemplates) { - count++; - comment.append(count).append(commentTemplate); - } + } + if (commentToUse == null) { + wxUtil.sendTextMessage(fromWxid, "没有找到可用的评论数据", 1, fromWxid, false); + return; + } - String deepSeekResponse = "ds响应失败"; - try { - deepSeekResponse = deepSeekClientUtil.getDeepSeekResponse(COMMENT_TEMPLATES_DS + comment); - } catch (IOException e) { - logger.error("生成评论异常 - 用户: {}, ", fromWxid, e); - } - wxUtil.sendTextMessage(fromWxid, deepSeekResponse, 1, fromWxid, false); - // 重置状态 - resetState(fromWxid, loadOrCreateState(INTERACTION_STATE_PREFIX + fromWxid)); - } else { - if (!superOrder.isEmpty()) { - productType = productType.replace("+", ""); - // 追加评论模板到redis - redisTemplate.opsForList().leftPush(COMMENT_TEMPLATES + productType, superOrder); - wxUtil.sendTextMessage(fromWxid, "评论模板添加成功", 1, fromWxid, false); + // 调用 DeepSeek 生成新的评论内容 + String deepSeekPrompt = COMMENT_TEMPLATES_DS + commentToUse.getCommentText(); + String deepSeekResponse; + try { + deepSeekResponse = deepSeekClientUtil.getDeepSeekResponse(deepSeekPrompt); + } catch (IOException e) { + logger.error("生成评论异常 - 用户: {}", fromWxid, e); + wxUtil.sendTextMessage(fromWxid, "评论生成失败，请重试", 1, fromWxid, false); + return; + } + + // 发送生成的评论文本 + wxUtil.sendTextMessage(fromWxid, deepSeekResponse, 1, fromWxid, false); + + // 发送图片（如果有） + String pictureUrls = commentToUse.getPictureUrls(); + if (pictureUrls != null && !pictureUrls.isEmpty()) { + String[] urls = pictureUrls.split(","); + for (String url : urls) { + wxUtil.sendImageMessage(fromWxid, url.trim()); // 假设 sendImageMessage 支持 URL } } + + // 更新评论状态为已使用 + commentToUse.setIsUse(1); + commentRepository.save(commentToUse); + + // 重置状态 + resetState(fromWxid, loadOrCreateState(INTERACTION_STATE_PREFIX + fromWxid)); } diff --git a/src/main/java/cn/van/business/util/ds/pl/FetchEquivalent.java b/src/main/java/cn/van/business/util/ds/pl/FetchEquivalent.java deleted file mode 100644 index 72ecb36..0000000 --- a/src/main/java/cn/van/business/util/ds/pl/FetchEquivalent.java +++ /dev/null @@ -1,115 +0,0 @@ -package cn.van.business.util.ds.pl; - -/** - * @author Leo - * @version 1.0 - * @create 2025/5/1 16:33 - * @description： - */ -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - -public class FetchEquivalent { - private static final Logger logger = LoggerFactory.getLogger(FetchEquivalent.class); - - public static void main(String[] args) { - String url = "https://api.m.jd.com/client.action"; - - // 关键参数，可根据需求动态调整 - String appid = "pc-rate-qa"; - String functionId = "getCommentListPage"; - String spuId = "100104238904"; - String sku = "100104238904"; - int pageNum = 1; - int pageSize = 10; - String sortType = "5"; - String h5st = "20250501163137742;gggwzzg93mhaqp39;01a47;tk03wd91c1d5f18nmJkCbdT3fAvSVSRvhgBegKk_JURNSt4iorNQ5bdUGcosuVPknJ2RSpEjzcPmpjpW5mMLWtm95dd2;67e4ba5594c53473cec3c2ac9d0f2bb45a27256c33aa0cf0c0fc53f5c3e61f5f;5.1;1746088294742;ri_uxFOm22ciAuLmOGLm9aHWMusmk_Mm1irS9poUBhbgHFHQ3R4VHlsm0msSIlsmOGuj6mrm0mMTLhImOuMsCmsVJZIi3ebV1O7WLdrg_uLi3Kbi3ebg1e7h9iIhLJbi8msm0msSo94VMZ4RMusmk_MmIVrV7O7WKNbWJl4h5WIh8abi2aYh7qIhJZog2arg7WLmOGLm7pIRAp4WMusmk_siOGLm6aHWMusmk_Mm72ciAaLRPZoTFV4X5OImOGLm4lsmOGujMqnZMtLibd4i9VLmOGLmBxoVApISMusmk_Mm8iLTFRJmOGLmItHmOuMsC6nmOG_iOGLm9qbRMlsmOusmk_si9uMgMubi5lImOusmOGuj26sm0mMi9aHWMusmOuMsCmsQf16XNhai9Zph7WacMuMgM64TK1YW8lsmOusmk_siOGLm2aHWMusmOuMsCurm0m8h5lImOusmOGuj5irm0mMh5lImOusmOGuj_uMgMabRMlsmOusmk_siOGLm6aHWMusmOuMsCibh8uMgMibRMlsmOusmk_Mm12ciAuLmOGLm9aHWMusmOuMsCurm0m8U3lsmOusmk_chOGLm79ImOusmOGuj_uMgM_ImOusmOGuj_uMgMe4RMusmOuMsztMgMeITJdnQJlsmOGujxtsmkmMi9abW6a4iIRri4mohPdIUMuMgMmrSMusmOuMsztMgMunSMusmk_Mm6WrQOCrh42YUXt8g_2si9usZgt8S3xoVAJ4ZMuMgMqYR7lsmOG_Q;ecdd8239ff33baed5b5b19f9167325c045d13be878c9c3812dd9bcbc630d6b84;tenjKJKT-JoRL1YRI9MT-J4S8ZIZ61YVF94WCeHTJJoTL9cQKxIWCeYU_tXW"; - long t = System.currentTimeMillis(); - String uuid = "1744804395425546198998"; - - // 参数校验 - if (functionId == null || functionId.isEmpty() || appid == null || appid.isEmpty() || spuId == null || spuId.isEmpty() || sku == null || sku.isEmpty()) { - logger.error("关键参数不能为空"); - return; - } - - String body = String.format("appid=%s&body={\"requestSource\":\"pc\",\"shopComment\":0,\"sameComment\":0,\"channel\":null,\"extInfo\":{\"isQzc\":\"0\",\"spuId\":\"%s\",\"commentRate\":\"1\",\"needTopAlbum\":\"1\",\"bbtf\":\"\",\"userGroupComment\":\"1\"},\"num\":\"10\",\"pictureCommentType\":\"A\",\"scval\":null,\"shadowMainSku\":\"0\",\"shopType\":\"0\",\"firstCommentGuid\":\"e341be12da46b787bf5cd21a76ae3295\",\"sku\":\"%s\",\"category\":\"737;13297;1301\",\"shieldCurrentComment\":\"1\",\"pageSize\":\"%d\",\"isFirstRequest\":false,\"style\":\"0\",\"isCurrentSku\":false,\"sortType\":\"%s\",\"tagId\":\"\",\"tagType\":\"\",\"type\":\"4\",\"pageNum\":\"%d\"}&client=pc&clientVersion=1.0.0&functionId=%s&h5st=%s&loginType=3&t=%d&uuid=%s", - appid, spuId, sku, pageSize, sortType, pageNum, functionId, h5st, t, uuid); - - Map headers = new HashMap<>(); - headers.put("accept", "application/json, text/plain, */*"); - headers.put("accept-language", "zh-CN,zh;q=0.9,en;q=0.8"); - headers.put("cache-control", "no-cache"); - headers.put("content-type", "application/x-www-form-urlencoded"); - headers.put("pragma", "no-cache"); - headers.put("priority", "u=1, i"); - headers.put("sec-ch-ua", "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\""); - headers.put("sec-ch-ua-mobile", "?0"); - headers.put("sec-ch-ua-platform", "\"Windows\""); - headers.put("sec-fetch-dest", "empty"); - headers.put("sec-fetch-mode", "cors"); - headers.put("sec-fetch-site", "same-site"); - headers.put("x-referer-page", "https://item.jd.com/100104238904.html"); - headers.put("x-rp-client", "h5_1.0.0"); - headers.put("referrer", "https://item.jd.com/"); - headers.put("referrerPolicy", "strict-origin-when-cross-origin"); - - try { - URL obj = new URL(url); - HttpURLConnection con = (HttpURLConnection) obj.openConnection(); - - // 设置请求方法 - con.setRequestMethod("POST"); - - // 添加请求头 - for (Map.Entry entry : headers.entrySet()) { - con.setRequestProperty(entry.getKey(), entry.getValue()); - } - - // 允许输出流 - con.setDoOutput(true); - - // 写入请求体 - try (DataOutputStream wr = new DataOutputStream(con.getOutputStream())) { - wr.writeBytes(body); - wr.flush(); - } - - int responseCode = con.getResponseCode(); - logger.info("Response Code : " + responseCode); - - if (responseCode == HttpURLConnection.HTTP_OK) { - try (BufferedReader in = new BufferedReader( - new InputStreamReader(con.getInputStream()))) { - String inputLine; - StringBuilder response = new StringBuilder(); - while ((inputLine = in.readLine()) != null) { - response.append(inputLine); - } - logger.info("Response Content : " + response.toString()); - } - } else { - try (BufferedReader errorReader = new BufferedReader( - new InputStreamReader(con.getErrorStream()))) { - String errorLine; - StringBuilder errorResponse = new StringBuilder(); - while ((errorLine = errorReader.readLine()) != null) { - errorResponse.append(errorLine); - } - logger.error("请求失败，响应码: " + responseCode + "，错误信息: " + errorResponse.toString()); - } - } - } catch (IOException e) { - logger.error("请求过程中发生异常", e); - } - } -} diff --git a/src/main/java/cn/van/business/util/ds/pl/JDItemReviewCrawler.java b/src/main/java/cn/van/business/util/ds/pl/JDItemReviewCrawler.java deleted file mode 100644 index 9e5649f..0000000 --- a/src/main/java/cn/van/business/util/ds/pl/JDItemReviewCrawler.java +++ /dev/null @@ -1,78 +0,0 @@ -package cn.van.business.util.ds.pl; - -/** - * @author Leo - * @version 1.0 - * @create 2025/4/30 19:16 - * @description： - */ -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -public class JDItemReviewCrawler { - private static final String API_URL = "https://api.jd.com/routerjson"; - private static final String APP_KEY = "98e21c89ae5610240ec3f5f575f86a59"; - private static final String APP_SECRET = "3dcb6b23a1104639ac433fd07adb6dfb"; - - public static void main(String[] args) throws IOException { - String itemId = "100075270739"; // 商品ID - String page = "1"; // 页码 - String result = getItemReviews(itemId, page); - System.out.println(result); - } - - public static String getItemReviews(String itemId, String page) throws IOException { - Map params = new HashMap<>(); - params.put("method", "jd.item.review.get"); - params.put("app_key", APP_KEY); - params.put("v", "2.0"); - params.put("format", "json"); - params.put("sign_method", "md5"); - params.put("timestamp", String.valueOf(System.currentTimeMillis())); - params.put("num_iid", itemId); - params.put("page", page); - - String sign = generateSign(params, APP_SECRET); - params.put("sign", sign); - - String url = buildRequestUrl(params); - return sendHttpGetRequest(url); - } - - private static String generateSign(Map params, String appSecret) throws IOException { - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : params.entrySet()) { - sb.append(entry.getKey()).append(entry.getValue()); - } - sb.append(appSecret); - return URLEncoder.encode(sb.toString(), StandardCharsets.UTF_8.name()); - } - - private static String buildRequestUrl(Map params) throws IOException { - StringBuilder urlBuilder = new StringBuilder(API_URL); - urlBuilder.append("?"); - for (Map.Entry entry : params.entrySet()) { - urlBuilder.append(entry.getKey()).append("=").append(entry.getValue()).append("&"); - } - return urlBuilder.toString(); - } - - private static String sendHttpGetRequest(String url) throws IOException { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(url); - String result = httpClient.execute(httpGet, httpResponse -> EntityUtils.toString(httpResponse.getEntity())); - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode rootNode = objectMapper.readTree(result); - return rootNode.toString(); - } - } -} diff --git a/src/main/java/cn/van/business/util/ds/pl/SimpleCommentScraper.java b/src/main/java/cn/van/business/util/ds/pl/SimpleCommentScraper.java deleted file mode 100644 index 7d3a5ab..0000000 --- a/src/main/java/cn/van/business/util/ds/pl/SimpleCommentScraper.java +++ /dev/null @@ -1,109 +0,0 @@ -package cn.van.business.util.ds.pl; - -import cn.hutool.http.HttpRequest; -import cn.hutool.http.HttpResponse; - -import cn.hutool.json.JSONUtil; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.net.HttpURLConnection; - -import java.net.URL; -import java.util.Map; -import java.util.HashMap; - - -public class SimpleCommentScraper { - private static final Logger logger = LoggerFactory.getLogger(SimpleCommentScraper.class); - private static final int SLEEP_TIME = 1000; // 休眠时间，单位：毫秒 - private static final String JD_PROD_PIC_SAVE_PATH = "jd_prod_pic"; - private static final int PAGE_RANGE = 1; - private static final String BIG_PAG_PATH = "//img30.360buyimg.com/shaidan/s616x405_jfs/"; - private static final String TINY_PAG_PATH = "//img30.360buyimg.com/n0/s128x96_jfs/"; - - private static final Map headers = new HashMap<>(); - - static { - headers.put("cookie", "shshshfpa=0e6670b1-47be-d3a7-049d-3c9daa179153-1691582935; shshshfpx=0e6670b1-47be-d3a7-049d-3c9daa179153-1691582935; jcap_dvzw_fp=AMQNEJZc35A3KmQ7fLb-4FSnB2SbuA9auLr22ALve7Qyx9JarjpZCIHN8NIz4Bl_IB7RVdQ3LaO4xSrC0YsR3g==; __jdu=1744804395425546198998; whwswswws=; warehistory=\"10076121538879,10076121538879,10076121538879,\"; autoOpenApp_downCloseDate_autoOpenApp_autoPromptly=1744804996891_1; autoOpenApp_downCloseDate_jd_independent_coupon_openapp=1744948913486_1; pt_st=1_NWSLixpIv7w0rF2ENYK3Jf9y6xeCZi7by-s-cZ_os-zspi7V1bGhPcAZ3w02h0KlZuRNO4sgikpci-xjqrABXbJ-y524iEqm9uq2V0mkVH2RqLEesj0chjJtg-tcF6UBnUVWtVOqIaHczH4Ksux0Hlvk5oBhTziEhk7vR2wnLPciQ-W46Lld2YFdN_RldDpP-zVCY0pvepwAP8h8Ji4aNM-AueIvKCFFdqUYBq0K; autoOpenApp_downCloseDate_auto=1744948934067_1800000; unpl=JF8EAL1nNSttDxhRB0wEExJFTApdWwgASR4HO24GVgoKHFYNEgJMEBh7XlVdWRRKEh9vbxRUXlNLXA4ZASsSEHtdVV9eC04WAGdvNWRdWUpVBR8FGRUWe15Ublw4SxEGam4EUV9QT1EGHwYeFRBDVFxWVDhKJwNnYDVUXFhJXAAeBBwUGEleVG5tCXsUAWthBWRfWEpVDRsAExMTexw6Xl8KTREAa29IVFtdTl0EHgATFhVIWVBbWghDHgtnbjVVbVs; PCSYCityID=CN_440000_440400_0; __jdv=95931165%7Cbaidu%7C-%7Corganic%7Cnot%20set%7C1745683529792; areaId=19; 3AB9D23F7A4B3CSS=jdd03S5H5QMW2J75PKHW4AFEYH4WL3LTTPLHA33OM55RXWWZHHN54YMZYWQVIP72VO6IQPERN7263IEW3QS2MCHMDXKAHOAAAAAMWQXDYBRYAAAAAC66XIXT2PW7RNEX; TrackID=1HV4we0cR8GOSM8bCj15ATZWJRCkRxt_zC9NsV07_Rw_Hpjbg27hW-EOoflSpUAwRiCF64Gv0rs2svV4zySkqmD2IGl3twHr85Kl7cz6UD04; thor=BD7A0187F5FD056EAF98728DE029360B7A8A0541C8F77CB45F7B0147C979FC4B63263DC7060C0A0E024D245FED98C39E1965497BF8A52DD2B804B7A43DC8E62AAFCCC8375C8F1DAFD06EB149963A9C059E6A5A3794ED7091F3030BB35714F349356967BE1C5BCF323F577F074EBAB2972C8AFCEA4E692A2AB838FE098D171E77577D42ED90A56C77C4C41BED4EDEE82EBE2C73DD4186579625DD374F4EB7C081; light_key=AASBKE7rOxgWQziEhC_QY6yaHACwW-20kQ1Dcwl6grswVJjfXVytbgQhtenpZa5NPqZ_cGo0; pinId=0M8ocQ7iEWaVVjS3qk-0cbV9-x-f3wj7; pin=jd_4963d6886b0f2; unick=jd_136704tjkx; ceshi3.com=201; _tp=3CK%2BmBvk2ojeH2NRIdd5njR3CccoiLr28y8aBChbZuU%3D; _pst=jd_4963d6886b0f2; mail_times=1%2C1; cn=37; umc_count=1; source=PC; platform=pc; 3AB9D23F7A4B3C9B=S5H5QMW2J75PKHW4AFEYH4WL3LTTPLHA33OM55RXWWZHHN54YMZYWQVIP72VO6IQPERN7263IEW3QS2MCHMDXKAHOA; token=ed9f1fe0e7b340ca631c803a236bea46,3,970000; jsavif=1; __jda=181111935.1744804395425546198998.1744804395.1745683529.1746001166.13; __jdb=181111935.5.1744804395425546198998|13.1746001166; __jdc=181111935; flash=3_y4qAJzSRoYzofvat3DYn_5Klmf98wlnb_Rfv-1RxzFYoUuB2ue9FJsma9Kg_PtrLc8D00wMqYgHYuvmQMG1DLgI0h-0u-g9CY33yr4S_VQWoWVsGCMQFF-X_lYW455Y34pydLsc45q6G13Fs3KZzzRMoONO9VhAnXvXn5HyJW36pUpWl4XA0wV**; shshshfpb=BApXSqdjBhvNAkN6htfk9E7ZUVeoR0SCJB9MZLhp89xJ1MkNSh4O2; ipLoc-djd=19-1609-41653-0; sdtoken=AAbEsBpEIOVjqTAKCQtvQu17hnR3nn92tx0xIv0BMmdNKx5EOYU9EjhDQyt4HcTnNd06jjQqdJS-xPUQBOfQ5bB2jxUBytH6hCZmvHMLyZ5w55ox8Q80a0chH_sopEs"); - - headers.put("referer", "https://item.jd.com/"); - - headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36 Edg/91.0.864.37"); - } - - public static void main(String[] args) { - getJDProdComment("10111812014432"); - } - - public static void getJDProdComment(String prod_skuId) { - for (int i = 0; i < PAGE_RANGE; i++) { - long t = System.currentTimeMillis(); - String comments_url = "https://api.m.jd.com/?appid=item-v3&functionId=pc_club_productPageComments&client=pc&clientVersion=1.0.0"+"&t="+t+"9&loginType=3&uuid=181111935.16953916127351737253418.1695391613.1705847111.1705852812.35&productId=" + prod_skuId + "&score=0&sortType=5&page=" + i + "&pageSize=10&isShadowSku=0&fold=1&bbtf=&shield="; - try { - String response = sendRequest(comments_url); - if (response != null) { - JSONObject json_data = JSONObject.parseObject(response); - System.out.println(json_data); - if (i == 0) { - JSONObject productCommentSummary = json_data.getJSONObject("productCommentSummary"); - System.out.println(prod_skuId + " " + productCommentSummary); - JSONArray hotCommentTagStatistics = json_data.getJSONArray("hotCommentTagStatistics"); - System.out.println(prod_skuId + " " + hotCommentTagStatistics); - } - JSONArray comments = json_data.getJSONArray("comments"); - if (comments != null && comments.size() > 0) { - for (int j = 0; j < comments.size(); j++) { - JSONObject comment = comments.getJSONObject(j); - String prod_comment_guid = comment.getString("guid"); - String prod_comment_content = comment.getString("content"); - String prod_comment = comment.toString(); - JSONArray images = comment.getJSONArray("images"); - if (images != null && images.size() > 0) { - for (int k = 0; k < images.size(); k++) { - JSONObject image = images.getJSONObject(k); - String imgUrl = image.getString("imgUrl"); - logger.info("图片 {}", imgUrl); - - Thread.sleep(SLEEP_TIME); - } - } - System.out.println(prod_comment_guid + " " + prod_skuId + " " + prod_comment_content + " " + prod_comment); - } - } - } - Thread.sleep(SLEEP_TIME); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - private static String sendRequest(String urlStr) throws IOException { - URL url = new URL(urlStr); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestMethod("GET"); - for (Map.Entry entry : headers.entrySet()) { - connection.setRequestProperty(entry.getKey(), entry.getValue()); - } - int responseCode = connection.getResponseCode(); - if (responseCode == HttpURLConnection.HTTP_OK) { - BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); - StringBuilder response = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - response.append(line); - } - reader.close(); - return response.toString(); - } - return null; - } - - public static void downloadfile(String downloadurl) { - logger.info("downloadurl：{}", downloadurl); - } -} diff --git a/src/main/java/cn/van/business/util/ds/pl/cookie.txt b/src/main/java/cn/van/business/util/ds/pl/cookie.txt deleted file mode 100644 index 4e9290f..0000000 --- a/src/main/java/cn/van/business/util/ds/pl/cookie.txt +++ /dev/null @@ -1 +0,0 @@ - headers.put("cookie", "shshshfpa=0e6670b1-47be-d3a7-049d-3c9daa179153-1691582935; shshshfpx=0e6670b1-47be-d3a7-049d-3c9daa179153-1691582935; jcap_dvzw_fp=AMQNEJZc35A3KmQ7fLb-4FSnB2SbuA9auLr22ALve7Qyx9JarjpZCIHN8NIz4Bl_IB7RVdQ3LaO4xSrC0YsR3g==; __jdu=1744804395425546198998; whwswswws=; warehistory=\"10076121538879,10076121538879,10076121538879,\"; autoOpenApp_downCloseDate_autoOpenApp_autoPromptly=1744804996891_1; autoOpenApp_downCloseDate_jd_independent_coupon_openapp=1744948913486_1; pt_st=1_NWSLixpIv7w0rF2ENYK3Jf9y6xeCZi7by-s-cZ_os-zspi7V1bGhPcAZ3w02h0KlZuRNO4sgikpci-xjqrABXbJ-y524iEqm9uq2V0mkVH2RqLEesj0chjJtg-tcF6UBnUVWtVOqIaHczH4Ksux0Hlvk5oBhTziEhk7vR2wnLPciQ-W46Lld2YFdN_RldDpP-zVCY0pvepwAP8h8Ji4aNM-AueIvKCFFdqUYBq0K; autoOpenApp_downCloseDate_auto=1744948934067_1800000; unpl=JF8EAL1nNSttDxhRB0wEExJFTApdWwgASR4HO24GVgoKHFYNEgJMEBh7XlVdWRRKEh9vbxRUXlNLXA4ZASsSEHtdVV9eC04WAGdvNWRdWUpVBR8FGRUWe15Ublw4SxEGam4EUV9QT1EGHwYeFRBDVFxWVDhKJwNnYDVUXFhJXAAeBBwUGEleVG5tCXsUAWthBWRfWEpVDRsAExMTexw6Xl8KTREAa29IVFtdTl0EHgATFhVIWVBbWghDHgtnbjVVbVs; PCSYCityID=CN_440000_440400_0; __jdv=95931165%7Cbaidu%7C-%7Corganic%7Cnot%20set%7C1745683529792; areaId=19; 3AB9D23F7A4B3CSS=jdd03S5H5QMW2J75PKHW4AFEYH4WL3LTTPLHA33OM55RXWWZHHN54YMZYWQVIP72VO6IQPERN7263IEW3QS2MCHMDXKAHOAAAAAMWQXDYBRYAAAAAC66XIXT2PW7RNEX; TrackID=1HV4we0cR8GOSM8bCj15ATZWJRCkRxt_zC9NsV07_Rw_Hpjbg27hW-EOoflSpUAwRiCF64Gv0rs2svV4zySkqmD2IGl3twHr85Kl7cz6UD04; thor=BD7A0187F5FD056EAF98728DE029360B7A8A0541C8F77CB45F7B0147C979FC4B63263DC7060C0A0E024D245FED98C39E1965497BF8A52DD2B804B7A43DC8E62AAFCCC8375C8F1DAFD06EB149963A9C059E6A5A3794ED7091F3030BB35714F349356967BE1C5BCF323F577F074EBAB2972C8AFCEA4E692A2AB838FE098D171E77577D42ED90A56C77C4C41BED4EDEE82EBE2C73DD4186579625DD374F4EB7C081; light_key=AASBKE7rOxgWQziEhC_QY6yaHACwW-20kQ1Dcwl6grswVJjfXVytbgQhtenpZa5NPqZ_cGo0; pinId=0M8ocQ7iEWaVVjS3qk-0cbV9-x-f3wj7; pin=jd_4963d6886b0f2; unick=jd_136704tjkx; ceshi3.com=201; _tp=3CK%2BmBvk2ojeH2NRIdd5njR3CccoiLr28y8aBChbZuU%3D; _pst=jd_4963d6886b0f2; mail_times=1%2C1; cn=37; umc_count=1; source=PC; platform=pc; 3AB9D23F7A4B3C9B=S5H5QMW2J75PKHW4AFEYH4WL3LTTPLHA33OM55RXWWZHHN54YMZYWQVIP72VO6IQPERN7263IEW3QS2MCHMDXKAHOA; token=ed9f1fe0e7b340ca631c803a236bea46,3,970000; jsavif=1; __jda=181111935.1744804395425546198998.1744804395.1745683529.1746001166.13; __jdb=181111935.5.1744804395425546198998|13.1746001166; __jdc=181111935; flash=3_y4qAJzSRoYzofvat3DYn_5Klmf98wlnb_Rfv-1RxzFYoUuB2ue9FJsma9Kg_PtrLc8D00wMqYgHYuvmQMG1DLgI0h-0u-g9CY33yr4S_VQWoWVsGCMQFF-X_lYW455Y34pydLsc45q6G13Fs3KZzzRMoONO9VhAnXvXn5HyJW36pUpWl4XA0wV**; shshshfpb=BApXSqdjBhvNAkN6htfk9E7ZUVeoR0SCJB9MZLhp89xJ1MkNSh4O2; ipLoc-djd=19-1609-41653-0; sdtoken=AAbEsBpEIOVjqTAKCQtvQu17hnR3nn92tx0xIv0BMmdNKx5EOYU9EjhDQyt4HcTnNd06jjQqdJS-xPUQBOfQ5bB2jxUBytH6hCZmvHMLyZ5w55ox8Q80a0chH_sopEs");