This commit is contained in:
Leo
2026-02-04 23:40:21 +08:00
parent 7dce415c0b
commit 7e7250bc9c
2 changed files with 70 additions and 89 deletions

View File

@@ -57,6 +57,13 @@ public class TencentDocController extends BaseController {
/** Redis key前缀用于存储上次处理的最大行数 */ /** Redis key前缀用于存储上次处理的最大行数 */
private static final String LAST_PROCESSED_ROW_KEY_PREFIX = "tendoc:last_row:"; private static final String LAST_PROCESSED_ROW_KEY_PREFIX = "tendoc:last_row:";
/** 单次请求最大行数(腾讯文档 API range 限制,避免 invalid param range */
private static final int API_MAX_ROWS_PER_REQUEST = 200;
/** 回溯行数:下次起始行 = 本次扫描终点 + 1 - 回溯,用于覆盖物流变更或延后补填 */
private static final int BACKTRACK_ROWS = 80;
/** 最小前进行数:本次至少“扫过”这么多行,下次起始行至少 = 本次起始 + 最小前进(避免空行多时原地打转) */
private static final int MIN_ADVANCE_ROWS = 50;
/** Redis key前缀用于分布式锁 */ /** Redis key前缀用于分布式锁 */
private static final String TENCENT_DOC_LOCK_KEY = "tendoc:write:lock:"; private static final String TENCENT_DOC_LOCK_KEY = "tendoc:write:lock:";
@@ -999,48 +1006,36 @@ public class TencentDocController extends BaseController {
} }
} }
// 计算本次同步的起始行和结束行(带回溯机制 // 计算本次同步的起始行和结束行(完美推送回溯
// 确保 configStartRow 不为 null // 1. 单次请求不超过 API_MAX_ROWS_PER_REQUEST避免 range invalid
// 2. 下次起始 = 本次扫描终点 + 1 - BACKTRACK_ROWS兼顾物流变更与补填
// 3. 至少前进行数 MIN_ADVANCE_ROWS避免文档中间大量空行时原地打转
int effectiveStartRow = configStartRow != null ? configStartRow : (headerRow + 1); int effectiveStartRow = configStartRow != null ? configStartRow : (headerRow + 1);
int startRow; int startRow;
int endRow; int endRow;
if (forceStartRow != null) { if (forceStartRow != null) {
// 强制指定行(手动指定)
startRow = forceStartRow; startRow = forceStartRow;
endRow = startRow + 199; // 读取200行 endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
log.info("使用强制指定的起始行: {}, 结束行: {}", startRow, endRow); log.info("使用强制指定的起始行: {}, 结束行: {}(单批最多 {} 行)", startRow, endRow, API_MAX_ROWS_PER_REQUEST);
} else if (lastMaxRow != null && lastMaxRow >= effectiveStartRow) { } else if (lastMaxRow != null && lastMaxRow >= effectiveStartRow) {
// 计算回溯阈值:配置起始行 + 100 // 带回溯的起始:从 (上次最大行 + 1 - 回溯行数) 开始,保证覆盖物流变更与延后补填
int threshold = effectiveStartRow + 100; startRow = Math.max(effectiveStartRow, lastMaxRow + 1 - BACKTRACK_ROWS);
endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
if (lastMaxRow <= (effectiveStartRow + 199)) { log.info("上次最大行: {},回溯 {} 行后从第 {} 行开始,到第 {} 行(本批最多 {} 行)",
// 进度较小(一批数据内),重新从配置起始行开始 lastMaxRow, BACKTRACK_ROWS, startRow, endRow, API_MAX_ROWS_PER_REQUEST);
startRow = effectiveStartRow;
endRow = effectiveStartRow + 199; // 读取200行
log.info("上次进度 {} 较小(≤{}),重新从第 {} 行开始,到第 {} 行",
lastMaxRow, effectiveStartRow + 199, startRow, endRow);
} else if (lastMaxRow > threshold) {
// 进度较大(超过阈值)
// 起始行往回退100行但不能小于配置的起始行
startRow = Math.max(effectiveStartRow, lastMaxRow - 100);
// 结束行从上次位置往前读100行
endRow = lastMaxRow + 100;
log.info("上次进度 {} 较大(>{}回溯100行从第 {} 行开始往前读100行到第 {} 行(范围:{} 行)",
lastMaxRow, threshold, startRow, endRow, endRow - startRow + 1);
} else { } else {
// 进度在阈值范围内,重新从配置起始行开始
startRow = effectiveStartRow; startRow = effectiveStartRow;
endRow = effectiveStartRow + 199; // 读取200行 endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
log.info("上次进度 {} 在阈值范围内({}-{}),重新从第 {} 行开始,到第 {} 行", log.info("首次同步或配置已重置,从第 {} 行开始,到第 {} 行(单批最多 {} 行)", startRow, endRow, API_MAX_ROWS_PER_REQUEST);
lastMaxRow, effectiveStartRow + 200, threshold, startRow, endRow);
} }
} else {
// 首次同步或配置被重置,使用配置的起始行 // 严格限制单次 range 行数,防止 API 报 range invalid
startRow = effectiveStartRow; int rowCount = endRow - startRow + 1;
endRow = effectiveStartRow + 199; // 读取200行 if (rowCount > API_MAX_ROWS_PER_REQUEST) {
log.info("首次同步或配置已重置,从配置的起始行开始: {}, 到第 {} 行", startRow, endRow); endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
log.info("已截断结束行以符合 API 限制: endRow={}, 行数={}", endRow, API_MAX_ROWS_PER_REQUEST);
} }
log.info("开始填充物流链接 - 文件ID: {}, 工作表ID: {}, 起始行: {}, 结束行: {}, 上次最大行: {}", log.info("开始填充物流链接 - 文件ID: {}, 工作表ID: {}, 起始行: {}, 结束行: {}, 上次最大行: {}",
@@ -1157,26 +1152,31 @@ public class TencentDocController extends BaseController {
log.info("列位置识别完成 - 单号: {}, 物流单号: {}, 备注: {}, 是否安排: {}, 标记: {}, 下单电话: {}", log.info("列位置识别完成 - 单号: {}, 物流单号: {}, 备注: {}, 是否安排: {}, 标记: {}, 下单电话: {}",
orderNoColumn, logisticsLinkColumn, remarkColumn, arrangedColumn, markColumn, phoneColumn); orderNoColumn, logisticsLinkColumn, remarkColumn, arrangedColumn, markColumn, phoneColumn);
// 读取数据行(空数据行可能导致接口异常,对 endRow 尝试 -1、-10 重试以保证业务执行成功) // 读取数据行:严格限制单次行数(避免 range invalid失败时逐步缩小范围重试
int effectiveEndRow = endRow; int effectiveEndRow = Math.min(endRow, startRow + API_MAX_ROWS_PER_REQUEST - 1);
JSONObject sheetData = null; JSONObject sheetData = null;
int[] retryDecrements = new int[] { 0, 1, 10 }; int[] retryDecrements = new int[] { 0, 20, 50, 100 };
for (int decrement : retryDecrements) { for (int decrement : retryDecrements) {
int tryEndRow = Math.max(startRow, endRow - decrement); int tryEndRow = Math.max(startRow, effectiveEndRow - decrement);
int tryRowCount = tryEndRow - startRow + 1;
if (tryRowCount > API_MAX_ROWS_PER_REQUEST) {
tryEndRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
tryRowCount = API_MAX_ROWS_PER_REQUEST;
}
if (tryEndRow < startRow) { if (tryEndRow < startRow) {
continue; continue;
} }
String range = String.format("A%d:Z%d", startRow, tryEndRow); String range = String.format("A%d:Z%d", startRow, tryEndRow);
log.info("开始读取数据行 - 行号: {} ~ {}, range: {} (尝试 decrement={})", startRow, tryEndRow, range, decrement); log.info("开始读取数据行 - 行号: {} ~ {} (共 {} 行), range: {} (尝试 decrement={})", startRow, tryEndRow, tryRowCount, range, decrement);
try { try {
sheetData = tencentDocService.readSheetData(accessToken, fileId, sheetId, range); sheetData = tencentDocService.readSheetData(accessToken, fileId, sheetId, range);
if (sheetData != null) { if (sheetData != null) {
effectiveEndRow = tryEndRow; effectiveEndRow = tryEndRow;
log.info("数据行读取成功,响应: 有数据 (使用 endRow={})", effectiveEndRow); log.info("数据行读取成功 (使用 endRow={}, 行数={})", effectiveEndRow, tryRowCount);
break; break;
} }
} catch (Exception e) { } catch (Exception e) {
log.warn("读取数据行失败 (endRow={}, decrement={}),将重试更小范围: {}", tryEndRow, decrement, e.getMessage()); log.warn("读取数据行失败 (endRow={}, 行数={}, decrement={}),将重试更小范围: {}", tryEndRow, tryRowCount, decrement, e.getMessage());
if (decrement == retryDecrements[retryDecrements.length - 1]) { if (decrement == retryDecrements[retryDecrements.length - 1]) {
log.error("读取数据行多次重试后仍失败", e); log.error("读取数据行多次重试后仍失败", e);
return AjaxResult.error("读取数据行失败: " + e.getMessage()); return AjaxResult.error("读取数据行失败: " + e.getMessage());
@@ -1231,10 +1231,20 @@ public class TencentDocController extends BaseController {
log.warn("sheetData内容预览: {}", sheetData.toJSONString().substring(0, Math.min(500, sheetData.toJSONString().length()))); log.warn("sheetData内容预览: {}", sheetData.toJSONString().substring(0, Math.min(500, sheetData.toJSONString().length())));
} }
// 无数据时也推进进度,避免大量空行时原地打转;下次起始 = 本批终点 + 1 - 回溯
int emptyCurrentMax = effectiveEndRow;
int emptyNextStart = Math.max(effectiveStartRow,
Math.max(startRow + MIN_ADVANCE_ROWS, emptyCurrentMax + 1 - BACKTRACK_ROWS));
int emptyProgressToSave = Math.max(emptyCurrentMax, startRow + MIN_ADVANCE_ROWS + BACKTRACK_ROWS - 1);
redisCache.setCacheObject(redisKey, emptyProgressToSave, 30, TimeUnit.DAYS);
log.info("本批无数据,已推进进度: nextStartRow={}, Redis 进度={}", emptyNextStart, emptyProgressToSave);
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
result.put("startRow", startRow); result.put("startRow", startRow);
result.put("endRow", endRow); result.put("endRow", effectiveEndRow);
result.put("lastMaxRow", lastMaxRow); result.put("lastMaxRow", lastMaxRow);
result.put("nextStartRow", emptyNextStart);
result.put("currentMaxRow", emptyCurrentMax);
result.put("filledCount", 0); result.put("filledCount", 0);
result.put("skippedCount", 0); result.put("skippedCount", 0);
result.put("errorCount", 0); result.put("errorCount", 0);
@@ -1719,55 +1729,26 @@ public class TencentDocController extends BaseController {
successUpdates, skippedCount, errorCount, maxSuccessRow > 0 ? maxSuccessRow : ""); successUpdates, skippedCount, errorCount, maxSuccessRow > 0 ? maxSuccessRow : "");
} }
// 根据填充结果决定下次起始位置 // 完美推送回溯:根据本次扫描/写入结果决定下次起始行与 Redis 进度
// 使用实际成功写入的最大行号,如果没有成功写入,则使用 endRow // currentMaxRow = 本批实际涉及的最大行(有写入用最大成功行,否则用本批扫描终点)
int currentMaxRow = (maxSuccessRow > 0) ? maxSuccessRow : endRow; int currentMaxRow = (maxSuccessRow > 0) ? maxSuccessRow : effectiveEndRow;
int nextStartRow; // 下次起始 = 本批最大行 + 1 - 回溯行数,且不小于配置起始行;并保证至少前进行数(避免大量空行时原地打转)
String nextSyncHint; int nextStartRow = Math.max(effectiveStartRow,
Math.max(startRow + MIN_ADVANCE_ROWS, currentMaxRow + 1 - BACKTRACK_ROWS));
// 进度写入 Redis至少推进到 (startRow + 最小前进 + 回溯 - 1),保证下次不会重复扫同一段
int progressToSave = Math.max(currentMaxRow, startRow + MIN_ADVANCE_ROWS + BACKTRACK_ROWS - 1);
redisCache.setCacheObject(redisKey, progressToSave, 30, TimeUnit.DAYS);
// 使用 successUpdates实际成功写入数量而不是 filledCount准备写入数量 String nextSyncHint = String.format(
if (successUpdates > 0) { "下次将从第 %d 行开始(本批最大行: %d回溯 %d 行;已写入 %d 条,跳过 %d 条)",
// 有数据实际写入成功,说明当前范围有有效数据,继续往前推进 nextStartRow, currentMaxRow, BACKTRACK_ROWS, successUpdates, skippedCount);
// 下次从 currentMaxRow - 100 开始回溯100行防止遗漏
nextStartRow = Math.max(effectiveStartRow, currentMaxRow - 100);
nextSyncHint = String.format("下次将从第 %d 行开始(本次实际写入成功 %d 条,最大行号: %d回溯100行防止遗漏",
nextStartRow, successUpdates, currentMaxRow);
// 记录进度到实际成功写入的最大行号,持续往前
redisCache.setCacheObject(redisKey, currentMaxRow, 30, TimeUnit.DAYS);
log.info("========== 更新Redis进度 =========="); log.info("========== 更新Redis进度 ==========");
log.info("次实际写入成功 {} 条,最大成功行号: {},更新进度到第 {} 行,下次从第 {} 行开始", log.info("批范围: {}~{},实际最大行: {},下次起始: {}Redis 进度: {}", startRow, effectiveEndRow, currentMaxRow, nextStartRow, progressToSave);
successUpdates, maxSuccessRow, currentMaxRow, nextStartRow);
log.info("Redis Key: {}", redisKey); log.info("Redis Key: {}", redisKey);
} else {
// 没有数据实际写入成功,说明这个范围都是空行或已处理的数据
if (lastMaxRow != null && currentMaxRow > (lastMaxRow + 300)) {
// 如果跳跃太大(>300行可能已经超出数据区域回到配置起始行重新扫描
nextStartRow = effectiveStartRow;
nextSyncHint = String.format("下次将从第 %d 行重新开始(本次范围 %d-%d 无实际写入,重新扫描)",
nextStartRow, startRow, endRow);
// 不更新 Redis 进度,保持原有进度
log.info("本次无实际写入,且跳跃过大({} > {} + 300下次将从配置起始行 {} 重新开始",
currentMaxRow, lastMaxRow, effectiveStartRow);
} else {
// 跳跃不大,继续往前推进
nextStartRow = Math.max(effectiveStartRow, currentMaxRow - 100);
nextSyncHint = String.format("下次将从第 %d 行开始(本次范围 %d-%d 无实际写入,继续往前)",
nextStartRow, startRow, endRow);
// 记录进度到 endRow即使没有写入也要记录扫描进度
redisCache.setCacheObject(redisKey, currentMaxRow, 30, TimeUnit.DAYS);
log.info("========== 更新Redis进度 ==========");
log.info("本次无实际写入,更新扫描进度到第 {} 行,下次从第 {} 行继续",
currentMaxRow, nextStartRow);
log.info("Redis Key: {}", redisKey);
}
}
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
result.put("startRow", startRow); result.put("startRow", startRow);
result.put("endRow", endRow); result.put("endRow", effectiveEndRow);
result.put("currentMaxRow", currentMaxRow); result.put("currentMaxRow", currentMaxRow);
result.put("lastMaxRow", lastMaxRow); result.put("lastMaxRow", lastMaxRow);
result.put("nextStartRow", nextStartRow); result.put("nextStartRow", nextStartRow);
@@ -1782,12 +1763,12 @@ public class TencentDocController extends BaseController {
String message = String.format( String message = String.format(
"✓ 同步完成:成功填充 %d 条,跳过 %d 条,错误 %d 条\n" + "✓ 同步完成:成功填充 %d 条,跳过 %d 条,错误 %d 条\n" +
" 本次范围:第 %d-%d 行\n" + " 本次范围:第 %d-%d 行\n" +
" 当前进度:第 %d 行\n" + " 当前进度:第 %d 行,下次从第 %d 行开始(回溯 %d 行)\n" +
" 防重推送:%s\n" + " 防重推送:%s\n" +
" %s", " %s",
filledCount, skippedCount, errorCount, filledCount, skippedCount, errorCount,
startRow, currentMaxRow, startRow, effectiveEndRow,
currentMaxRow, currentMaxRow, nextStartRow, BACKTRACK_ROWS,
skipPushedOrders ? "已启用(跳过已推送订单)" : "已禁用", skipPushedOrders ? "已启用(跳过已推送订单)" : "已禁用",
nextSyncHint nextSyncHint
); );

View File

@@ -292,7 +292,7 @@ public class TencentDocDelayedPushServiceImpl implements ITencentDocDelayedPushS
"AUTO", "AUTO",
"DELAYED_TIMER", "DELAYED_TIMER",
startRow, startRow,
startRow + 199 // 暂定范围 startRow + 199 // 与 Controller API_MAX_ROWS_PER_REQUEST=200 一致单批最多200行
); );
log.info("✓ 创建批量推送记录批次ID: {}", batchId); log.info("✓ 创建批量推送记录批次ID: {}", batchId);