1
This commit is contained in:
@@ -57,6 +57,13 @@ public class TencentDocController extends BaseController {
|
|||||||
/** Redis key前缀,用于存储上次处理的最大行数 */
|
/** Redis key前缀,用于存储上次处理的最大行数 */
|
||||||
private static final String LAST_PROCESSED_ROW_KEY_PREFIX = "tendoc:last_row:";
|
private static final String LAST_PROCESSED_ROW_KEY_PREFIX = "tendoc:last_row:";
|
||||||
|
|
||||||
|
/** 单次请求最大行数(腾讯文档 API range 限制,避免 invalid param range) */
|
||||||
|
private static final int API_MAX_ROWS_PER_REQUEST = 200;
|
||||||
|
/** 回溯行数:下次起始行 = 本次扫描终点 + 1 - 回溯,用于覆盖物流变更或延后补填 */
|
||||||
|
private static final int BACKTRACK_ROWS = 80;
|
||||||
|
/** 最小前进行数:本次至少“扫过”这么多行,下次起始行至少 = 本次起始 + 最小前进(避免空行多时原地打转) */
|
||||||
|
private static final int MIN_ADVANCE_ROWS = 50;
|
||||||
|
|
||||||
/** Redis key前缀,用于分布式锁 */
|
/** Redis key前缀,用于分布式锁 */
|
||||||
private static final String TENCENT_DOC_LOCK_KEY = "tendoc:write:lock:";
|
private static final String TENCENT_DOC_LOCK_KEY = "tendoc:write:lock:";
|
||||||
|
|
||||||
@@ -999,48 +1006,36 @@ public class TencentDocController extends BaseController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 计算本次同步的起始行和结束行(带回溯机制)
|
// 计算本次同步的起始行和结束行(完美推送回溯)
|
||||||
// 确保 configStartRow 不为 null
|
// 1. 单次请求不超过 API_MAX_ROWS_PER_REQUEST,避免 range invalid
|
||||||
|
// 2. 下次起始 = 本次扫描终点 + 1 - BACKTRACK_ROWS,兼顾物流变更与补填
|
||||||
|
// 3. 至少前进行数 MIN_ADVANCE_ROWS,避免文档中间大量空行时原地打转
|
||||||
int effectiveStartRow = configStartRow != null ? configStartRow : (headerRow + 1);
|
int effectiveStartRow = configStartRow != null ? configStartRow : (headerRow + 1);
|
||||||
|
|
||||||
int startRow;
|
int startRow;
|
||||||
int endRow;
|
int endRow;
|
||||||
|
|
||||||
if (forceStartRow != null) {
|
if (forceStartRow != null) {
|
||||||
// 强制指定行(手动指定)
|
|
||||||
startRow = forceStartRow;
|
startRow = forceStartRow;
|
||||||
endRow = startRow + 199; // 读取200行
|
endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
|
||||||
log.info("使用强制指定的起始行: {}, 结束行: {}", startRow, endRow);
|
log.info("使用强制指定的起始行: {}, 结束行: {}(单批最多 {} 行)", startRow, endRow, API_MAX_ROWS_PER_REQUEST);
|
||||||
} else if (lastMaxRow != null && lastMaxRow >= effectiveStartRow) {
|
} else if (lastMaxRow != null && lastMaxRow >= effectiveStartRow) {
|
||||||
// 计算回溯阈值:配置起始行 + 100
|
// 带回溯的起始:从 (上次最大行 + 1 - 回溯行数) 开始,保证覆盖物流变更与延后补填
|
||||||
int threshold = effectiveStartRow + 100;
|
startRow = Math.max(effectiveStartRow, lastMaxRow + 1 - BACKTRACK_ROWS);
|
||||||
|
endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
|
||||||
if (lastMaxRow <= (effectiveStartRow + 199)) {
|
log.info("上次最大行: {},回溯 {} 行后从第 {} 行开始,到第 {} 行(本批最多 {} 行)",
|
||||||
// 进度较小(一批数据内),重新从配置起始行开始
|
lastMaxRow, BACKTRACK_ROWS, startRow, endRow, API_MAX_ROWS_PER_REQUEST);
|
||||||
startRow = effectiveStartRow;
|
|
||||||
endRow = effectiveStartRow + 199; // 读取200行
|
|
||||||
log.info("上次进度 {} 较小(≤{}),重新从第 {} 行开始,到第 {} 行",
|
|
||||||
lastMaxRow, effectiveStartRow + 199, startRow, endRow);
|
|
||||||
} else if (lastMaxRow > threshold) {
|
|
||||||
// 进度较大(超过阈值)
|
|
||||||
// 起始行:往回退100行,但不能小于配置的起始行
|
|
||||||
startRow = Math.max(effectiveStartRow, lastMaxRow - 100);
|
|
||||||
// 结束行:从上次位置往前读100行
|
|
||||||
endRow = lastMaxRow + 100;
|
|
||||||
log.info("上次进度 {} 较大(>{}),回溯100行从第 {} 行开始,往前读100行到第 {} 行(范围:{} 行)",
|
|
||||||
lastMaxRow, threshold, startRow, endRow, endRow - startRow + 1);
|
|
||||||
} else {
|
} else {
|
||||||
// 进度在阈值范围内,重新从配置起始行开始
|
|
||||||
startRow = effectiveStartRow;
|
startRow = effectiveStartRow;
|
||||||
endRow = effectiveStartRow + 199; // 读取200行
|
endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
|
||||||
log.info("上次进度 {} 在阈值范围内({}-{}),重新从第 {} 行开始,到第 {} 行",
|
log.info("首次同步或配置已重置,从第 {} 行开始,到第 {} 行(单批最多 {} 行)", startRow, endRow, API_MAX_ROWS_PER_REQUEST);
|
||||||
lastMaxRow, effectiveStartRow + 200, threshold, startRow, endRow);
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// 首次同步或配置被重置,使用配置的起始行
|
// 严格限制单次 range 行数,防止 API 报 range invalid
|
||||||
startRow = effectiveStartRow;
|
int rowCount = endRow - startRow + 1;
|
||||||
endRow = effectiveStartRow + 199; // 读取200行
|
if (rowCount > API_MAX_ROWS_PER_REQUEST) {
|
||||||
log.info("首次同步或配置已重置,从配置的起始行开始: {}, 到第 {} 行", startRow, endRow);
|
endRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
|
||||||
|
log.info("已截断结束行以符合 API 限制: endRow={}, 行数={}", endRow, API_MAX_ROWS_PER_REQUEST);
|
||||||
}
|
}
|
||||||
|
|
||||||
log.info("开始填充物流链接 - 文件ID: {}, 工作表ID: {}, 起始行: {}, 结束行: {}, 上次最大行: {}",
|
log.info("开始填充物流链接 - 文件ID: {}, 工作表ID: {}, 起始行: {}, 结束行: {}, 上次最大行: {}",
|
||||||
@@ -1157,26 +1152,31 @@ public class TencentDocController extends BaseController {
|
|||||||
log.info("列位置识别完成 - 单号: {}, 物流单号: {}, 备注: {}, 是否安排: {}, 标记: {}, 下单电话: {}",
|
log.info("列位置识别完成 - 单号: {}, 物流单号: {}, 备注: {}, 是否安排: {}, 标记: {}, 下单电话: {}",
|
||||||
orderNoColumn, logisticsLinkColumn, remarkColumn, arrangedColumn, markColumn, phoneColumn);
|
orderNoColumn, logisticsLinkColumn, remarkColumn, arrangedColumn, markColumn, phoneColumn);
|
||||||
|
|
||||||
// 读取数据行(空数据行可能导致接口异常,对 endRow 尝试 -1、-10 重试以保证业务执行成功)
|
// 读取数据行:严格限制单次行数(避免 range invalid),失败时逐步缩小范围重试
|
||||||
int effectiveEndRow = endRow;
|
int effectiveEndRow = Math.min(endRow, startRow + API_MAX_ROWS_PER_REQUEST - 1);
|
||||||
JSONObject sheetData = null;
|
JSONObject sheetData = null;
|
||||||
int[] retryDecrements = new int[] { 0, 1, 10 };
|
int[] retryDecrements = new int[] { 0, 20, 50, 100 };
|
||||||
for (int decrement : retryDecrements) {
|
for (int decrement : retryDecrements) {
|
||||||
int tryEndRow = Math.max(startRow, endRow - decrement);
|
int tryEndRow = Math.max(startRow, effectiveEndRow - decrement);
|
||||||
|
int tryRowCount = tryEndRow - startRow + 1;
|
||||||
|
if (tryRowCount > API_MAX_ROWS_PER_REQUEST) {
|
||||||
|
tryEndRow = startRow + API_MAX_ROWS_PER_REQUEST - 1;
|
||||||
|
tryRowCount = API_MAX_ROWS_PER_REQUEST;
|
||||||
|
}
|
||||||
if (tryEndRow < startRow) {
|
if (tryEndRow < startRow) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
String range = String.format("A%d:Z%d", startRow, tryEndRow);
|
String range = String.format("A%d:Z%d", startRow, tryEndRow);
|
||||||
log.info("开始读取数据行 - 行号: {} ~ {}, range: {} (尝试 decrement={})", startRow, tryEndRow, range, decrement);
|
log.info("开始读取数据行 - 行号: {} ~ {} (共 {} 行), range: {} (尝试 decrement={})", startRow, tryEndRow, tryRowCount, range, decrement);
|
||||||
try {
|
try {
|
||||||
sheetData = tencentDocService.readSheetData(accessToken, fileId, sheetId, range);
|
sheetData = tencentDocService.readSheetData(accessToken, fileId, sheetId, range);
|
||||||
if (sheetData != null) {
|
if (sheetData != null) {
|
||||||
effectiveEndRow = tryEndRow;
|
effectiveEndRow = tryEndRow;
|
||||||
log.info("数据行读取成功,响应: 有数据 (使用 endRow={})", effectiveEndRow);
|
log.info("数据行读取成功 (使用 endRow={}, 行数={})", effectiveEndRow, tryRowCount);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.warn("读取数据行失败 (endRow={}, decrement={}),将重试更小范围: {}", tryEndRow, decrement, e.getMessage());
|
log.warn("读取数据行失败 (endRow={}, 行数={}, decrement={}),将重试更小范围: {}", tryEndRow, tryRowCount, decrement, e.getMessage());
|
||||||
if (decrement == retryDecrements[retryDecrements.length - 1]) {
|
if (decrement == retryDecrements[retryDecrements.length - 1]) {
|
||||||
log.error("读取数据行多次重试后仍失败", e);
|
log.error("读取数据行多次重试后仍失败", e);
|
||||||
return AjaxResult.error("读取数据行失败: " + e.getMessage());
|
return AjaxResult.error("读取数据行失败: " + e.getMessage());
|
||||||
@@ -1231,10 +1231,20 @@ public class TencentDocController extends BaseController {
|
|||||||
log.warn("sheetData内容预览: {}", sheetData.toJSONString().substring(0, Math.min(500, sheetData.toJSONString().length())));
|
log.warn("sheetData内容预览: {}", sheetData.toJSONString().substring(0, Math.min(500, sheetData.toJSONString().length())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 无数据时也推进进度,避免大量空行时原地打转;下次起始 = 本批终点 + 1 - 回溯
|
||||||
|
int emptyCurrentMax = effectiveEndRow;
|
||||||
|
int emptyNextStart = Math.max(effectiveStartRow,
|
||||||
|
Math.max(startRow + MIN_ADVANCE_ROWS, emptyCurrentMax + 1 - BACKTRACK_ROWS));
|
||||||
|
int emptyProgressToSave = Math.max(emptyCurrentMax, startRow + MIN_ADVANCE_ROWS + BACKTRACK_ROWS - 1);
|
||||||
|
redisCache.setCacheObject(redisKey, emptyProgressToSave, 30, TimeUnit.DAYS);
|
||||||
|
log.info("本批无数据,已推进进度: nextStartRow={}, Redis 进度={}", emptyNextStart, emptyProgressToSave);
|
||||||
|
|
||||||
JSONObject result = new JSONObject();
|
JSONObject result = new JSONObject();
|
||||||
result.put("startRow", startRow);
|
result.put("startRow", startRow);
|
||||||
result.put("endRow", endRow);
|
result.put("endRow", effectiveEndRow);
|
||||||
result.put("lastMaxRow", lastMaxRow);
|
result.put("lastMaxRow", lastMaxRow);
|
||||||
|
result.put("nextStartRow", emptyNextStart);
|
||||||
|
result.put("currentMaxRow", emptyCurrentMax);
|
||||||
result.put("filledCount", 0);
|
result.put("filledCount", 0);
|
||||||
result.put("skippedCount", 0);
|
result.put("skippedCount", 0);
|
||||||
result.put("errorCount", 0);
|
result.put("errorCount", 0);
|
||||||
@@ -1719,55 +1729,26 @@ public class TencentDocController extends BaseController {
|
|||||||
successUpdates, skippedCount, errorCount, maxSuccessRow > 0 ? maxSuccessRow : "无");
|
successUpdates, skippedCount, errorCount, maxSuccessRow > 0 ? maxSuccessRow : "无");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 根据填充结果决定下次的起始位置
|
// 完美推送回溯:根据本次扫描/写入结果决定下次起始行与 Redis 进度
|
||||||
// 使用实际成功写入的最大行号,如果没有成功写入,则使用 endRow
|
// currentMaxRow = 本批实际涉及的最大行(有写入用最大成功行,否则用本批扫描终点)
|
||||||
int currentMaxRow = (maxSuccessRow > 0) ? maxSuccessRow : endRow;
|
int currentMaxRow = (maxSuccessRow > 0) ? maxSuccessRow : effectiveEndRow;
|
||||||
int nextStartRow;
|
// 下次起始 = 本批最大行 + 1 - 回溯行数,且不小于配置起始行;并保证至少前进行数(避免大量空行时原地打转)
|
||||||
String nextSyncHint;
|
int nextStartRow = Math.max(effectiveStartRow,
|
||||||
|
Math.max(startRow + MIN_ADVANCE_ROWS, currentMaxRow + 1 - BACKTRACK_ROWS));
|
||||||
|
// 进度写入 Redis:至少推进到 (startRow + 最小前进 + 回溯 - 1),保证下次不会重复扫同一段
|
||||||
|
int progressToSave = Math.max(currentMaxRow, startRow + MIN_ADVANCE_ROWS + BACKTRACK_ROWS - 1);
|
||||||
|
redisCache.setCacheObject(redisKey, progressToSave, 30, TimeUnit.DAYS);
|
||||||
|
|
||||||
// 使用 successUpdates(实际成功写入数量)而不是 filledCount(准备写入数量)
|
String nextSyncHint = String.format(
|
||||||
if (successUpdates > 0) {
|
"下次将从第 %d 行开始(本批最大行: %d,回溯 %d 行;已写入 %d 条,跳过 %d 条)",
|
||||||
// 有数据实际写入成功,说明当前范围有有效数据,继续往前推进
|
nextStartRow, currentMaxRow, BACKTRACK_ROWS, successUpdates, skippedCount);
|
||||||
// 下次从 currentMaxRow - 100 开始(回溯100行,防止遗漏)
|
|
||||||
nextStartRow = Math.max(effectiveStartRow, currentMaxRow - 100);
|
|
||||||
nextSyncHint = String.format("下次将从第 %d 行开始(本次实际写入成功 %d 条,最大行号: %d,回溯100行防止遗漏)",
|
|
||||||
nextStartRow, successUpdates, currentMaxRow);
|
|
||||||
|
|
||||||
// 记录进度到实际成功写入的最大行号,持续往前
|
|
||||||
redisCache.setCacheObject(redisKey, currentMaxRow, 30, TimeUnit.DAYS);
|
|
||||||
log.info("========== 更新Redis进度 ==========");
|
log.info("========== 更新Redis进度 ==========");
|
||||||
log.info("本次实际写入成功 {} 条,最大成功行号: {},更新进度到第 {} 行,下次从第 {} 行开始",
|
log.info("本批范围: {}~{},实际最大行: {},下次起始: {},Redis 进度: {}", startRow, effectiveEndRow, currentMaxRow, nextStartRow, progressToSave);
|
||||||
successUpdates, maxSuccessRow, currentMaxRow, nextStartRow);
|
|
||||||
log.info("Redis Key: {}", redisKey);
|
log.info("Redis Key: {}", redisKey);
|
||||||
} else {
|
|
||||||
// 没有数据实际写入成功,说明这个范围都是空行或已处理的数据
|
|
||||||
if (lastMaxRow != null && currentMaxRow > (lastMaxRow + 300)) {
|
|
||||||
// 如果跳跃太大(>300行),可能已经超出数据区域,回到配置起始行重新扫描
|
|
||||||
nextStartRow = effectiveStartRow;
|
|
||||||
nextSyncHint = String.format("下次将从第 %d 行重新开始(本次范围 %d-%d 无实际写入,重新扫描)",
|
|
||||||
nextStartRow, startRow, endRow);
|
|
||||||
|
|
||||||
// 不更新 Redis 进度,保持原有进度
|
|
||||||
log.info("本次无实际写入,且跳跃过大({} > {} + 300),下次将从配置起始行 {} 重新开始",
|
|
||||||
currentMaxRow, lastMaxRow, effectiveStartRow);
|
|
||||||
} else {
|
|
||||||
// 跳跃不大,继续往前推进
|
|
||||||
nextStartRow = Math.max(effectiveStartRow, currentMaxRow - 100);
|
|
||||||
nextSyncHint = String.format("下次将从第 %d 行开始(本次范围 %d-%d 无实际写入,继续往前)",
|
|
||||||
nextStartRow, startRow, endRow);
|
|
||||||
|
|
||||||
// 记录进度到 endRow(即使没有写入,也要记录扫描进度)
|
|
||||||
redisCache.setCacheObject(redisKey, currentMaxRow, 30, TimeUnit.DAYS);
|
|
||||||
log.info("========== 更新Redis进度 ==========");
|
|
||||||
log.info("本次无实际写入,更新扫描进度到第 {} 行,下次从第 {} 行继续",
|
|
||||||
currentMaxRow, nextStartRow);
|
|
||||||
log.info("Redis Key: {}", redisKey);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONObject result = new JSONObject();
|
JSONObject result = new JSONObject();
|
||||||
result.put("startRow", startRow);
|
result.put("startRow", startRow);
|
||||||
result.put("endRow", endRow);
|
result.put("endRow", effectiveEndRow);
|
||||||
result.put("currentMaxRow", currentMaxRow);
|
result.put("currentMaxRow", currentMaxRow);
|
||||||
result.put("lastMaxRow", lastMaxRow);
|
result.put("lastMaxRow", lastMaxRow);
|
||||||
result.put("nextStartRow", nextStartRow);
|
result.put("nextStartRow", nextStartRow);
|
||||||
@@ -1782,12 +1763,12 @@ public class TencentDocController extends BaseController {
|
|||||||
String message = String.format(
|
String message = String.format(
|
||||||
"✓ 同步完成:成功填充 %d 条,跳过 %d 条,错误 %d 条\n" +
|
"✓ 同步完成:成功填充 %d 条,跳过 %d 条,错误 %d 条\n" +
|
||||||
" 本次范围:第 %d-%d 行\n" +
|
" 本次范围:第 %d-%d 行\n" +
|
||||||
" 当前进度:第 %d 行\n" +
|
" 当前进度:第 %d 行,下次从第 %d 行开始(回溯 %d 行)\n" +
|
||||||
" 防重推送:%s\n" +
|
" 防重推送:%s\n" +
|
||||||
" %s",
|
" %s",
|
||||||
filledCount, skippedCount, errorCount,
|
filledCount, skippedCount, errorCount,
|
||||||
startRow, currentMaxRow,
|
startRow, effectiveEndRow,
|
||||||
currentMaxRow,
|
currentMaxRow, nextStartRow, BACKTRACK_ROWS,
|
||||||
skipPushedOrders ? "已启用(跳过已推送订单)" : "已禁用",
|
skipPushedOrders ? "已启用(跳过已推送订单)" : "已禁用",
|
||||||
nextSyncHint
|
nextSyncHint
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -292,7 +292,7 @@ public class TencentDocDelayedPushServiceImpl implements ITencentDocDelayedPushS
|
|||||||
"AUTO",
|
"AUTO",
|
||||||
"DELAYED_TIMER",
|
"DELAYED_TIMER",
|
||||||
startRow,
|
startRow,
|
||||||
startRow + 199 // 暂定范围
|
startRow + 199 // 与 Controller API_MAX_ROWS_PER_REQUEST=200 一致(单批最多200行)
|
||||||
);
|
);
|
||||||
log.info("✓ 创建批量推送记录,批次ID: {}", batchId);
|
log.info("✓ 创建批量推送记录,批次ID: {}", batchId);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user