lihongjie/coal
1
0
Fork 0
mirror of https://codeup.aliyun.com/64f7d6b8ce01efaafef1e678/coal/coal.git synced 2026-07-25 07:07:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(Mtjywpm): add retry mechanism for API data fetching with proxy IP support

Browse Source
This commit is contained in:
lihongjie0209
2025-10-19 17:56:23 +08:00
parent a15709acaf
commit bef8134dee
1 changed files with 121 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
src/main/java/cn/lihongjie/coal/mtjywpm/service/MtjywpmService.java
View File

@@ -59,6 +59,8 @@ public class MtjywpmService extends BaseService<MtjywpmEntity, MtjywpmRepository
@Autowired private CommonMapper commonMapper;
private static final String API_URL = "https://www.ctctc.cn/search_list_gg.jspx";
private static final int PAGE_SIZE = 50;
private static final int FETCH_API_MAX_RETRY = 3;
private static final long FETCH_API_RETRY_DELAY_MS = 3000L;
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
@Autowired private MtjywpmRepository repository;
@@ -321,105 +323,133 @@ public class MtjywpmService extends BaseService<MtjywpmEntity, MtjywpmRepository
/** 从API获取数据，支持按交易单号查询 */
private List<Map<String, Object>> fetchDataFromApi(
int page, String tradercode, Integer pageSize) {
ProxyIp currentProxy = null;
RestTemplate currentRestTemplate = restTemplate;
try {
// 获取可用的代理IP
Optional<ProxyIp> proxyOptional = proxyIpService.getAvailableProxyIp();
if (proxyOptional.isPresent()) {
currentProxy = proxyOptional.get();
currentRestTemplate = createProxyRestTemplate(currentProxy);
log.info("使用代理IP进行API请求：{}:{}", currentProxy.getIp(), currentProxy.getPort());
} else {
log.warn("未获取到可用代理IP，使用直接连接");
}
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
headers.set("Accept", "application/json, text/javascript, */*; q=0.01");
headers.set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7");
headers.set("Cache-Control", "no-cache");
headers.set("Connection", "keep-alive");
headers.set("Origin", "https://www.ctctc.cn");
headers.set("Pragma", "no-cache");
headers.set("Referer", "https://www.ctctc.cn/pmsearch.jspx");
headers.set("Sec-Fetch-Dest", "empty");
headers.set("Sec-Fetch-Mode", "cors");
headers.set("Sec-Fetch-Site", "same-origin");
headers.set(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36");
headers.set("X-Requested-With", "XMLHttpRequest");
headers.set(
"sec-ch-ua",
"\"Chromium\";v=\"140\", \"Not=A?Brand\";v=\"24\", \"Google Chrome\";v=\"140\"");
headers.set("sec-ch-ua-mobile", "?0");
headers.set("sec-ch-ua-platform", "\"Windows\"");
headers.set("Cookie", "zgtymtjyzxex2ectctcx2ecnx3a80=1756670144.20480");
for (int attempt = 1; attempt <= FETCH_API_MAX_RETRY; attempt++) {
ProxyIp currentProxy = null;
RestTemplate currentRestTemplate = restTemplate;
// 设置请求参数
MultiValueMap<String, String> params = new LinkedMultiValueMap<>();
params.add("page", String.valueOf(page));
params.add("limit", String.valueOf(pageSize != null ? pageSize : PAGE_SIZE));
params.add("trademodeid", "3");
params.add("postatus", "-1");
params.add("commodityname", "");
// 如果指定了交易单号，添加到查询参数中
if (tradercode != null && !tradercode.trim().isEmpty()) {
params.add("tradercode", tradercode.trim());
}
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(params, headers);
// 发送请求
ResponseEntity<String> response =
currentRestTemplate.postForEntity(API_URL, request, String.class);
if (response.getStatusCode() == HttpStatus.OK) {
// 标记代理IP使用成功
if (currentProxy != null) {
proxyIpService.markProxyAsUsed(currentProxy.getIp(), currentProxy.getPort());
log.debug("代理IP {}:{} 请求成功", currentProxy.getIp(), currentProxy.getPort());
try {
// 获取可用的代理IP
Optional<ProxyIp> proxyOptional = proxyIpService.getAvailableProxyIp();
if (proxyOptional.isPresent()) {
currentProxy = proxyOptional.get();
currentRestTemplate = createProxyRestTemplate(currentProxy);
log.info(
"使用代理IP进行API请求：{}:{}，第{}次尝试",
currentProxy.getIp(),
currentProxy.getPort(),
attempt);
} else {
log.warn("未获取到可用代理IP，使用直接连接，第{}次尝试", attempt);
}
String responseBody = response.getBody();
Map<String, Object> responseMap =
objectMapper.readValue(
responseBody, new TypeReference<Map<String, Object>>() {});
Integer code = (Integer) responseMap.get("code");
if (code != null && code == 0) {
Object dataObj = responseMap.get("data");
if (dataObj instanceof List) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> dataList = (List<Map<String, Object>>) dataObj;
return dataList;
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
headers.set("Accept", "application/json, text/javascript, */*; q=0.01");
headers.set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7");
headers.set("Cache-Control", "no-cache");
headers.set("Connection", "keep-alive");
headers.set("Origin", "https://www.ctctc.cn");
headers.set("Pragma", "no-cache");
headers.set("Referer", "https://www.ctctc.cn/pmsearch.jspx");
headers.set("Sec-Fetch-Dest", "empty");
headers.set("Sec-Fetch-Mode", "cors");
headers.set("Sec-Fetch-Site", "same-origin");
headers.set(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36");
headers.set("X-Requested-With", "XMLHttpRequest");
headers.set(
"sec-ch-ua",
"\"Chromium\";v=\"140\", \"Not=A?Brand\";v=\"24\", \"Google Chrome\";v=\"140\"");
headers.set("sec-ch-ua-mobile", "?0");
headers.set("sec-ch-ua-platform", "\"Windows\"");
headers.set("Cookie", "zgtymtjyzxex2ectctcx2ecnx3a80=1756670144.20480");
// 设置请求参数
MultiValueMap<String, String> params = new LinkedMultiValueMap<>();
params.add("page", String.valueOf(page));
params.add("limit", String.valueOf(pageSize != null ? pageSize : PAGE_SIZE));
params.add("trademodeid", "3");
params.add("postatus", "-1");
params.add("commodityname", "");
// 如果指定了交易单号，添加到查询参数中
if (tradercode != null && !tradercode.trim().isEmpty()) {
params.add("tradercode", tradercode.trim());
}
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(params, headers);
// 发送请求
ResponseEntity<String> response =
currentRestTemplate.postForEntity(API_URL, request, String.class);
if (response.getStatusCode() == HttpStatus.OK) {
// 标记代理IP使用成功
if (currentProxy != null) {
proxyIpService.markProxyAsUsed(currentProxy.getIp(), currentProxy.getPort());
log.debug("代理IP {}:{} 请求成功", currentProxy.getIp(), currentProxy.getPort());
}
String responseBody = response.getBody();
Map<String, Object> responseMap =
objectMapper.readValue(
responseBody, new TypeReference<Map<String, Object>>() {});
Integer code = (Integer) responseMap.get("code");
if (code != null && code == 0) {
Object dataObj = responseMap.get("data");
if (dataObj instanceof List) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> dataList = (List<Map<String, Object>>) dataObj;
return dataList;
}
}
} else {
log.warn("API请求返回非200状态码：{}", response.getStatusCode());
// 如果是代理相关的错误，标记代理IP为不可用
if (currentProxy != null
&& (response.getStatusCode().is4xxClientError()
|| response.getStatusCode().is5xxServerError())) {
proxyIpService.markProxyAsUnavailable(
currentProxy.getIp(), currentProxy.getPort());
log.warn(
"代理IP {}:{} 可能不可用，状态码：{}",
currentProxy.getIp(),
currentProxy.getPort(),
response.getStatusCode());
}
}
} else {
log.warn("API请求返回非200状态码：{}", response.getStatusCode());
// 如果是代理相关的错误，标记代理IP为不可用
if (currentProxy != null && (response.getStatusCode().is4xxClientError() || response.getStatusCode().is5xxServerError())) {
proxyIpService.markProxyAsUnavailable(currentProxy.getIp(), currentProxy.getPort());
log.warn("代理IP {}:{} 可能不可用，状态码：{}", currentProxy.getIp(), currentProxy.getPort(), response.getStatusCode());
return null;
} catch (Exception e) {
log.error("从API获取数据失败，页码：{}，第{}次尝试", page, attempt, e);
// 如果使用了代理IP且请求失败，标记为不可用
if (currentProxy != null) {
proxyIpService.markProxyAsUnavailable(
currentProxy.getIp(), currentProxy.getPort());
log.warn(
"代理IP {}:{} 请求失败，标记为不可用：{}",
currentProxy.getIp(),
currentProxy.getPort(),
e.getMessage());
}
if (attempt < FETCH_API_MAX_RETRY) {
log.info("将在{}毫秒后重试({}/{})", FETCH_API_RETRY_DELAY_MS, attempt, FETCH_API_MAX_RETRY);
try {
Thread.sleep(FETCH_API_RETRY_DELAY_MS);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
log.warn("重试等待被中断，停止后续重试");
return null;
}
}
}
return null;
} catch (Exception e) {
log.error("从API获取数据失败，页码：{}", page, e);
// 如果使用了代理IP且请求失败，标记为不可用
if (currentProxy != null) {
proxyIpService.markProxyAsUnavailable(currentProxy.getIp(), currentProxy.getPort());
log.warn("代理IP {}:{} 请求失败，标记为不可用：{}", currentProxy.getIp(), currentProxy.getPort(), e.getMessage());
}
return null;
}
return null;
}
/** 保存数据到数据库 */