feat(PurchaseOrder, SaleOrder): add djgClientIds field to requests and update services for client information processing

This commit is contained in:
2025-10-20 23:34:32 +08:00
parent 304650a749
commit 527560aec3
13 changed files with 1466 additions and 0 deletions

View File

@@ -0,0 +1,97 @@
package cn.lihongjie.coal.zmswpm.controller;
import cn.lihongjie.coal.annotation.SysLog;
import cn.lihongjie.coal.base.dto.CommonQuery;
import cn.lihongjie.coal.base.dto.IdRequest;
import cn.lihongjie.coal.zmswpm.dto.CreateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.SyncSingleBidRequest;
import cn.lihongjie.coal.zmswpm.dto.UpdateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.ZmswpmDto;
import cn.lihongjie.coal.zmswpm.service.ZmswpmService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.Map;
@RestController
@RequestMapping("/zmswpm")
@SysLog(module = "中煤商务拍卖")
@Slf4j
public class ZmswpmController {
@Autowired private ZmswpmService service;
@PostMapping("/create")
public ZmswpmDto create(@RequestBody CreateZmswpmDto request) {
return this.service.create(request);
}
@PostMapping("/update")
public ZmswpmDto update(@RequestBody UpdateZmswpmDto request) {
return this.service.update(request);
}
@PostMapping("/delete")
public Object delete(@RequestBody IdRequest request) {
this.service.delete(request);
return true;
}
@PostMapping("/getById")
public ZmswpmDto getById(@RequestBody IdRequest request) {
return this.service.getById(request.getId());
}
@PostMapping("/list")
public Page<ZmswpmDto> list(@RequestBody CommonQuery request) {
return this.service.list(request);
}
/**
* 同步煤炭商务网拍卖数据
*/
@PostMapping("/syncAuctionData")
public Object syncAuctionData() {
try {
this.service.syncAuctionData();
return Map.of("success", true, "message", "同步成功");
} catch (Exception e) {
log.error("同步煤炭商务网拍卖数据失败", e);
return Map.of("success", false, "message", "同步失败: " + e.getMessage());
}
}
/**
* 同步单个标的名称的数据
*/
@PostMapping("/syncSingleBid")
public Object syncSingleBid(@RequestBody SyncSingleBidRequest request) {
try {
this.service.syncSingleBid(request.getBidname());
return Map.of("success", true, "message", "同步成功");
} catch (Exception e) {
log.error("同步单个标的数据失败", e);
return Map.of("success", false, "message", "同步失败: " + e.getMessage());
}
}
/**
* 同步进行中的拍卖数据
*/
@PostMapping("/syncInProgressData")
public Object syncInProgressData() {
try {
this.service.syncInProgressData();
return Map.of("success", true, "message", "同步成功");
} catch (Exception e) {
log.error("同步进行中拍卖数据失败", e);
return Map.of("success", false, "message", "同步失败: " + e.getMessage());
}
}
}

View File

@@ -0,0 +1,66 @@
package cn.lihongjie.coal.zmswpm.dto;
import lombok.Data;
import java.math.BigDecimal;
@Data
public class CreateZmswpmDto {
private String bidId;
private String productid;
private String productname;
private String bidname;
private String quantity;
private BigDecimal price;
private String splitstate;
private String splitstateName;
private String delivery;
private String tenantCode;
private String tenantCodeName;
private String tenantName;
private String tenantSname;
private String begintime;
private String endtime;
private String outbidstate;
private String bidType;
private String trbegintime;
private String trendtime;
private String img;
private String targetType;
private Long turnover;
private Long newQuantity;
private String producttypeName;
private String producttype;
private String deliveryType;
private String startShipment;
private String endShipment;
private String productDescription;
private String urlList;
private String priceExpiration;
private String contacts;
private String tel;
private String distance;
private String notice;
private String firstProductName;
private String threeProductName;
private String preInstalledPort;
private String transitPort;
private String portOfDestination;
private String shippingDate;
private String percentage;
private String overt;
private String overtEnterpriseCode;
private String warehouseName;
private String warehouseArea;
private String startDeparture;
private String leaseDuration;
private String qualification;
private String priceType;
private String formula;
private String htype;
private String coalType;
private String deptId;
private String companyname;
private String mineName;
}

View File

@@ -0,0 +1,9 @@
package cn.lihongjie.coal.zmswpm.dto;
import lombok.Data;
@Data
public class SyncSingleBidRequest {
private String bidname; // 标的名称,用于同步单个标的
}

View File

@@ -0,0 +1,67 @@
package cn.lihongjie.coal.zmswpm.dto;
import lombok.Data;
import java.math.BigDecimal;
@Data
public class UpdateZmswpmDto {
private String id;
private String bidId;
private String productid;
private String productname;
private String bidname;
private String quantity;
private BigDecimal price;
private String splitstate;
private String splitstateName;
private String delivery;
private String tenantCode;
private String tenantCodeName;
private String tenantName;
private String tenantSname;
private String begintime;
private String endtime;
private String outbidstate;
private String bidType;
private String trbegintime;
private String trendtime;
private String img;
private String targetType;
private Long turnover;
private Long newQuantity;
private String producttypeName;
private String producttype;
private String deliveryType;
private String startShipment;
private String endShipment;
private String productDescription;
private String urlList;
private String priceExpiration;
private String contacts;
private String tel;
private String distance;
private String notice;
private String firstProductName;
private String threeProductName;
private String preInstalledPort;
private String transitPort;
private String portOfDestination;
private String shippingDate;
private String percentage;
private String overt;
private String overtEnterpriseCode;
private String warehouseName;
private String warehouseArea;
private String startDeparture;
private String leaseDuration;
private String qualification;
private String priceType;
private String formula;
private String htype;
private String coalType;
private String deptId;
private String companyname;
private String mineName;
}

View File

@@ -0,0 +1,197 @@
package cn.lihongjie.coal.zmswpm.dto;
import cn.lihongjie.coal.base.dto.CommonDto;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.hibernate.annotations.Comment;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@EqualsAndHashCode(callSuper = false)
public class ZmswpmDto extends CommonDto {
@Comment("标的ID")
private String bidId;
@Comment("产品ID")
private String productid;
@Comment("产品名称")
private String productname;
@Comment("标的名称")
private String bidname;
@Comment("数量")
private String quantity;
@Comment("价格")
private BigDecimal price;
@Comment("拆分状态")
private String splitstate;
@Comment("拆分状态名称")
private String splitstateName;
@Comment("交货地点")
private String delivery;
@Comment("租户代码")
private String tenantCode;
@Comment("租户代码名称")
private String tenantCodeName;
@Comment("租户名称")
private String tenantName;
@Comment("租户简称")
private String tenantSname;
@Comment("开始时间")
private String begintime;
@Comment("结束时间")
private String endtime;
@Comment("出价状态")
private String outbidstate;
@Comment("标的类型")
private String bidType;
@Comment("交易开始时间")
private String trbegintime;
@Comment("交易结束时间")
private String trendtime;
@Comment("图片URL")
private String img;
@Comment("目标类型")
private String targetType;
@Comment("营业额")
private Long turnover;
@Comment("新数量")
private Long newQuantity;
@Comment("产品类型名称")
private String producttypeName;
@Comment("产品类型")
private String producttype;
@Comment("交货类型")
private String deliveryType;
@Comment("开始发货")
private String startShipment;
@Comment("结束发货")
private String endShipment;
@Comment("产品描述")
private String productDescription;
@Comment("URL列表")
private String urlList;
@Comment("价格到期")
private String priceExpiration;
@Comment("联系人")
private String contacts;
@Comment("电话")
private String tel;
@Comment("距离")
private String distance;
@Comment("公告")
private String notice;
@Comment("第一产品名称")
private String firstProductName;
@Comment("第三产品名称")
private String threeProductName;
@Comment("预装港口")
private String preInstalledPort;
@Comment("中转港口")
private String transitPort;
@Comment("目的港口")
private String portOfDestination;
@Comment("发货日期")
private String shippingDate;
@Comment("百分比")
private String percentage;
@Comment("公开")
private String overt;
@Comment("公开企业代码")
private String overtEnterpriseCode;
@Comment("仓库名称")
private String warehouseName;
@Comment("仓库区域")
private String warehouseArea;
@Comment("开始出发")
private String startDeparture;
@Comment("租赁期限")
private String leaseDuration;
@Comment("资格")
private String qualification;
@Comment("价格类型")
private String priceType;
@Comment("公式")
private String formula;
@Comment("H类型")
private String htype;
@Comment("煤炭类型")
private String coalType;
@Comment("部门ID")
private String deptId;
@Comment("公司名称")
private String companyname;
@Comment("矿名")
private String mineName;
// 解析后的时间字段
@Comment("开始时间(解析后)")
private LocalDateTime begintimeParsed;
@Comment("结束时间(解析后)")
private LocalDateTime endtimeParsed;
@Comment("交易开始时间(解析后)")
private LocalDateTime trbegintimeParsed;
@Comment("交易结束时间(解析后)")
private LocalDateTime trendtimeParsed;
}

View File

@@ -0,0 +1,203 @@
package cn.lihongjie.coal.zmswpm.entity;
import cn.lihongjie.coal.base.entity.CommonEntity;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.hibernate.annotations.Comment;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@EqualsAndHashCode(callSuper = false)
@Entity
public class ZmswpmEntity extends CommonEntity {
@Comment("标的ID")
@Column(unique = true)
private String bidId;
@Comment("产品ID")
private String productid;
@Comment("产品名称")
private String productname;
@Comment("标的名称")
private String bidname;
@Comment("数量")
private String quantity;
@Comment("价格")
private BigDecimal price;
@Comment("拆分状态")
private String splitstate;
@Comment("拆分状态名称")
private String splitstateName;
@Comment("交货地点")
private String delivery;
@Comment("租户代码")
private String tenantCode;
@Comment("租户代码名称")
private String tenantCodeName;
@Comment("租户名称")
private String tenantName;
@Comment("租户简称")
private String tenantSname;
@Comment("开始时间")
private String begintime;
@Comment("结束时间")
private String endtime;
@Comment("出价状态")
private String outbidstate;
@Comment("标的类型")
private String bidType;
@Comment("交易开始时间")
private String trbegintime;
@Comment("交易结束时间")
private String trendtime;
@Comment("图片URL")
private String img;
@Comment("目标类型")
private String targetType;
@Comment("营业额")
private Long turnover;
@Comment("新数量")
private Long newQuantity;
@Comment("产品类型名称")
private String producttypeName;
@Comment("产品类型")
private String producttype;
@Comment("交货类型")
private String deliveryType;
@Comment("开始发货")
private String startShipment;
@Comment("结束发货")
private String endShipment;
@Comment("产品描述")
private String productDescription;
@Comment("URL列表")
private String urlList;
@Comment("价格到期")
private String priceExpiration;
@Comment("联系人")
private String contacts;
@Comment("电话")
private String tel;
@Comment("距离")
private String distance;
@Comment("公告")
@Column(columnDefinition = "TEXT")
private String notice;
@Comment("第一产品名称")
private String firstProductName;
@Comment("第三产品名称")
private String threeProductName;
@Comment("预装港口")
private String preInstalledPort;
@Comment("中转港口")
private String transitPort;
@Comment("目的港口")
private String portOfDestination;
@Comment("发货日期")
private String shippingDate;
@Comment("百分比")
private String percentage;
@Comment("公开")
private String overt;
@Comment("公开企业代码")
private String overtEnterpriseCode;
@Comment("仓库名称")
private String warehouseName;
@Comment("仓库区域")
private String warehouseArea;
@Comment("开始出发")
private String startDeparture;
@Comment("租赁期限")
private String leaseDuration;
@Comment("资格")
private String qualification;
@Comment("价格类型")
private String priceType;
@Comment("公式")
private String formula;
@Comment("H类型")
private String htype;
@Comment("煤炭类型")
private String coalType;
@Comment("部门ID")
private String deptId;
@Comment("公司名称")
private String companyname;
@Comment("矿名")
private String mineName;
// 解析后的时间字段
@Comment("开始时间(解析后)")
private LocalDateTime begintimeParsed;
@Comment("结束时间(解析后)")
private LocalDateTime endtimeParsed;
@Comment("交易开始时间(解析后)")
private LocalDateTime trbegintimeParsed;
@Comment("交易结束时间(解析后)")
private LocalDateTime trendtimeParsed;
}

View File

@@ -0,0 +1,19 @@
package cn.lihongjie.coal.zmswpm.mapper;
import cn.lihongjie.coal.base.mapper.BaseMapper;
import cn.lihongjie.coal.base.mapper.CommonEntityMapper;
import cn.lihongjie.coal.base.mapper.CommonMapper;
import cn.lihongjie.coal.zmswpm.dto.CreateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.UpdateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.ZmswpmDto;
import cn.lihongjie.coal.zmswpm.entity.ZmswpmEntity;
import org.mapstruct.Mapper;
import org.mapstruct.control.DeepClone;
@Mapper(
componentModel = org.mapstruct.MappingConstants.ComponentModel.SPRING,
uses = {CommonMapper.class, CommonEntityMapper.class},
mappingControl = DeepClone.class)
public interface ZmswpmMapper
extends BaseMapper<ZmswpmEntity, ZmswpmDto, CreateZmswpmDto, UpdateZmswpmDto> {}

View File

@@ -0,0 +1,30 @@
package cn.lihongjie.coal.zmswpm.repository;
import cn.lihongjie.coal.base.dao.BaseRepository;
import cn.lihongjie.coal.zmswpm.entity.ZmswpmEntity;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
@Repository
public interface ZmswpmRepository extends BaseRepository<ZmswpmEntity> {
@Query("select false")
boolean isLinked(List<String> ids);
/**
* 根据标的ID查找实体
*/
Optional<ZmswpmEntity> findByBidId(String bidId);
/**
* 根据标的名称更新状态
*/
@Modifying
@Query("UPDATE ZmswpmEntity e SET e.status = :status WHERE e.bidname = :bidname")
void updateStatusByBidname(@Param("bidname") String bidname, @Param("status") int status);
}

View File

@@ -0,0 +1,730 @@
package cn.lihongjie.coal.zmswpm.service;
import cn.lihongjie.coal.base.dto.CommonQuery;
import cn.lihongjie.coal.base.dto.IdRequest;
import cn.lihongjie.coal.base.service.BaseService;
import cn.lihongjie.coal.exception.BizException;
import cn.lihongjie.coal.proxyIp.dto.ProxyIp;
import cn.lihongjie.coal.proxyIp.service.ProxyIpService;
import cn.lihongjie.coal.rabbitmq.RabbitMQService;
import cn.lihongjie.coal.zmswpm.dto.CreateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.UpdateZmswpmDto;
import cn.lihongjie.coal.zmswpm.dto.ZmswpmDto;
import cn.lihongjie.coal.zmswpm.entity.ZmswpmEntity;
import cn.lihongjie.coal.zmswpm.mapper.ZmswpmMapper;
import cn.lihongjie.coal.zmswpm.repository.ZmswpmRepository;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaQuery;
import jakarta.persistence.criteria.Predicate;
import jakarta.persistence.criteria.Root;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.convert.ConversionService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.http.*;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;
import java.math.BigDecimal;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
@Transactional
public class ZmswpmService extends BaseService<ZmswpmEntity, ZmswpmRepository> {
private static final String API_URL =
"https://www.chinacoalst.cn/electronicbusiness/shoppingApi/bidttargpage";
private static final int PAGE_SIZE = 50;
private static final DateTimeFormatter DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Autowired private ZmswpmRepository repository;
@Autowired private ZmswpmMapper mapper;
@Autowired private ConversionService conversionService;
@Autowired private RestTemplate restTemplate;
@Autowired private ObjectMapper objectMapper;
@Autowired private ProxyIpService proxyIpService;
@Autowired private RabbitMQService rabbitMQService;
/** 创建带代理的RestTemplate */
private RestTemplate createProxyRestTemplate(ProxyIp proxyIp) {
if (proxyIp == null) {
return restTemplate; // 返回默认的RestTemplate
}
try {
// 创建代理
Proxy proxy =
new Proxy(
Proxy.Type.HTTP,
new InetSocketAddress(proxyIp.getIp(), proxyIp.getPort()));
// 创建带代理的请求工厂
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setProxy(proxy);
factory.setConnectTimeout(10000); // 10秒连接超时
factory.setReadTimeout(15000); // 15秒读取超时
// 创建新的RestTemplate
RestTemplate proxyRestTemplate = new RestTemplate(factory);
log.info("创建带代理的RestTemplate代理地址{}:{}", proxyIp.getIp(), proxyIp.getPort());
return proxyRestTemplate;
} catch (Exception e) {
log.error("创建代理RestTemplate失败使用默认RestTemplate{}", e.getMessage());
return restTemplate;
}
}
public ZmswpmDto create(CreateZmswpmDto request) {
ZmswpmEntity entity = mapper.toEntity(request);
this.repository.save(entity);
return getById(entity.getId());
}
public ZmswpmDto update(UpdateZmswpmDto request) {
ZmswpmEntity entity = this.repository.get(request.getId());
this.mapper.updateEntity(entity, request);
this.repository.save(entity);
return getById(entity.getId());
}
public void delete(IdRequest request) {
boolean linked = this.repository.isLinked(request.getIds());
if (linked) {
throw new BizException("数据已被关联,无法删除");
}
this.repository.deleteAllById(request.getIds());
}
public ZmswpmDto getById(String id) {
ZmswpmEntity entity = repository.get(id);
return mapper.toDto(entity);
}
@SuppressWarnings("unchecked")
public Page<ZmswpmDto> list(CommonQuery query) {
Page<ZmswpmEntity> page =
repository.findAll(
(Specification<ZmswpmEntity>) query.specification(conversionService),
PageRequest.of(
query.getPageNo(),
query.getPageSize(),
Sort.by(query.getOrders())));
return page.map(this.mapper::toDto);
}
/** 同步单个标的名称的数据 */
public void syncSingleBid(String bidname) {
log.info("开始同步标的名称为 {} 的数据", bidname);
if (bidname == null || bidname.trim().isEmpty()) {
throw new BizException("标的名称不能为空");
}
try {
// 调用API查询特定标的名称的数据
List<Map<String, Object>> dataList = fetchDataFromApi(1, bidname.trim());
if (dataList == null || dataList.isEmpty()) {
log.info("未找到标的名称为 {} 的数据", bidname);
// 标记为禁用
repository.updateStatusByBidname(bidname.trim(), 0);
return;
}
// 处理返回的数据
for (Map<String, Object> dataMap : dataList) {
try {
String apiBidname = getStringValue(dataMap, "bidname");
// 检查是否包含指定的标的名称
if (apiBidname != null && apiBidname.contains(bidname.trim())) {
// 使用id作为唯一标识符来判断是否已存在
Object idObj = dataMap.get("id");
if (idObj != null) {
String bidId = idObj.toString();
Optional<ZmswpmEntity> existingEntity = repository.findByBidId(bidId);
if (existingEntity.isPresent()) {
// 更新现有记录
ZmswpmEntity entity = existingEntity.get();
ZmswpmDto old = mapper.toDto(entity);
updateEntityFromMap(entity, dataMap);
repository.save(entity);
log.info("已更新bid ID为 {} 标的名称为 {} 的记录", bidId, apiBidname);
rabbitMQService.sendToSysExchange(
"zmswpm.updatedata",
Map.of("old", old, "new", mapper.toDto(entity)));
} else {
// 新增记录
ZmswpmEntity entity = convertMapToEntity(dataMap);
repository.save(entity);
rabbitMQService.sendToSysExchange(
"zmswpm.newdata", mapper.toDto(entity));
log.info("已新增bid ID为 {} 标的名称为 {} 的记录", bidId, apiBidname);
}
}
}
} catch (Exception e) {
log.error("处理标的名称为 {} 的数据失败", bidname, e);
}
}
log.info("标的名称为 {} 的数据同步完成", bidname);
} catch (Exception e) {
log.error("同步标的名称为 {} 的数据失败", bidname, e);
throw new BizException("同步数据失败: " + e.getMessage());
}
}
/** 同步煤炭商务网拍卖数据 */
public void syncAuctionData() {
log.info("开始同步煤炭商务网拍卖数据");
try {
// 检查数据库是否为空
long totalCount = repository.count();
if (totalCount == 0) {
// 数据库为空,调用接口直到没有数据返回
syncAllData();
} else {
// 数据库不为空,同步当前天和前三天的数据
syncRecentData(LocalDate.now(), LocalDate.now().minusDays(7));
}
log.info("煤炭商务网拍卖数据同步完成");
} catch (Exception e) {
log.error("同步煤炭商务网拍卖数据失败", e);
throw new BizException("同步数据失败: " + e.getMessage());
}
}
/** 同步所有数据(数据库为空时使用) */
private void syncAllData() {
int currentPage = 1;
boolean hasMoreData = true;
while (hasMoreData) {
log.info("正在同步第{}页数据", currentPage);
List<Map<String, Object>> dataList = fetchDataFromApi(currentPage);
if (dataList == null || dataList.isEmpty()) {
hasMoreData = false;
log.info("第{}页没有数据,同步完成", currentPage);
} else {
// 保存数据到数据库
saveDataToDatabase(dataList);
log.info("第{}页数据保存完成,共{}条记录", currentPage, dataList.size());
if (dataList.size() < PAGE_SIZE) {
hasMoreData = false;
log.info("数据量小于分页大小,同步完成");
}
currentPage++;
}
}
}
/**
* 同步近期数据(当前天和前三天)
*
* @param start
* @param end
*/
public void syncRecentData(LocalDate start, LocalDate end) {
String begintime = end.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + " 00:00:00";
String endtime = start.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + " 23:59:59";
log.info("同步{}到{}的数据", begintime, endtime);
int currentPage = 1;
boolean hasMoreData = true;
while (hasMoreData) {
log.info("正在同步第{}页数据", currentPage);
List<Map<String, Object>> dataList =
fetchDataFromApi(currentPage, null, begintime, endtime);
if (dataList == null || dataList.isEmpty()) {
hasMoreData = false;
log.info("第{}页没有数据,同步完成", currentPage);
} else {
// 更新或新增数据到数据库
updateOrSaveDataToDatabase(dataList);
log.info("第{}页数据处理完成,共{}条记录", currentPage, dataList.size());
if (dataList.size() < PAGE_SIZE) {
hasMoreData = false;
log.info("数据量小于分页大小,同步完成");
}
currentPage++;
}
}
}
/** 从API获取数据 */
private List<Map<String, Object>> fetchDataFromApi(int page) {
return fetchDataFromApi(page, null, null, null);
}
/** 从API获取数据支持按标的名称查询 */
private List<Map<String, Object>> fetchDataFromApi(int page, String bidname) {
return fetchDataFromApi(page, bidname, null, null);
}
/** 从API获取数据支持按标的名称和时间范围查询 */
private List<Map<String, Object>> fetchDataFromApi(
int page, String bidname, String begintime, String endtime) {
ProxyIp currentProxy = null;
RestTemplate currentRestTemplate = restTemplate;
try {
// 获取可用的代理IP
Optional<ProxyIp> proxyOptional = proxyIpService.getAvailableProxyIp();
if (proxyOptional.isPresent()) {
currentProxy = proxyOptional.get();
currentRestTemplate = createProxyRestTemplate(currentProxy);
log.info("使用代理IP进行API请求{}:{}", currentProxy.getIp(), currentProxy.getPort());
} else {
log.warn("未获取到可用代理IP使用直接连接");
}
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.set("Accept", "*/*");
headers.set("Accept-Language", "zh-CN,zh;q=0.9");
headers.set("Cache-Control", "no-cache");
headers.set("Connection", "keep-alive");
headers.set("host", "www.chinacoalst.cn");
headers.set("x-auth-token", "");
headers.set("sec-ch-ua-platform", "\"Windows\"");
headers.set(
"sec-ch-ua",
"\"Chromium\";v=\"140\", \"Not=A?Brand\";v=\"24\", \"Google Chrome\";v=\"140\"");
headers.set("sec-ch-ua-mobile", "?0");
headers.set("language-type-code", "null");
headers.set("id", "9db233983befbae5");
headers.set(
"user-agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36");
headers.set("token", "null");
headers.set("sec-fetch-site", "same-origin");
headers.set("sec-fetch-mode", "cors");
headers.set("sec-fetch-dest", "empty");
headers.set("referer", "https://www.chinacoalst.cn/");
headers.set("accept-encoding", "gzip, deflate, br, zstd");
headers.set("priority", "u=1, i");
headers.set(
"Cookie",
"acw_tc=0bd17c0e17606243767095896e3ece80da4344e5f095bfd2618afec87f6899");
// 构建URL参数
StringBuilder urlBuilder = new StringBuilder(API_URL);
urlBuilder.append("?page=").append(page);
urlBuilder.append("&limit=").append(PAGE_SIZE);
urlBuilder.append("&productid=");
urlBuilder.append("&delivery=");
urlBuilder.append("&outbidstate=");
urlBuilder.append("&coalType=");
urlBuilder.append("&splitstate=");
urlBuilder.append("&tenantCode=");
urlBuilder.append("&type=1");
if (begintime != null && !begintime.isEmpty()) {
urlBuilder.append("&begintime=").append(begintime);
} else {
urlBuilder.append("&begintime=");
}
if (endtime != null && !endtime.isEmpty()) {
urlBuilder.append("&endtime=").append(endtime);
} else {
urlBuilder.append("&endtime=");
}
if (bidname != null && !bidname.trim().isEmpty()) {
urlBuilder.append("&bidname=").append(bidname.trim());
} else {
urlBuilder.append("&bidname=");
}
urlBuilder.append("&mineName=");
HttpEntity<String> request = new HttpEntity<>(headers);
String url = urlBuilder.toString();
// 发送请求
ResponseEntity<String> response =
currentRestTemplate.getForEntity(url, String.class, request);
if (response.getStatusCode() == HttpStatus.OK) {
// 标记代理IP使用成功
if (currentProxy != null) {
proxyIpService.markProxyAsUsed(currentProxy.getIp(), currentProxy.getPort());
log.debug("代理IP {}:{} 请求成功", currentProxy.getIp(), currentProxy.getPort());
}
String responseBody = response.getBody();
Map<String, Object> responseMap =
objectMapper.readValue(
responseBody, new TypeReference<Map<String, Object>>() {});
Integer code = (Integer) responseMap.get("code");
if (code != null && code == 0) {
Object dataObj = responseMap.get("data");
if (dataObj instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> dataMap = (Map<String, Object>) dataObj;
Object listObj = dataMap.get("list");
if (listObj instanceof List) {
@SuppressWarnings("unchecked")
List<Map<String, Object>> dataList =
(List<Map<String, Object>>) listObj;
return dataList;
}
}
}
} else {
log.warn("API请求返回非200状态码{}", response.getStatusCode());
// 如果是代理相关的错误标记代理IP为不可用
if (currentProxy != null
&& (response.getStatusCode().is4xxClientError()
|| response.getStatusCode().is5xxServerError())) {
proxyIpService.markProxyAsUnavailable(
currentProxy.getIp(), currentProxy.getPort());
log.warn(
"代理IP {}:{} 可能不可用,状态码:{}",
currentProxy.getIp(),
currentProxy.getPort(),
response.getStatusCode());
}
}
return null;
} catch (Exception e) {
log.error("从API获取数据失败页码{}", page, e);
// 如果使用了代理IP且请求失败标记为不可用
if (currentProxy != null) {
proxyIpService.markProxyAsUnavailable(currentProxy.getIp(), currentProxy.getPort());
log.warn(
"代理IP {}:{} 请求失败,标记为不可用:{}",
currentProxy.getIp(),
currentProxy.getPort(),
e.getMessage());
}
return null;
}
}
/** 保存数据到数据库 */
private void saveDataToDatabase(List<Map<String, Object>> dataList) {
for (Map<String, Object> dataMap : dataList) {
try {
ZmswpmEntity entity = convertMapToEntity(dataMap);
repository.save(entity);
} catch (Exception e) {
log.error("保存数据失败,数据:{}", dataMap, e);
}
}
repository.flush();
}
/** 更新或保存数据到数据库 */
private void updateOrSaveDataToDatabase(List<Map<String, Object>> dataList) {
for (Map<String, Object> dataMap : dataList) {
try {
Object idObj = dataMap.get("id");
if (idObj != null) {
String bidId = idObj.toString();
Optional<ZmswpmEntity> existingEntity = repository.findByBidId(bidId);
if (existingEntity.isPresent()) {
// 更新现有记录
ZmswpmEntity entity = existingEntity.get();
ZmswpmDto old = mapper.toDto(entity);
updateEntityFromMap(entity, dataMap);
repository.save(entity);
log.info("已更新bid ID为 {} 的记录", bidId);
rabbitMQService.sendToSysExchange(
"zmswpm.updatedata",
Map.of("old", old, "new", mapper.toDto(entity)));
} else {
// 新增记录
ZmswpmEntity entity = convertMapToEntity(dataMap);
repository.save(entity);
log.info("已新增bid ID为 {} 的记录", bidId);
rabbitMQService.sendToSysExchange("zmswpm.newdata", mapper.toDto(entity));
}
}
} catch (Exception e) {
log.error("更新或保存数据失败,数据:{}", dataMap, e);
}
}
}
/** 将Map数据转换为Entity */
private ZmswpmEntity convertMapToEntity(Map<String, Object> dataMap) {
ZmswpmEntity entity = new ZmswpmEntity();
// 基本字段映射
entity.setBidId(getStringValue(dataMap, "id"));
entity.setProductid(getStringValue(dataMap, "productid"));
entity.setProductname(getStringValue(dataMap, "productname"));
entity.setBidname(getStringValue(dataMap, "bidname"));
entity.setQuantity(getStringValue(dataMap, "quantity"));
entity.setPrice(getBigDecimalValue(dataMap, "price"));
entity.setSplitstate(getStringValue(dataMap, "splitstate"));
entity.setSplitstateName(getStringValue(dataMap, "splitstateName"));
entity.setDelivery(getStringValue(dataMap, "delivery"));
entity.setTenantCode(getStringValue(dataMap, "tenantCode"));
entity.setTenantCodeName(getStringValue(dataMap, "tenantCodeName"));
entity.setTenantName(getStringValue(dataMap, "tenantName"));
entity.setTenantSname(getStringValue(dataMap, "tenantSname"));
entity.setBegintime(getStringValue(dataMap, "begintime"));
entity.setEndtime(getStringValue(dataMap, "endtime"));
entity.setOutbidstate(getStringValue(dataMap, "outbidstate"));
entity.setBidType(getStringValue(dataMap, "bidType"));
entity.setTrbegintime(getStringValue(dataMap, "trbegintime"));
entity.setTrendtime(getStringValue(dataMap, "trendtime"));
entity.setImg(getStringValue(dataMap, "img"));
entity.setTargetType(getStringValue(dataMap, "targetType"));
entity.setTurnover(getLongValue(dataMap, "turnover"));
entity.setNewQuantity(getLongValue(dataMap, "newQuantity"));
entity.setProducttypeName(getStringValue(dataMap, "producttypeName"));
entity.setProducttype(getStringValue(dataMap, "producttype"));
entity.setDeliveryType(getStringValue(dataMap, "deliveryType"));
entity.setStartShipment(getStringValue(dataMap, "startShipment"));
entity.setEndShipment(getStringValue(dataMap, "endShipment"));
entity.setProductDescription(getStringValue(dataMap, "productDescription"));
entity.setUrlList(getStringValue(dataMap, "urlList"));
entity.setPriceExpiration(getStringValue(dataMap, "priceExpiration"));
entity.setContacts(getStringValue(dataMap, "contacts"));
entity.setTel(getStringValue(dataMap, "tel"));
entity.setDistance(getStringValue(dataMap, "distance"));
entity.setNotice(getStringValue(dataMap, "notice"));
entity.setFirstProductName(getStringValue(dataMap, "firstProductName"));
entity.setThreeProductName(getStringValue(dataMap, "threeProductName"));
entity.setPreInstalledPort(getStringValue(dataMap, "preInstalledPort"));
entity.setTransitPort(getStringValue(dataMap, "transitPort"));
entity.setPortOfDestination(getStringValue(dataMap, "portOfDestination"));
entity.setShippingDate(getStringValue(dataMap, "shippingDate"));
entity.setPercentage(getStringValue(dataMap, "percentage"));
entity.setOvert(getStringValue(dataMap, "overt"));
entity.setOvertEnterpriseCode(getStringValue(dataMap, "overtEnterpriseCode"));
entity.setWarehouseName(getStringValue(dataMap, "warehouseName"));
entity.setWarehouseArea(getStringValue(dataMap, "warehouseArea"));
entity.setStartDeparture(getStringValue(dataMap, "startDeparture"));
entity.setLeaseDuration(getStringValue(dataMap, "leaseDuration"));
entity.setQualification(getStringValue(dataMap, "qualification"));
entity.setPriceType(getStringValue(dataMap, "priceType"));
entity.setFormula(getStringValue(dataMap, "formula"));
entity.setHtype(getStringValue(dataMap, "htype"));
entity.setCoalType(getStringValue(dataMap, "coalType"));
entity.setDeptId(getStringValue(dataMap, "deptId"));
entity.setCompanyname(getStringValue(dataMap, "companyname"));
entity.setMineName(getStringValue(dataMap, "mineName"));
// 解析时间字段
entity.setBegintimeParsed(getLocalDateTimeValue(dataMap, "begintime"));
entity.setEndtimeParsed(getLocalDateTimeValue(dataMap, "endtime"));
entity.setTrbegintimeParsed(getLocalDateTimeValue(dataMap, "trbegintime"));
entity.setTrendtimeParsed(getLocalDateTimeValue(dataMap, "trendtime"));
return entity;
}
/** 更新现有Entity的数据 */
private void updateEntityFromMap(ZmswpmEntity entity, Map<String, Object> dataMap) {
// 基本字段映射
entity.setBidId(getStringValue(dataMap, "id"));
entity.setProductid(getStringValue(dataMap, "productid"));
entity.setProductname(getStringValue(dataMap, "productname"));
entity.setBidname(getStringValue(dataMap, "bidname"));
entity.setQuantity(getStringValue(dataMap, "quantity"));
entity.setPrice(getBigDecimalValue(dataMap, "price"));
entity.setSplitstate(getStringValue(dataMap, "splitstate"));
entity.setSplitstateName(getStringValue(dataMap, "splitstateName"));
entity.setDelivery(getStringValue(dataMap, "delivery"));
entity.setTenantCode(getStringValue(dataMap, "tenantCode"));
entity.setTenantCodeName(getStringValue(dataMap, "tenantCodeName"));
entity.setTenantName(getStringValue(dataMap, "tenantName"));
entity.setTenantSname(getStringValue(dataMap, "tenantSname"));
entity.setBegintime(getStringValue(dataMap, "begintime"));
entity.setEndtime(getStringValue(dataMap, "endtime"));
entity.setOutbidstate(getStringValue(dataMap, "outbidstate"));
entity.setBidType(getStringValue(dataMap, "bidType"));
entity.setTrbegintime(getStringValue(dataMap, "trbegintime"));
entity.setTrendtime(getStringValue(dataMap, "trendtime"));
entity.setImg(getStringValue(dataMap, "img"));
entity.setTargetType(getStringValue(dataMap, "targetType"));
entity.setTurnover(getLongValue(dataMap, "turnover"));
entity.setNewQuantity(getLongValue(dataMap, "newQuantity"));
entity.setProducttypeName(getStringValue(dataMap, "producttypeName"));
entity.setProducttype(getStringValue(dataMap, "producttype"));
entity.setDeliveryType(getStringValue(dataMap, "deliveryType"));
entity.setStartShipment(getStringValue(dataMap, "startShipment"));
entity.setEndShipment(getStringValue(dataMap, "endShipment"));
entity.setProductDescription(getStringValue(dataMap, "productDescription"));
entity.setUrlList(getStringValue(dataMap, "urlList"));
entity.setPriceExpiration(getStringValue(dataMap, "priceExpiration"));
entity.setContacts(getStringValue(dataMap, "contacts"));
entity.setTel(getStringValue(dataMap, "tel"));
entity.setDistance(getStringValue(dataMap, "distance"));
entity.setNotice(getStringValue(dataMap, "notice"));
entity.setFirstProductName(getStringValue(dataMap, "firstProductName"));
entity.setThreeProductName(getStringValue(dataMap, "threeProductName"));
entity.setPreInstalledPort(getStringValue(dataMap, "preInstalledPort"));
entity.setTransitPort(getStringValue(dataMap, "transitPort"));
entity.setPortOfDestination(getStringValue(dataMap, "portOfDestination"));
entity.setShippingDate(getStringValue(dataMap, "shippingDate"));
entity.setPercentage(getStringValue(dataMap, "percentage"));
entity.setOvert(getStringValue(dataMap, "overt"));
entity.setOvertEnterpriseCode(getStringValue(dataMap, "overtEnterpriseCode"));
entity.setWarehouseName(getStringValue(dataMap, "warehouseName"));
entity.setWarehouseArea(getStringValue(dataMap, "warehouseArea"));
entity.setStartDeparture(getStringValue(dataMap, "startDeparture"));
entity.setLeaseDuration(getStringValue(dataMap, "leaseDuration"));
entity.setQualification(getStringValue(dataMap, "qualification"));
entity.setPriceType(getStringValue(dataMap, "priceType"));
entity.setFormula(getStringValue(dataMap, "formula"));
entity.setHtype(getStringValue(dataMap, "htype"));
entity.setCoalType(getStringValue(dataMap, "coalType"));
entity.setDeptId(getStringValue(dataMap, "deptId"));
entity.setCompanyname(getStringValue(dataMap, "companyname"));
entity.setMineName(getStringValue(dataMap, "mineName"));
// 解析时间字段
entity.setBegintimeParsed(getLocalDateTimeValue(dataMap, "begintime"));
entity.setEndtimeParsed(getLocalDateTimeValue(dataMap, "endtime"));
entity.setTrbegintimeParsed(getLocalDateTimeValue(dataMap, "trbegintime"));
entity.setTrendtimeParsed(getLocalDateTimeValue(dataMap, "trendtime"));
}
// 辅助方法用于安全地从Map中获取值
private String getStringValue(Map<String, Object> map, String key) {
Object value = map.get(key);
return value != null ? value.toString() : null;
}
private Long getLongValue(Map<String, Object> map, String key) {
Object value = map.get(key);
if (value == null) return null;
try {
return Long.valueOf(value.toString());
} catch (NumberFormatException e) {
return null;
}
}
private BigDecimal getBigDecimalValue(Map<String, Object> map, String key) {
Object value = map.get(key);
if (value == null) return null;
try {
return new BigDecimal(value.toString());
} catch (NumberFormatException e) {
return null;
}
}
private LocalDateTime getLocalDateTimeValue(Map<String, Object> map, String key) {
Object value = map.get(key);
if (value == null) return null;
try {
String dateStr = value.toString();
return LocalDateTime.parse(dateStr, DATE_TIME_FORMATTER);
} catch (Exception e) {
return null;
}
}
/** 同步进行中的拍卖数据 */
public void syncInProgressData() {
List<ZmswpmEntity> all =
this.repository.findAll(
new Specification<ZmswpmEntity>() {
@Override
public Predicate toPredicate(
Root<ZmswpmEntity> root,
CriteriaQuery<?> query,
CriteriaBuilder criteriaBuilder) {
// 只同步状态为进行中的拍卖outbidstate等于特定值
// 只同步当天的拍卖
// 只同步结束时间小于当前时间的拍卖 表示已经结束
return criteriaBuilder.and(
//
// criteriaBuilder.equal(root.get("outbidstate"), "4"), //
// 4或者3表示需要同步
criteriaBuilder
.in(root.get("outbidstate"))
.value("3")
.value("4"),
criteriaBuilder.equal(root.get("status"), 1),
criteriaBuilder.lessThanOrEqualTo(
root.get("endtimeParsed"), LocalDateTime.now()));
}
});
if (all.isEmpty()) {
log.info("没有需要同步的进行中拍卖");
return;
}
Set<String> set = all.stream().map(x -> x.getBidname()).collect(Collectors.toSet());
for (String s : set) {
try {
syncSingleBid(s);
log.info("同步标的名称为 {} 的进行中拍卖完成", s);
} catch (Exception e) {
log.error("同步标的名称为 {} 的进行中拍卖失败", s, e);
}
}
}
void syncIncrement(Integer maxPage, Integer pageSize) {}
}

View File

@@ -0,0 +1,9 @@
import cn.lihongjie.coal.zmswpm.service.ZmswpmService
import org.springframework.context.ApplicationContext
ApplicationContext ioc = ioc
def service = ioc.getBean(ZmswpmService.class)
service.syncAuctionData()

View File

@@ -0,0 +1,11 @@
import cn.lihongjie.coal.zmswpm.service.ZmswpmService
import org.springframework.context.ApplicationContext
import java.time.LocalDate
ApplicationContext ioc = ioc
def service = ioc.getBean(ZmswpmService.class)
service.syncRecentData(LocalDate.now(), LocalDate.now())

View File

@@ -0,0 +1,9 @@
import cn.lihongjie.coal.zmswpm.service.ZmswpmService
import org.springframework.context.ApplicationContext
ApplicationContext ioc = ioc
def service = ioc.getBean(ZmswpmService.class)
service.syncInProgressData()

View File

@@ -0,0 +1,19 @@
package scripts.dict
import cn.lihongjie.coal.base.dto.CommonQuery
import cn.lihongjie.coal.zmswpm.controller.ZmswpmController
import com.fasterxml.jackson.databind.ObjectMapper
import org.springframework.context.ApplicationContext
ApplicationContext ioc = ioc
def controller = ioc.getBean(ZmswpmController.class)
def objectMapper = ioc.getBean(ObjectMapper.class) as ObjectMapper
return controller.list(params!=null ? objectMapper.convertValue(params, CommonQuery.class ) : new CommonQuery())