Middleware/cn-data-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

1.优化数据清洗文件上传目录结构，结构导致因目录过多导致阿里云文件服务器卡顿问题；

Browse Source 
2.优化算法执行目录删除多余日志输出；
This commit is contained in:
wr
2026-06-23 15:01:40 +08:00
parent 5090e023b1
commit e56dc8c4be
14 changed files with 101 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/LnDataDealServiceImpl.java
View File

@@ -73,11 +73,10 @@ public class LnDataDealServiceImpl implements LnDataDealService {
private Executor executor;
@Override
public void batchInsertion(LnDataDTO lnDataDTO) {
long start = System.currentTimeMillis();
List<CompletableFuture<Void>> futures = new ArrayList<>();
// 提交每个批量插入任务
// 提交每个批量插入任务
futures.add(CompletableFuture.runAsync(() -> dataVService.batchInsertion(lnDataDTO.getDataVList()), executor));
futures.add(CompletableFuture.runAsync(() -> dataHarmrateVQuery.batchInsertion(lnDataDTO.getDataHarmrateVDTOList()), executor));
futures.add(CompletableFuture.runAsync(() -> dataFlickerQuery.batchInsertion(lnDataDTO.getDataFlickerDTOList()), executor));
@@ -95,8 +94,6 @@ public class LnDataDealServiceImpl implements LnDataDealService {
// 等待所有任务完成，如果任一任务抛出异常，这里会重新抛出
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
long end = System.currentTimeMillis();
log.info("入库方法执行耗时：" + (end - start) + " ms");
// dataVService.batchInsertion(lnDataDTO.getDataVList());
@@ -118,8 +115,6 @@ public class LnDataDealServiceImpl implements LnDataDealService {
List<String> lineIdList = lnDataDTO.getDataVList().stream().map(DataVDTO::getLineid).distinct().collect(Collectors.toList());
List<LineDeviceStateVO> data = lineFeignClient.getAllLine(lineIdList).getData();
long end2 = System.currentTimeMillis();
log.info("查询台账耗时：" + (end2 - end) + " ms");
Map<String, String> map = data.stream().collect(Collectors.toMap(LineDeviceStateVO::getId, temp -> temp.getPids().split(",")[4]));
@@ -144,9 +139,6 @@ public class LnDataDealServiceImpl implements LnDataDealService {
});
CompletableFuture.allOf(updatefutures.toArray(new CompletableFuture[0])).join();
long end3 = System.currentTimeMillis();
log.info("更新device表：" + (end3 - end2) + " ms");
}

15
data-processing/data-processing-boot/src/main/java/com/njcn/dataProcess/service/impl/influxdb/InfluxdbPqsCommunicateImpl.java
View File

@@ -110,19 +110,16 @@ public class InfluxdbPqsCommunicateImpl implements IPqsCommunicate {
@Override
public void insertion(PqsCommunicateDto pqsCommunicateDto) {
// log.info("进出Influxdb实现类");
//获取最新一条数据
if(Objects.equals(pqsCommunicateDto.getUpdateCommunicateFlag(),0)){
long start = System.currentTimeMillis();
PqsCommunicate dto = new PqsCommunicate();
InfluxQueryWrapper influxQueryWrapper = new InfluxQueryWrapper(PqsCommunicate.class);
influxQueryWrapper.eq(PqsCommunicate::getDevId,pqsCommunicateDto.getDevId()).timeDesc().limit(1);
List<PqsCommunicate> pqsCommunicates = pqsCommunicateMapper.selectByQueryWrapper(influxQueryWrapper);
long end1 = System.currentTimeMillis();
log.info("查询通讯表最新数据：" + (end1 - start) + " ms");
PqsCommunicate pqsCommunicate = new PqsCommunicate();
pqsCommunicate.setTime(LocalDateTime.parse(pqsCommunicateDto.getTime(), DATE_TIME_FORMATTER).atZone(ZoneId.systemDefault()).toInstant());
pqsCommunicate.setDevId(pqsCommunicateDto.getDevId());
@@ -133,11 +130,10 @@ public class InfluxdbPqsCommunicateImpl implements IPqsCommunicate {
pqsCommunicateMapper.insertOne(pqsCommunicate);
}
long end2 = System.currentTimeMillis();
log.info("更新通讯表最新数据：" + (end2 - end1) + " ms");
}
long end3 = System.currentTimeMillis();
//更新mysql数据
@@ -149,8 +145,7 @@ public class InfluxdbPqsCommunicateImpl implements IPqsCommunicate {
devComFlagDTO.setStatus(pqsCommunicateDto.getType());
deviceFeignClient.updateDevComFlag(devComFlagDTO);
long end4 = System.currentTimeMillis();
log.info("更新device表状态和时间：" + (end4 - end3) + " ms");
}