lnk/microser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改mq为C++库并固定生产者和消费者

Browse Source
This commit is contained in:
lnk
2026-04-28 11:44:40 +08:00
parent b87da0f454
commit b1d8440e6a
9 changed files with 636 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
json/save2json.cpp
View File

@@ -34,7 +34,8 @@ using namespace std;
#include "../json/save2json.h"
#include "../json/mms_json_inter.h"
#include "kafka_producer.h"
#include "../rocketmq/CPushConsumer.h"
//#include "../rocketmq/CPushConsumer.h"
#include "../rocketmq/DefaultMQPushConsumer.h"
#include <vector>
#include "../json/cjson.h" //解json
#include <sstream> //创建xml
@@ -45,6 +46,9 @@ extern std::string intToString(int number);
int StringToInt(const std::string& str);
extern pthread_mutex_t mtx;//lnk20250115
extern bool should_process_after_start(const rocketmq::MQMessageExt& msg);
extern int64_t G_APP_START_MS;
extern int RECALL_ONLY_FLAG; //lnk20260309添加一个全局变量，标志是否只运行补招程序
extern void SendFileWeb(const std::string& strUrl,
@@ -156,6 +160,8 @@ extern std::string Topic_Reply_Key;
extern std::string WEB_FILEUPLOAD;
extern std::string WEB_FILEDOWNLOAD;
extern std::string G_ROCKETMQ_CONSUMER;
bool showinshellflag =false;
@@ -363,7 +369,7 @@ void my_rocketmq_send(Ckafka_data_t& data)
}
//rocketmq_producer_send(const_cast<char*>(senddata.c_str()),const_cast<char*>(topic.c_str()));
rocketmq_producer_send(senddata.c_str(), topic.c_str());//lnk20250623修复偶发性doublefree
rocketmq_producer_send(senddata, topic,FRONT_INST,key);//lnk20250623修复偶发性doublefree
}
#if 0
@@ -722,7 +728,7 @@ bool parseJsonMessageRT(const std::string& body, std::string& devSeries, std::st
//回复消息
//执行结果直接看实时数据，不需要再回复，1是收到消息
send_reply_to_kafka(guid,"1","收到三秒数据指令");
send_reply_to_kafka("11111","1","收到三秒数据指令");//实时数据没有下发guid
} else {
std::cerr << "Missing expected fields in JSON message." << std::endl;
@@ -1471,14 +1477,14 @@ int parse_log(const std::string& json_str) {
return 0;
}
DIY_INFOLOG_CODE("process",0,LOG_CODE_LOG_REQUEST,"【NORMAL】前置的%s%d号进程处理日志上送消息",get_front_msg_from_subdir(), g_front_seg_index);
DIY_INFOLOG_CODE("process",0,LOG_CODE_LOG_REQUEST,"【NORMAL】前置的%s%d号进程处理日志控制消息",get_front_msg_from_subdir(), g_front_seg_index);
//进程号和匹配上
std::cout << "msg index:"<< processNo <<" self index:" << g_front_seg_index << std::endl;
std::cout << "msg frontType:"<< frontType <<" self frontType:" << subdir << std::endl;
//回复消息
send_reply_to_kafka(guid,"1","收到实时日志指令");
send_reply_to_kafka(guid,"1","收到日志控制指令");
if (code_str == "set_log") {
//校验数据
@@ -1492,7 +1498,7 @@ int parse_log(const std::string& json_str) {
}
else if((level == "measurepoint") && (grade == "TRACE") && (!id.empty() && !is_blank(id))){ //数据追踪
//打开监测点数据追踪开关
process_trace_command(id,5); //5表示追踪次数
process_trace_command(id,3); //3表示追踪次数
}
else{
std::cout << "type doesnt match" <<std::endl;
@@ -2320,6 +2326,7 @@ void HandleFileDirReqForChannel(chnl_usr_t *chnl_usr)
/* 统一回 Kafka */
Ckafka_data_t dir_info;
dir_info.strTopic = QString::fromStdString(Topic_Reply_Topic);
dir_info.mp_id = QString::fromLocal8Bit(req->guid);
dir_info.strText = QString::fromStdString(jsonString);
kafka_data_list_mutex.lock();
@@ -2373,21 +2380,45 @@ char* find_mp_name_from_mp_id(const char* mp_id)
}
int myMessageCallbackrtdata(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbackrtdata(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbackrtdata(
const rocketmq::MQMessageExt& msg)
{
if(INITFLAG != 1)return 1;//防止崩溃
if (msg == NULL) {
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
else{
//记录日志
@@ -2395,7 +2426,8 @@ int myMessageCallbackrtdata(CPushConsumer* consumer, CMessageExt* msg)
// 处理消息（例如，打印消息内容）
std::cout << "rt data Callback received message: " << body << std::endl;
if (key) {
//if (key) {
if (!key.empty()) {
std::cout << "Message Key: " << key << std::endl;
}
else {
@@ -2411,7 +2443,8 @@ int myMessageCallbackrtdata(CPushConsumer* consumer, CMessageExt* msg)
std::cerr << "Failed to parse the JSON message." << std::endl;
//记录日志
DIY_ERRORLOG_CODE("process",0,LOG_CODE_RT_DATA,"【ERROR】前置消费topic:%s_%s的实时触发消息失败,消息的json格式不正确",FRONT_INST.c_str(),G_MQCONSUMER_TOPIC_RT.c_str());
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
//mq处理实时数据指令查询台账时添加锁
@@ -2424,42 +2457,70 @@ int myMessageCallbackrtdata(CPushConsumer* consumer, CMessageExt* msg)
if(dev_index == 0 || mp_index == 0){
std::cerr << "dev index or mp index is 0" << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
// 创建 XML 文件
if (!createXmlFile(dev_index, mp_index, realData, soeData, limit,"new")) {
DIY_ERRORLOG_CODE("process",0,LOG_CODE_RT_DATA,"【ERROR】前置无法创建实时数据触发文件");
std::cerr << "Failed to create the XML file." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
}
// 根据业务逻辑决定返回状态
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
int myMessageCallbackupdate(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbackupdate(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbackupdate(
const rocketmq::MQMessageExt& msg)
{
if(INITFLAG != 1)return 1;//防止崩溃
if (msg == NULL) {
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is empty." << std::endl;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
else{
//处理消费数据
std::cout << "ledger update Callback received message: " << body << std::endl;
if (key) {
//if (key) {
if (!key.empty()) {
std::cout << "Message Key: " << key << std::endl;
}
else {
@@ -2481,28 +2542,55 @@ int myMessageCallbackupdate(CPushConsumer* consumer, CMessageExt* msg)
}
// 根据业务逻辑决定返回状态
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
int myMessageCallbackset(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbackset(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbackset(
const rocketmq::MQMessageExt& msg)
{
if(INITFLAG != 1)return 1;//防止崩溃
if (msg == NULL) {
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
else{
//处理消费数据
std::cout << "process Callback received message: " << body << std::endl;
if (key) {
//if (key) {
if (!key.empty()) {
std::cout << "Message Key: " << key << std::endl;
}
else {
@@ -2517,28 +2605,55 @@ int myMessageCallbackset(CPushConsumer* consumer, CMessageExt* msg)
}
// 根据业务逻辑决定返回状态
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
int myMessageCallbacklog(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbacklog(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbacklog(
const rocketmq::MQMessageExt& msg)
{
if(INITFLAG != 1)return 1;//防止崩溃
if (msg == NULL) {
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
else{
//处理消费数据
std::cout << "process Callback received message: " << body << std::endl;
if (key) {
//if (key) {
if (!key.empty()) {
std::cout << "Message Key: " << key << std::endl;
}
else {
@@ -2553,38 +2668,65 @@ int myMessageCallbacklog(CPushConsumer* consumer, CMessageExt* msg)
}
// 根据业务逻辑决定返回状态
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
int myMessageCallbackrecall(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbackrecall(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbackrecall(
const rocketmq::MQMessageExt& msg)
{
if(INITFLAG != 1)return 1;//防止崩溃
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
//调试
std::cout << "myMessageCallbackrecall"<< std::endl;
if (msg == NULL) {
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
else{
// 处理消息（例如，打印消息内容）
std::cout << "recall Callback received message: " << body << std::endl;
if (key) {
//if (key) {
if (!key.empty()) {
std::cout << "Message Key: " << key << std::endl;
}
else {
std::cout << "Message Key: N/A" << std::endl;
}
//处理消费数据
std::string result = extractDataJson(body); // 使用 std::string 代替 malloc
std::string result = extractDataJson(body.c_str()); // 使用 std::string 代替 malloc
//调试
std::cout << "extractDataJson:"<< result.c_str() <<std::endl;
@@ -2602,37 +2744,63 @@ int myMessageCallbackrecall(CPushConsumer* consumer, CMessageExt* msg)
}
// 根据业务逻辑决定返回状态
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
int myMessageCallbackfile(CPushConsumer* consumer, CMessageExt* msg)
//int myMessageCallbackfile(CPushConsumer* consumer, CMessageExt* msg)
ConsumeStatus myMessageCallbackfile(
const rocketmq::MQMessageExt& msg)
{
if (INITFLAG != 1) return 1;
if (msg == NULL) {
if (!should_process_after_start(msg)) {
std::cout << "[MQ] skip old message: "
<< "topic=" << msg.getTopic()
<< ", queueId=" << msg.getQueueId()
<< ", offset=" << msg.getQueueOffset()
<< ", bornTs=" << msg.getBornTimestamp()
<< ", appStart=" << G_APP_START_MS
<< std::endl;
return rocketmq::CONSUME_SUCCESS;
}
/*if (msg == NULL) {
std::cerr << "Received null message." << std::endl;
return E_RECONSUME_LATER;
}
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}*/
if (msg.getBody().empty()) {
std::cout << "empty msg body" << std::endl;
}
const char* body = GetMessageBody(msg);
const char* key = GetMessageKeys(msg);
//const char* body = GetMessageBody(msg);
//const char* key = GetMessageKeys(msg);
std::string body = msg.getBody();
std::string key = msg.getKeys();
std::string tag = msg.getTags();
std::string topic = msg.getTopic();
if (body == NULL) {
//if (body == NULL) {
if (body.empty()) {
std::cerr << "Message body is NULL." << std::endl;
return E_RECONSUME_LATER;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
DIY_INFOLOG_CODE("process",0,LOG_CODE_FILE_CONTROL,"【NORMAL】前置消费topic:%s_%s的文件控制消息",
FRONT_INST.c_str(), G_MQCONSUMER_TOPIC_FILE.c_str());
std::cout << "file Callback received message: " << body << std::endl;
std::cout << "Message Key: " << (key ? key : "N/A") << std::endl;
std::cout << "Message Key: " << (key.empty() ? "N/A" : key) << std::endl;
file_dir_req_t req;
if (ParseFileDirReq(body, &req) != 0)
if (ParseFileDirReq(body.c_str(), &req) != 0)
{
DIY_WARNLOG_CODE(req.devid,1,LOG_CODE_FILE_CONTROL,"【WARN】文件控制消息解析失败: %s", body);
return E_CONSUME_SUCCESS;
//return E_RECONSUME_LATER;
return rocketmq::RECONSUME_LATER;
}
PushFileDirReq(&req);
@@ -2641,13 +2809,15 @@ int myMessageCallbackfile(CPushConsumer* consumer, CMessageExt* msg)
"【NORMAL】文件目录请求已入队 guid=%s devid=%s path=%s",
req.guid, req.devid, req.path);
return E_CONSUME_SUCCESS;
//return E_CONSUME_SUCCESS;
return rocketmq::CONSUME_SUCCESS;
}
//lnk20260424修改消费者topic去除id，id在配置文件中放入每个topic的tag
void mqconsumerThread::run()
{
// 配置消费者参数
std::string consumerName = subdir + intToString(g_front_seg_index) + "_start_" + QDateTime::currentDateTime().toString("yyyyMMddhhmmss").toStdString(); // 消费者组ID+启动时间，不消费历史消息
std::string consumerName = G_ROCKETMQ_CONSUMER;//配置文件的消费者名称
std::string nameServer = G_MQCONSUMER_IPPORT; // NameServer地址
@@ -2657,23 +2827,23 @@ void mqconsumerThread::run()
if(g_node_id == THREE_SECS_DATA_BASE_NODE_ID){
//lnk20260310添加文件管理
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_FILE, G_MQCONSUMER_TAG_FILE, myMessageCallbackfile));
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_RT, G_MQCONSUMER_TAG_RT, myMessageCallbackrtdata));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_FILE, G_MQCONSUMER_TAG_FILE, myMessageCallbackfile));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_RT, G_MQCONSUMER_TAG_RT, myMessageCallbackrtdata));
}
// 初始化消费者2 //所有进程都会订阅台账更新topic，不同功能进程的台账不能互相影响
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_UD, G_MQCONSUMER_TAG_UD, myMessageCallbackupdate));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_UD, G_MQCONSUMER_TAG_UD, myMessageCallbackupdate));
// 初始化消费者3 //lnk20241230只有补招进程会订阅补招topic，不订阅补招topic的进程无法触发补招数据
if(g_node_id == RECALL_HIS_DATA_BASE_NODE_ID){
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_RC, G_MQCONSUMER_TAG_RC, myMessageCallbackrecall));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_RC, G_MQCONSUMER_TAG_RC, myMessageCallbackrecall));
}
// 初始化消费者4 //lnk20250108只有稳态进程1会控制reset，
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_SET, G_MQCONSUMER_TAG_SET, myMessageCallbackset));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_SET, G_MQCONSUMER_TAG_SET, myMessageCallbackset));
// 初始化消费者5 //所有进程都会订阅日志上送topic，不同功能进程的日志上送不能互相影响
subscriptions.push_back(Subscription(std::string(FRONT_INST) + "_" + G_MQCONSUMER_TOPIC_LOG, G_MQCONSUMER_TAG_LOG, myMessageCallbacklog));
subscriptions.push_back(Subscription(G_MQCONSUMER_TOPIC_LOG, G_MQCONSUMER_TAG_LOG, myMessageCallbacklog));
try {
rocketmq_consumer_receive(consumerName, nameServer, subscriptions);