xinyin025/hotgo
1
0
Fork 0
mirror of https://github.com/bufanyun/hotgo.git synced 2025-11-08 10:13:47 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

版本预发布

Browse Source
This commit is contained in:
孟帅
2023-02-08 20:29:34 +08:00
parent f11c7c5bf2
commit 2068d05c93
269 changed files with 16122 additions and 12075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
server/internal/library/queue/init.go
View File

@@ -79,8 +79,9 @@ var (
func init() {
mqProducerInstanceMap = make(map[string]MqProducer)
mqConsumerInstanceMap = make(map[string]MqConsumer)
get := g.Cfg().MustGet(ctx, "queue")
get.Scan(&config)
if err := g.Cfg().MustGet(ctx, "queue").Scan(&config); err != nil {
g.Log().Infof(ctx, "queue init err:%+v", err)
}
}
// InstanceConsumer 实例化消费者
@@ -100,30 +101,34 @@ func NewProducer(groupName string) (mqClient MqProducer, err error) {
}
if groupName == "" {
return mqClient, gerror.New("mq groupName is empty.")
err = gerror.New("mq groupName is empty.")
return
}
switch config.Driver {
case "rocketmq":
if len(config.Rocketmq.Address) == 0 {
g.Log().Fatal(ctx, "queue rocketmq address is not support")
err = gerror.New("queue rocketmq address is not support")
return
}
mqClient = RegisterRocketProducerMust(config.Rocketmq.Address, groupName, config.Retry)
mqClient, err = RegisterRocketProducer(config.Rocketmq.Address, groupName, config.Retry)
case "kafka":
if len(config.Kafka.Address) == 0 {
g.Log().Fatal(ctx, "queue kafka address is not support")
err = gerror.New("queue kafka address is not support")
return
}
mqClient = RegisterKafkaProducerMust(KafkaConfig{
mqClient, err = RegisterKafkaProducer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
})
case "redis":
address := g.Cfg().MustGet(ctx, "queue.redis.address", nil)
if len(address.String()) == 0 {
g.Log().Fatal(ctx, "queue redis address is not support")
address := g.Cfg().MustGet(ctx, "queue.redis.address", nil).String()
if len(address) == 0 {
err = gerror.New("queue redis address is not support")
return
}
mqClient = RegisterRedisMqProducerMust(RedisOption{
mqClient, err = RegisterRedisMqProducer(RedisOption{
Addr: config.Redis.Address,
Passwd: config.Redis.Pass,
DBnum: config.Redis.Db,
@@ -133,14 +138,18 @@ func NewProducer(groupName string) (mqClient MqProducer, err error) {
}, groupName, config.Retry)
default:
g.Log().Fatal(ctx, "queue driver is not support")
err = gerror.New("queue driver is not support")
}
if err != nil {
return
}
mutex.Lock()
defer mutex.Unlock()
mqProducerInstanceMap[groupName] = mqClient
return mqClient, nil
return
}
// NewConsumer 初始化消费者实例
@@ -157,18 +166,21 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
}
if groupName == "" {
return mqClient, gerror.New("mq groupName is empty.")
err = gerror.New("mq groupName is empty.")
return
}
switch config.Driver {
case "rocketmq":
if len(config.Rocketmq.Address) == 0 {
return nil, gerror.New("queue.rocketmq.address is empty.")
err = gerror.New("queue.rocketmq.address is empty.")
return
}
mqClient = RegisterRocketConsumerMust(config.Rocketmq.Address, groupName)
mqClient, err = RegisterRocketConsumer(config.Rocketmq.Address, groupName)
case "kafka":
if len(config.Kafka.Address) == 0 {
g.Log().Fatal(ctx, "queue kafka address is not support")
err = gerror.New("queue kafka address is not support")
return
}
clientId := "HOTGO-Consumer-" + groupName
@@ -176,7 +188,7 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
clientId += "-" + randTag
}
mqClient = RegisterKafkaMqConsumerMust(KafkaConfig{
mqClient, err = RegisterKafkaMqConsumer(KafkaConfig{
Brokers: config.Kafka.Address,
GroupID: groupName,
Version: config.Kafka.Version,
@@ -184,10 +196,11 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
})
case "redis":
if len(config.Redis.Address) == 0 {
g.Log().Fatal(ctx, "queue redis address is not support")
err = gerror.New("queue redis address is not support")
return
}
mqClient = RegisterRedisMqConsumerMust(RedisOption{
mqClient, err = RegisterRedisMqConsumer(RedisOption{
Addr: config.Redis.Address,
Passwd: config.Redis.Pass,
DBnum: config.Redis.Db,
@@ -196,14 +209,18 @@ func NewConsumer(groupName string) (mqClient MqConsumer, err error) {
5, 50, 5,
}, groupName)
default:
g.Log().Fatal(ctx, "queue driver is not support")
err = gerror.New("queue driver is not support")
}
if err != nil {
return
}
mutex.Lock()
defer mutex.Unlock()
mqConsumerInstanceMap[groupName] = mqClient
return mqClient, nil
return
}
// BodyString 返回消息体

109
server/internal/library/queue/kafkamq.go
View File

@@ -48,12 +48,13 @@ func (r *KafkaMq) SendByteMsg(topic string, body []byte) (mqMsg MqMsg, err error
}
if r.producerIns == nil {
return mqMsg, gerror.New("queue kafka producerIns is nil")
err = gerror.New("queue kafka producerIns is nil")
return
}
r.producerIns.Input() <- msg
ctx, cancle := context.WithTimeout(context.Background(), 5*time.Second)
defer cancle()
sendCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
select {
case info := <-r.producerIns.Successes():
@@ -68,7 +69,7 @@ func (r *KafkaMq) SendByteMsg(topic string, body []byte) (mqMsg MqMsg, err error
if nil != fail {
return mqMsg, fail.Err
}
case <-ctx.Done():
case <-sendCtx.Done():
return mqMsg, gerror.New("send mqMst timeout")
}
@@ -86,22 +87,23 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
receiveDoFun: receiveDo,
}
ctx, cancel := context.WithCancel(context.Background())
go func(ctx context.Context) {
consumerCtx, cancel := context.WithCancel(context.Background())
go func(consumerCtx context.Context) {
for {
if err := r.consumerIns.Consume(ctx, []string{topic}, &consumer); err != nil {
if err = r.consumerIns.Consume(consumerCtx, []string{topic}, &consumer); err != nil {
g.Log().Fatalf(ctx, "kafka Error from consumer, err%+v", err)
}
if ctx.Err() != nil {
g.Log().Debugf(ctx, fmt.Sprintf("kafka consoumer stop : %v", ctx.Err()))
if consumerCtx.Err() != nil {
g.Log().Debugf(ctx, fmt.Sprintf("kafka consoumer stop : %v", consumerCtx.Err()))
return
}
consumer.ready = make(chan bool)
}
}(ctx)
}(consumerCtx)
<-consumer.ready // Await till the consumer has been set up
// await till the consumer has been set up
<-consumer.ready
g.Log().Debug(ctx, "kafka consumer up and running!...")
signal.AppDefer(func() {
@@ -115,85 +117,94 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
return
}
// RegisterKafkaMqConsumerMust 注册消费者
func RegisterKafkaMqConsumerMust(connOpt KafkaConfig) (client MqConsumer) {
// RegisterKafkaMqConsumer 注册消费者
func RegisterKafkaMqConsumer(connOpt KafkaConfig) (client MqConsumer, err error) {
mqIns := &KafkaMq{}
kfkVersion, _ := sarama.ParseKafkaVersion(connOpt.Version)
kfkVersion, err := sarama.ParseKafkaVersion(connOpt.Version)
if err != nil {
return
}
if validateVersion(kfkVersion) == false {
kfkVersion = sarama.V2_4_0_0
}
brokers := connOpt.Brokers
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
config.Version = kfkVersion
conf := sarama.NewConfig()
conf.Consumer.Return.Errors = true
conf.Version = kfkVersion
if connOpt.UserName != "" {
config.Net.SASL.Enable = true
config.Net.SASL.User = connOpt.UserName
config.Net.SASL.Password = connOpt.Password
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
}
// 默认按随机方式消费
config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
config.Consumer.Offsets.Initial = sarama.OffsetNewest
config.Consumer.Offsets.AutoCommit.Interval = 10 * time.Millisecond
config.ClientID = connOpt.ClientId
conf.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
conf.Consumer.Offsets.Initial = sarama.OffsetNewest
conf.Consumer.Offsets.AutoCommit.Interval = 10 * time.Millisecond
conf.ClientID = connOpt.ClientId
consumerClient, err := sarama.NewConsumerGroup(brokers, connOpt.GroupID, config)
consumerClient, err := sarama.NewConsumerGroup(brokers, connOpt.GroupID, conf)
if err != nil {
g.Log().Fatal(ctx, err)
return
}
mqIns.consumerIns = consumerClient
return mqIns
return mqIns, err
}
// RegisterKafkaProducerMust 注册并启动生产者接口实现
func RegisterKafkaProducerMust(connOpt KafkaConfig) (client MqProducer) {
// RegisterKafkaProducer 注册并启动生产者接口实现
func RegisterKafkaProducer(connOpt KafkaConfig) (client MqProducer, err error) {
mqIns := &KafkaMq{}
connOpt.ClientId = "HOTGO-Producer"
RegisterKafkaProducer(connOpt, mqIns) //这里如果使用go程需要处理chan同步问题
return mqIns
// 这里如果使用go程需要处理chan同步问题
if err = doRegisterKafkaProducer(connOpt, mqIns); err != nil {
return nil, err
}
return mqIns, nil
}
// RegisterKafkaProducer 注册同步类型实例
func RegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) {
kfkVersion, _ := sarama.ParseKafkaVersion(connOpt.Version)
// doRegisterKafkaProducer 注册同步类型实例
func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
kfkVersion, err := sarama.ParseKafkaVersion(connOpt.Version)
if err != nil {
return
}
if validateVersion(kfkVersion) == false {
kfkVersion = sarama.V2_4_0_0
}
brokers := connOpt.Brokers
config := sarama.NewConfig()
conf := sarama.NewConfig()
// 等待服务器所有副本都保存成功后的响应
config.Producer.RequiredAcks = sarama.WaitForAll
conf.Producer.RequiredAcks = sarama.WaitForAll
// 随机向partition发送消息
config.Producer.Partitioner = sarama.NewRandomPartitioner
conf.Producer.Partitioner = sarama.NewRandomPartitioner
// 是否等待成功和失败后的响应,只有上面的RequireAcks设置不是NoReponse这里才有用.
config.Producer.Return.Successes = true
conf.Producer.Return.Successes = true
config.Producer.Return.Errors = true
config.Producer.Compression = sarama.CompressionNone
config.ClientID = connOpt.ClientId
conf.Producer.Return.Errors = true
conf.Producer.Compression = sarama.CompressionNone
conf.ClientID = connOpt.ClientId
config.Version = kfkVersion
conf.Version = kfkVersion
if connOpt.UserName != "" {
config.Net.SASL.Enable = true
config.Net.SASL.User = connOpt.UserName
config.Net.SASL.Password = connOpt.Password
conf.Net.SASL.Enable = true
conf.Net.SASL.User = connOpt.UserName
conf.Net.SASL.Password = connOpt.Password
}
var err error
mqIns.producerIns, err = sarama.NewAsyncProducer(brokers, config)
mqIns.producerIns, err = sarama.NewAsyncProducer(brokers, conf)
if err != nil {
g.Log().Fatal(ctx, err)
return
}
signal.AppDefer(func() {
g.Log().Debug(ctx, "kafka producer AsyncClose...")
mqIns.producerIns.AsyncClose()
})
return
}
// validateVersion 验证版本是否有效

4
server/internal/library/queue/push.go
View File

@@ -1,7 +1,6 @@
package queue
import (
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
@@ -9,8 +8,7 @@ import (
func Push(topic string, data interface{}) (err error) {
q, err := InstanceProducer()
if err != nil {
g.Log().Fatalf(ctx, "queue.InstanceProducer err:%+v", err)
return err
return
}
mqMsg, err := q.SendMsg(topic, gconv.String(data))
ProducerLog(ctx, topic, mqMsg.MsgId, err)

31
server/internal/library/queue/redismq.go
View File

@@ -146,25 +146,23 @@ func (r *RedisMq) loopReadQueue(queueName string) (mqMsgList []MqMsg) {
return mqMsgList
}
func RegisterRedisMqProducerMust(connOpt RedisOption, poolOpt PoolOption, groupName string, retry int) (client MqProducer) {
var err error
func RegisterRedisMqProducer(connOpt RedisOption, poolOpt PoolOption, groupName string, retry int) (client MqProducer, err error) {
client, err = RegisterRedisMq(connOpt, poolOpt, groupName, retry)
if err != nil {
g.Log().Fatal(ctx, "RegisterRedisMqProducerMust err:%+v", err)
err = gerror.Newf("RegisterRedisMqProducer err:%+v", err)
return
}
return client
return
}
// RegisterRedisMqConsumerMust 注册消费者
func RegisterRedisMqConsumerMust(connOpt RedisOption, poolOpt PoolOption, groupName string) (client MqConsumer) {
var err error
// RegisterRedisMqConsumer 注册消费者
func RegisterRedisMqConsumer(connOpt RedisOption, poolOpt PoolOption, groupName string) (client MqConsumer, err error) {
client, err = RegisterRedisMq(connOpt, poolOpt, groupName, 0)
if err != nil {
g.Log().Fatal(ctx, "RegisterRedisMqConsumerMust err:%+v", err)
err = gerror.Newf("RegisterRedisMqConsumer err:%+v", err)
return
}
return client
return
}
// RegisterRedisMq 注册redis实例
@@ -200,12 +198,12 @@ func registerRedis(host, pass string, dbNum int, opt PoolOption) (poolName strin
return nil, err
}
if pass != "" {
if _, err := conn.Do("AUTH", pass); err != nil {
if _, err = conn.Do("AUTH", pass); err != nil {
return nil, err
}
}
if dbNum > 0 {
if _, err := conn.Do("SELECT", dbNum); err != nil {
if _, err = conn.Do("SELECT", dbNum); err != nil {
return nil, err
}
}
@@ -270,19 +268,20 @@ func getRedis(poolName string, retry int) (db redis.Conn, put func(), err error)
if err != nil {
return nil, put, err
}
put = func() {
redisPool.Put(conn)
if err = redisPool.Put(conn); err != nil {
return
}
}
db = conn.(redis.Conn)
return db, put, nil
}
func getRandMsgId() (msgId string) {
func getRandMsgId() string {
rand.Seed(time.Now().UnixNano())
radium := rand.Intn(999) + 1
timeCode := time.Now().UnixNano()
msgId = fmt.Sprintf("%d%.4d", timeCode, radium)
return msgId
return fmt.Sprintf("%d%.4d", timeCode, radium)
}

24
server/internal/library/queue/rocketmq.go
View File

@@ -26,30 +26,27 @@ type RocketMq struct {
// rewriteLog 重写日志
func rewriteLog() {
level := g.Cfg().MustGet(ctx, "queue.rocketmq.logLevel", "debug").String()
rlog.SetLogger(&RocketMqLogger{Flag: "[rocket_mq]", LevelLog: level})
rlog.SetLogger(&RocketMqLogger{Flag: "[rocket_mq]", LevelLog: g.Cfg().MustGet(ctx, "queue.rocketmq.logLevel", "debug").String()})
}
// RegisterRocketProducerMust 注册并启动生产者接口实现
func RegisterRocketProducerMust(endPoints []string, groupName string, retry int) (client MqProducer) {
// RegisterRocketProducer 注册并启动生产者接口实现
func RegisterRocketProducer(endPoints []string, groupName string, retry int) (client MqProducer, err error) {
rewriteLog()
var err error
client, err = RegisterRocketMqProducer(endPoints, groupName, retry)
if err != nil {
panic(err)
return
}
return client
return
}
// RegisterRocketConsumerMust 注册消费者
func RegisterRocketConsumerMust(endPoints []string, groupName string) (client MqConsumer) {
// RegisterRocketConsumer 注册消费者
func RegisterRocketConsumer(endPoints []string, groupName string) (client MqConsumer, err error) {
rewriteLog()
var err error
client, err = RegisterRocketMqConsumer(endPoints, groupName)
if err != nil {
panic(err)
return
}
return client
return
}
// SendMsg 按字符串类型生产数据
@@ -90,8 +87,7 @@ func (r *RocketMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
return errors.New("RocketMq consumer not register")
}
err = r.consumerIns.Subscribe(topic, consumer.MessageSelector{}, func(ctx context.Context,
msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
err = r.consumerIns.Subscribe(topic, consumer.MessageSelector{}, func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
for _, item := range msgs {
go receiveDo(MqMsg{
RunType: ReceiveMsg,