From a7482bb81a365aedfa6f34460dc45746f4b6172b Mon Sep 17 00:00:00 2001 From: wangjian Date: Sun, 2 Apr 2023 23:35:12 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E4=BF=AE=E6=94=B9=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E5=BA=93=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/server.go | 10 ++++++---- global/global.go | 10 ++++++---- go.mod | 4 ++-- go.sum | 8 ++++---- minio/index.go | 14 +++----------- model/dataset.go | 15 +++++++++++++++ mq/index.go | 55 ++++++++++++++++++++++++++++++++++++++++--------------- 7 files changed, 76 insertions(+), 40 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 542f0ba..6ba2955 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -2,6 +2,8 @@ package cmd import ( "context" + "data_minio/minio" + "data_minio/mq" "fmt" "github.com/spf13/cobra" "os" @@ -13,7 +15,6 @@ import ( "data_minio/config" "data_minio/global" "data_minio/model" - "data_minio/mq" ) var ( @@ -65,12 +66,13 @@ func NewStartCmd() *cobra.Command { } global.Cfg = cfg global.Logger = config.LoadLoggerConfig(cfg.Logging) - - //创建消息连接点 - mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node) + global.MinioClient, err = minio.NewMinioClient(cfg.Minio.Endpoint, cfg.Minio.AccessKeyId, cfg.Minio.SecretAccessKey) global.Must(err) //连接数据库 global.DB = model.New(cfg.Db.DriveName, cfg.Db.Conn, cfg.Mode == "dev") + //创建消息连接点 + mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node) + global.Must(err) // 退出channel exitChannel := make(chan os.Signal) defer close(exitChannel) diff --git a/global/global.go b/global/global.go index 5166b9d..8145bd0 100644 --- a/global/global.go +++ b/global/global.go @@ -2,6 +2,7 @@ package global import ( "fmt" + "github.com/minio/minio-go/v7" "os" "git.hpds.cc/Component/logging" @@ -12,9 +13,10 @@ import ( ) var ( - DB *xorm.Engine - Cfg *config.DataMinioConfig - Logger *logging.Logger + DB *xorm.Engine + Cfg *config.DataMinioConfig + Logger *logging.Logger + MinioClient *minio.Client ) func Must(err error) { @@ -23,6 +25,6 @@ func Must(err error) { Logger.With( zap.String("应用名称", Cfg.Name), ).Error(err.Error()) - os.Exit(1) + //os.Exit(1) } } diff --git a/go.mod b/go.mod index 0da9363..34f5f9c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b - git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052 + git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930 github.com/go-sql-driver/mysql v1.7.0 github.com/hashicorp/consul/api v1.20.0 github.com/klauspost/compress v1.16.3 @@ -23,7 +23,7 @@ require ( cloud.google.com/go/firestore v1.9.0 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 // indirect - git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86 // indirect + git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c // indirect github.com/armon/go-metrics v0.4.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect diff --git a/go.sum b/go.sum index 1d7fb82..74cc827 100644 --- a/go.sum +++ b/go.sum @@ -50,10 +50,10 @@ git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b h1:YyC4oRJbB+bU git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b/go.mod h1:FUm2QSkHtinWaXyLdrzvIO9Wl4zNF+riur8wQ9xbEJY= git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 h1:A5cB4upzFkatySn2uT7hKdDXoZkr7AKIKahzjVD55bQ= git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340/go.mod h1:NSyd8CKLY6HFXf/7qUUvPKOjvIeRNEqp8TwpZ8rpEsg= -git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86 h1:ST2VBwjQlbPBpX8VnF/Y7zMONX2bEGFvqbCLQJ11BVE= -git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86/go.mod h1:OnMupEWYrkS/WR19m0+iA7lVCaEkbG4WPb2XKCOvhIg= -git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052 h1:uvV16dXYjffd9/T/esyS5w+WLqF7rSXU3VmuGckIxmg= -git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052/go.mod h1:oXZYfH6kVVuIEBukzodUeEDsYoREUzMJ1Wc4a4yRftE= +git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c h1:8cm6OnOtiTFJ3LPr9nNXO3fy7yDL1W4/HJRCIvO6PHk= +git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c/go.mod h1:OnMupEWYrkS/WR19m0+iA7lVCaEkbG4WPb2XKCOvhIg= +git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930 h1:XJkWwOtBCIN5bBViAxv8MNhzodd6eBBfDt5Vs1J1XU0= +git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930/go.mod h1:E1NJu3aBpwLgYaZiCvaIwQ8V+8xgoaVzTN9hlvRlHbw= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE= diff --git a/minio/index.go b/minio/index.go index 663cb5e..6dc19de 100644 --- a/minio/index.go +++ b/minio/index.go @@ -1,23 +1,15 @@ package minio import ( - minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - - "data_minio/global" ) -func NewMinioClient(endpoint, successKeyId, secretAccessKey string) *minio.Client { +func NewMinioClient(endpoint, successKeyId, secretAccessKey string) (*minio.Client, error) { opt := &minio.Options{ Creds: credentials.NewStaticV4(successKeyId, secretAccessKey, ""), Secure: false, } minioClient, err := minio.New(endpoint, opt) - if err != nil { - global.Logger.With( - zap.String("应用名称", Cfg.Name), - ).Error(err.Error()) - return nil - } - return minioClient + return minioClient, err } diff --git a/model/dataset.go b/model/dataset.go index 5d190e7..053eff1 100644 --- a/model/dataset.go +++ b/model/dataset.go @@ -30,3 +30,18 @@ func GetDatasetById(id int64) (data *Dataset, err error) { } return data, nil } + +func GetDatasetScene(categoryId int) string { + switch categoryId { + case 1: + return "road" + case 2: + return "bridge" + case 3: + return "tunnel" + case 4: + return "sideSlope" + default: + return "other" + } +} diff --git a/mq/index.go b/mq/index.go index 97a84b1..1cfa8c9 100644 --- a/mq/index.go +++ b/mq/index.go @@ -1,8 +1,12 @@ package mq import ( + "bytes" + "context" + "encoding/base64" "encoding/json" "fmt" + "github.com/minio/minio-go/v7" "os" "time" @@ -101,22 +105,43 @@ func handler(data []byte) (byte, []byte) { switch req.Command { case DatasetRequest: payload := req.Payload.(map[string]interface{}) - dt, err := model.GetDatasetById(int64(payload["datasetId"].(float64))) - global.Must(err) - fileContent, _ := UnCompress([]byte(payload["file"].(string))) + if int64(payload["fileSize"].(float64)) > 0 { + dt, err := model.GetDatasetById(int64(payload["datasetId"].(float64))) + global.Must(err) + var fileContent []byte + if payload["isCompress"].(bool) { + decodeBytes, _ := base64.StdEncoding.DecodeString(payload["file"].(string)) + fileContent, err = UnCompress(decodeBytes) + } else { + fileContent = []byte(payload["file"].(string)) + } + if err != nil { + global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.Int64("文件大小", int64(payload["fileSize"].(float64)))). + Error("错误信息", zap.Error(err)) + } + fileReader := bytes.NewReader(fileContent) + info, err := global.MinioClient.PutObject(context.Background(), global.Cfg.Minio.Bucket, fmt.Sprintf("%s%s", model.GetDatasetScene(dt.CategoryId), payload["filePath"].(string)), fileReader, int64(payload["fileSize"].(float64)), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.Int64("文件大小", int64(payload["fileSize"].(float64)))). + Error("错误信息", zap.Error(err)) + } + accessUrl := fmt.Sprintf("%s://%s/%s/%s%s", global.Cfg.Minio.Protocol, global.Cfg.Minio.Endpoint, global.Cfg.Minio.Bucket, model.GetDatasetScene(dt.CategoryId), payload["filePath"].(string)) - fm := new(model.FileManager) - fm.FileName = payload["fileName"].(string) - //fm.AccessUrl = payload["filePath"].(string) - fm.Scene = fmt.Sprintf("%d", dt.CategoryId) - fm.DatasetId = dt.DatasetId - fm.FileSize = int64(payload["fileSize"].(float64)) - fm.FileMd5 = payload["fileMd5"].(string) - fm.Creator = 0 - fm.CreateAt = time.Now().Unix() - fm.UpdateAt = time.Now().Unix() - _, err = global.DB.Insert(fm) - global.Must(err) + global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.String("访问路径", accessUrl)). + With(zap.Any("返回数据", info)).Info("文件上传到minio") + fm := new(model.FileManager) + fm.FileName = payload["fileName"].(string) + fm.AccessUrl = accessUrl + fm.Scene = fmt.Sprintf("%d", dt.CategoryId) + fm.DatasetId = dt.DatasetId + fm.FileSize = int64(payload["fileSize"].(float64)) + fm.FileMd5 = payload["fileMd5"].(string) + fm.Creator = 0 + fm.CreateAt = time.Now().Unix() + fm.UpdateAt = time.Now().Unix() + _, err = global.DB.Insert(fm) + global.Must(err) + } } } return 0x12, nil