pavement
/
data2minio
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

1、修改基础库版本

This commit is contained in:
wangjian 2023-04-02 23:35:12 +08:00
parent 32bad0506b
commit a7482bb81a
7 changed files with 76 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/server.go
View File

@ -2,6 +2,8 @@ package cmd
import (
"context"
"data_minio/minio"
"data_minio/mq"
"fmt"
"github.com/spf13/cobra"
"os"
@ -13,7 +15,6 @@ import (
"data_minio/config"
"data_minio/global"
"data_minio/model"
"data_minio/mq"
)
var (
@ -65,12 +66,13 @@ func NewStartCmd() *cobra.Command {
}
global.Cfg = cfg
global.Logger = config.LoadLoggerConfig(cfg.Logging)
//创建消息连接点
mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node)
global.MinioClient, err = minio.NewMinioClient(cfg.Minio.Endpoint, cfg.Minio.AccessKeyId, cfg.Minio.SecretAccessKey)
global.Must(err)
//连接数据库
global.DB = model.New(cfg.Db.DriveName, cfg.Db.Conn, cfg.Mode == "dev")
//创建消息连接点
mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node)
global.Must(err)
// 退出channel
exitChannel := make(chan os.Signal)
defer close(exitChannel)

10
global/global.go
View File

@ -2,6 +2,7 @@ package global
import (
"fmt"
"github.com/minio/minio-go/v7"
"os"
"git.hpds.cc/Component/logging"
@ -12,9 +13,10 @@ import (
)
var (
DB *xorm.Engine
Cfg *config.DataMinioConfig
Logger *logging.Logger
DB *xorm.Engine
Cfg *config.DataMinioConfig
Logger *logging.Logger
MinioClient *minio.Client
)
func Must(err error) {
@ -23,6 +25,6 @@ func Must(err error) {
Logger.With(
zap.String("应用名称", Cfg.Name),
).Error(err.Error())
os.Exit(1)
//os.Exit(1)
}
}

4
go.mod
View File

@ -4,7 +4,7 @@ go 1.19
require (
git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b
git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052
git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930
github.com/go-sql-driver/mysql v1.7.0
github.com/hashicorp/consul/api v1.20.0
github.com/klauspost/compress v1.16.3
@ -23,7 +23,7 @@ require (
cloud.google.com/go/firestore v1.9.0 // indirect
cloud.google.com/go/longrunning v0.3.0 // indirect
git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 // indirect
git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86 // indirect
git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c // indirect
github.com/armon/go-metrics v0.4.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect

8
go.sum
View File

@ -50,10 +50,10 @@ git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b h1:YyC4oRJbB+bU
git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b/go.mod h1:FUm2QSkHtinWaXyLdrzvIO9Wl4zNF+riur8wQ9xbEJY=
git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 h1:A5cB4upzFkatySn2uT7hKdDXoZkr7AKIKahzjVD55bQ=
git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340/go.mod h1:NSyd8CKLY6HFXf/7qUUvPKOjvIeRNEqp8TwpZ8rpEsg=
git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86 h1:ST2VBwjQlbPBpX8VnF/Y7zMONX2bEGFvqbCLQJ11BVE=
git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86/go.mod h1:OnMupEWYrkS/WR19m0+iA7lVCaEkbG4WPb2XKCOvhIg=
git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052 h1:uvV16dXYjffd9/T/esyS5w+WLqF7rSXU3VmuGckIxmg=
git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052/go.mod h1:oXZYfH6kVVuIEBukzodUeEDsYoREUzMJ1Wc4a4yRftE=
git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c h1:8cm6OnOtiTFJ3LPr9nNXO3fy7yDL1W4/HJRCIvO6PHk=
git.hpds.cc/Component/network v0.0.0-20230402152327-d10a80765e7c/go.mod h1:OnMupEWYrkS/WR19m0+iA7lVCaEkbG4WPb2XKCOvhIg=
git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930 h1:XJkWwOtBCIN5bBViAxv8MNhzodd6eBBfDt5Vs1J1XU0=
git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930/go.mod h1:E1NJu3aBpwLgYaZiCvaIwQ8V+8xgoaVzTN9hlvRlHbw=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE=

14
minio/index.go
View File

@ -1,23 +1,15 @@
package minio
import (
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"data_minio/global"
)
func NewMinioClient(endpoint, successKeyId, secretAccessKey string) *minio.Client {
func NewMinioClient(endpoint, successKeyId, secretAccessKey string) (*minio.Client, error) {
opt := &minio.Options{
Creds: credentials.NewStaticV4(successKeyId, secretAccessKey, ""),
Secure: false,
}
minioClient, err := minio.New(endpoint, opt)
if err != nil {
global.Logger.With(
zap.String("应用名称", Cfg.Name),
).Error(err.Error())
return nil
}
return minioClient
return minioClient, err
}

15
model/dataset.go
View File

@ -30,3 +30,18 @@ func GetDatasetById(id int64) (data *Dataset, err error) {
}
return data, nil
}
func GetDatasetScene(categoryId int) string {
switch categoryId {
case 1:
return "road"
case 2:
return "bridge"
case 3:
return "tunnel"
case 4:
return "sideSlope"
default:
return "other"
}
}

55
mq/index.go
View File

@ -1,8 +1,12 @@
package mq
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/minio/minio-go/v7"
"os"
"time"
@ -101,22 +105,43 @@ func handler(data []byte) (byte, []byte) {
switch req.Command {
case DatasetRequest:
payload := req.Payload.(map[string]interface{})
dt, err := model.GetDatasetById(int64(payload["datasetId"].(float64)))
global.Must(err)
fileContent, _ := UnCompress([]byte(payload["file"].(string)))
if int64(payload["fileSize"].(float64)) > 0 {
dt, err := model.GetDatasetById(int64(payload["datasetId"].(float64)))
global.Must(err)
var fileContent []byte
if payload["isCompress"].(bool) {
decodeBytes, _ := base64.StdEncoding.DecodeString(payload["file"].(string))
fileContent, err = UnCompress(decodeBytes)
} else {
fileContent = []byte(payload["file"].(string))
}
if err != nil {
global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.Int64("文件大小", int64(payload["fileSize"].(float64)))).
Error("错误信息", zap.Error(err))
}
fileReader := bytes.NewReader(fileContent)
info, err := global.MinioClient.PutObject(context.Background(), global.Cfg.Minio.Bucket, fmt.Sprintf("%s%s", model.GetDatasetScene(dt.CategoryId), payload["filePath"].(string)), fileReader, int64(payload["fileSize"].(float64)), minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil {
global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.Int64("文件大小", int64(payload["fileSize"].(float64)))).
Error("错误信息", zap.Error(err))
}
accessUrl := fmt.Sprintf("%s://%s/%s/%s%s", global.Cfg.Minio.Protocol, global.Cfg.Minio.Endpoint, global.Cfg.Minio.Bucket, model.GetDatasetScene(dt.CategoryId), payload["filePath"].(string))
fm := new(model.FileManager)
fm.FileName = payload["fileName"].(string)
//fm.AccessUrl = payload["filePath"].(string)
fm.Scene = fmt.Sprintf("%d", dt.CategoryId)
fm.DatasetId = dt.DatasetId
fm.FileSize = int64(payload["fileSize"].(float64))
fm.FileMd5 = payload["fileMd5"].(string)
fm.Creator = 0
fm.CreateAt = time.Now().Unix()
fm.UpdateAt = time.Now().Unix()
_, err = global.DB.Insert(fm)
global.Must(err)
global.Logger.With(zap.String("文件名称", payload["filePath"].(string))).With(zap.String("访问路径", accessUrl)).
With(zap.Any("返回数据", info)).Info("文件上传到minio")
fm := new(model.FileManager)
fm.FileName = payload["fileName"].(string)
fm.AccessUrl = accessUrl
fm.Scene = fmt.Sprintf("%d", dt.CategoryId)
fm.DatasetId = dt.DatasetId
fm.FileSize = int64(payload["fileSize"].(float64))
fm.FileMd5 = payload["fileMd5"].(string)
fm.Creator = 0
fm.CreateAt = time.Now().Unix()
fm.UpdateAt = time.Now().Unix()
_, err = global.DB.Insert(fm)
global.Must(err)
}
}
}
return 0x12, nil