From 15917e926eed43b7b975bc2be6a6e0fec1b11b69 Mon Sep 17 00:00:00 2001 From: wangjian Date: Fri, 12 May 2023 16:53:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 115 +++++++++++++++++++++++ cmd/server.go | 119 ++++++++++++++++++++++++ config/config.go | 73 +++++++++++++++ config/config.yaml | 27 ++++++ global/global.go | 39 ++++++++ go.mod | 72 +++++++++++++++ internal/handler/index.go | 55 +++++++++++ internal/middleware/cors.go | 24 +++++ internal/proto/request.go | 13 +++ internal/proto/response.go | 28 ++++++ internal/router/router.go | 33 +++++++ internal/service/index.go | 220 ++++++++++++++++++++++++++++++++++++++++++++ main.go | 26 ++++++ mq/handler.go | 103 +++++++++++++++++++++ mq/index.go | 199 +++++++++++++++++++++++++++++++++++++++ mq/instruction.go | 16 ++++ pkg/err/err_code.go | 83 +++++++++++++++++ pkg/err/err_encode.go | 58 ++++++++++++ pkg/utils/file.go | 88 ++++++++++++++++++ pkg/utils/http.go | 126 +++++++++++++++++++++++++ pkg/utils/image.go | 216 +++++++++++++++++++++++++++++++++++++++++++ pkg/utils/network.go | 40 ++++++++ store/index.go | 91 ++++++++++++++++++ 23 files changed, 1864 insertions(+) create mode 100644 README.md create mode 100644 cmd/server.go create mode 100644 config/config.go create mode 100644 config/config.yaml create mode 100644 global/global.go create mode 100644 go.mod create mode 100644 internal/handler/index.go create mode 100644 internal/middleware/cors.go create mode 100644 internal/proto/request.go create mode 100644 internal/proto/response.go create mode 100644 internal/router/router.go create mode 100644 internal/service/index.go create mode 100644 main.go create mode 100644 mq/handler.go create mode 100644 mq/index.go create mode 100644 mq/instruction.go create mode 100644 pkg/err/err_code.go create mode 100644 pkg/err/err_encode.go create mode 100644 pkg/utils/file.go create mode 100644 pkg/utils/http.go create mode 100644 pkg/utils/image.go create mode 100644 pkg/utils/network.go create mode 100644 store/index.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..d919815 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +# 边缘设备标注服务 + +## API + +### 获取目录文件列表 + +- 访问地址 + +``` +POST /api/directory/list +``` + +- 请求头参数 + +Content-Type : application/json + +- 请求参数 + +``` +{ + "path":"/home/goroot/hpds_annotation/logs/" +} +``` + +- 返回值 + + 主题包含的字段result 中的数据结构体，当前返回的是数组类型 + +| 序号 | 字段名称 | 数据类型 | 说明 | +|-----|--------|---------|------| +| 1 | name | string | 文件名 | +| 2 | path | string | 文件所在的路径 | +| 3 | isDir | bool | 是否文件夹 | +| 4 | size | int | 文件大小 | +| 5 | modTime | int | 文件最后修改日期 | + + +### 获取目录文件详情 + +- 访问地址 + +``` +POST /api/directory/info +``` + +- 请求头参数 + +Content-Type : application/json + +- 请求参数 + +``` +{ + "path":"/home/goroot/hpds_annotation/logs/hpds-edge-web-error.log" +} +``` + +- 返回值 + +主题包含的字段result 中的数据结构体，当前返回的是对象 + +| 序号 | 字段名称 | 数据类型 | 说明 | +|-----|--------|---------|------| +| 1 | name | string | 文件名 | +| 2 | path | string | 文件所在的路径 | +| 3 | isDir | bool | 是否文件夹 | +| 4 | size | int | 文件大小 | +| 5 | modTime | int | 文件最后修改日期 | +| 6 | contentBase | string | 文件内容，用base64进行编码 | +| 7 | labelStatus | int | 标注状态，0:未进行标注；1:有病害;2:无病害; | + + + +### 提交标注数据 + +- 访问地址 + +``` +POST /api/label/submit +``` + +- 请求头参数 + +Content-Type : application/json + +- 请求参数 + +``` +{ + "fileList": ["/home/data/bridge_capture/crack/0001.tif", "/home/data/bridge_capture/crack/0002.tif"], + "labelStatus": true, + "trainingSet": "5月10日隧道数据标注集", + "bizType": 3 +} +``` + + + 说明 + + +| 序号 | 字段名称 | 数据类型 | 说明 | +|-----|--------|-------|-------------------------------------| +| 1 | fileList | 字符串数组 | 文件的全路径组成的数组 | +| 2 | labelStatus | 布尔值 | 标注状态，true: 有病害; false: 无病害 | +| 3 | trainingSet | 字符串 | 训练集名称，上传到云端的训练集名称，如果云端已经存在，将合并训练集操作 | +| 4 | bizType | 整型 | 1: 道路; 2: 桥梁; 3: 隧道; 4: 边坡; | + +- 返回值 + +``` +{ + "code": 200, + "message": "成功", + "type": "OK" +} +``` \ No newline at end of file diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 0000000..e6a22b7 --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "context" + "fmt" + "git.hpds.cc/Component/logging" + "github.com/spf13/cobra" + "go.uber.org/zap" + "hpds_annotation/config" + "hpds_annotation/global" + router2 "hpds_annotation/internal/router" + "hpds_annotation/mq" + "hpds_annotation/store" + "net/http" + "os" + "os/signal" + "syscall" +) + +var ( + ConfigFileFlag string = "./config/config.yaml" + NodeName string = "edge-node" + Mode string = "dev" +) + +func must(err error) { + if err != nil { + _, _ = fmt.Fprint(os.Stderr, err) + os.Exit(1) + } +} + +func NewStartCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Start hpds_web application", + Run: func(cmd *cobra.Command, args []string) { + var ( + cfg *config.WebConfig + err error + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + must(err) + configFileFlag, err := cmd.Flags().GetString("c") + if err != nil { + fmt.Println("get local config err: ", err) + return + } + + NodeName, err = cmd.Flags().GetString("n") + if err != nil { + fmt.Println("get remote path config err: ", err) + return + } + Mode, err = cmd.Flags().GetString("m") + if err != nil { + fmt.Println("get remote path config err: ", err) + return + } + cfg, err = config.ParseConfigByFile(configFileFlag) + must(err) + global.Cfg = cfg + global.Logger = LoadLoggerConfig(cfg.Logging) + logger := LoadLoggerConfig(cfg.Logging) + global.FileLabelList = store.Load(cfg.TempPath) + + //创建消息连接点 + mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node, logger) + must(err) + // 退出channel + exitChannel := make(chan os.Signal) + defer close(exitChannel) + router := router2.InitRouter(cfg, logger) + // 退出信号监听 + go func(c chan os.Signal) { + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) + }(exitChannel) + go func() { + fmt.Printf("Http Server start at port %d \n", cfg.Port) + err = http.ListenAndServe(fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), router) + must(err) + }() + select { + case <-ctx.Done(): + store.Save(cfg.TempPath, global.FileLabelList) + logger.With( + zap.String("web", "exit"), + ).Error(ctx.Err().Error()) + return + case errs := <-exitChannel: + store.Save(cfg.TempPath, global.FileLabelList) + logger.With( + zap.String("web", "服务退出"), + ).Info(errs.String()) + return + } + }, + } + cmd.Flags().StringVar(&ConfigFileFlag, "c", "./config/config.yaml", "The configuration file path") + cmd.Flags().StringVar(&NodeName, "n", "main-node", "The configuration name") + cmd.Flags().StringVar(&Mode, "m", "dev", "run mode : dev | test | releases") + return cmd +} +func LoadLoggerConfig(opt config.LogOptions) *logging.Logger { + return logging.NewLogger( + logging.SetPath(opt.Path), + logging.SetPrefix(opt.Prefix), + logging.SetDevelopment(opt.Development), + logging.SetDebugFileSuffix(opt.DebugFileSuffix), + logging.SetWarnFileSuffix(opt.WarnFileSuffix), + logging.SetErrorFileSuffix(opt.ErrorFileSuffix), + logging.SetInfoFileSuffix(opt.InfoFileSuffix), + logging.SetMaxAge(opt.MaxAge), + logging.SetMaxBackups(opt.MaxBackups), + logging.SetMaxSize(opt.MaxSize), + logging.SetLevel(logging.LogLevel["debug"]), + ) +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..d3a9f0b --- /dev/null +++ b/config/config.go @@ -0,0 +1,73 @@ +package config + +import ( + "os" + + yaml "gopkg.in/yaml.v3" +) + +type WebConfig struct { + Name string `yaml:"name,omitempty"` + Host string `yaml:"host,omitempty"` + Port int `yaml:"port,omitempty"` + Mode string `yaml:"mode,omitempty"` + TempPath string `yaml:"tempPath,omitempty"` + Logging LogOptions `yaml:"logging"` + Node HpdsNode `yaml:"node,omitempty"` + Funcs []FuncConfig `yaml:"functions,omitempty"` +} + +type LogOptions struct { + Path string `yaml:"path" json:"path" toml:"path"` // 文件保存地方 + Prefix string `yaml:"prefix" json:"prefix" toml:"prefix"` // 日志文件前缀 + ErrorFileSuffix string `yaml:"errorFileSuffix" json:"errorFileSuffix" toml:"errorFileSuffix"` // error日志文件后缀 + WarnFileSuffix string `yaml:"warnFileSuffix" json:"warnFileSuffix" toml:"warnFileSuffix"` // warn日志文件后缀 + InfoFileSuffix string `yaml:"infoFileSuffix" json:"infoFileSuffix" toml:"infoFileSuffix"` // info日志文件后缀 + DebugFileSuffix string `yaml:"debugFileSuffix" json:"debugFileSuffix" toml:"debugFileSuffix"` // debug日志文件后缀 + Level string `yaml:"level" json:"level" toml:"level"` // 日志等级 + MaxSize int `yaml:"maxSize" json:"maxSize" toml:"maxSize"` // 日志文件大小（M） + MaxBackups int `yaml:"maxBackups" json:"maxBackups" toml:"maxBackups"` // 最多存在多少个切片文件 + MaxAge int `yaml:"maxAge" json:"maxAge" toml:"maxAge"` // 保存的最大天数 + Development bool `yaml:"development" json:"development" toml:"development"` // 是否是开发模式 +} + +type HpdsNode struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + Token string `yaml:"token,omitempty"` +} + +type FuncConfig struct { + Name string `yaml:"name"` + DataTag uint8 `yaml:"dataTag"` + MqType uint `yaml:"mqType"` //消息类型， 发布，1；订阅；2 +} + +func ParseConfigByFile(path string) (cfg *WebConfig, err error) { + buffer, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return load(buffer) +} + +func load(buf []byte) (cfg *WebConfig, err error) { + cfg = new(WebConfig) + cfg.Funcs = make([]FuncConfig, 0) + //cViper.ReadConfig(bytes.NewBuffer(buf)) + err = yaml.Unmarshal(buf, cfg) + //err = cViper.Unmarshal(cfg) + //if err != nil { + // return nil, err + //} + return +} + +func UpdateLocalConfig(cfg *WebConfig, fn string) error { + data, err := yaml.Marshal(cfg) + if err != nil { + return err + } + err = os.WriteFile(fn, data, 0600) + return err +} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..efd6502 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,27 @@ +name: edge-web +host: 0.0.0.0 +port: 8099 +mode: dev +tempPath: "./tmp/" +logging: + path: ./logs + prefix: hpds-edge-web + errorFileSuffix: error.log + warnFileSuffix: warn.log + infoFileSuffix: info.log + debugFileSuffix: debug.log + maxSize: 100 + maxBackups: 3000 + maxAge: 30 + development: true +node: + host: 127.0.0.1 + port: 27188 + token: 06d36c6f5705507dae778fdce90d0767 +functions: + - name: edge-cmd-request + dataTag : 30 + mqType: 2 + - name: edge-cmd-response + dataTag: 32 + mqType: 1 \ No newline at end of file diff --git a/global/global.go b/global/global.go new file mode 100644 index 0000000..e9a6738 --- /dev/null +++ b/global/global.go @@ -0,0 +1,39 @@ +package global + +import ( + "crypto/md5" + "encoding/hex" + "git.hpds.cc/Component/logging" + "go.uber.org/zap" + "hpds_annotation/config" + "io" + "os" + "unsafe" +) + +var ( + FileLabelList map[string]bool + Cfg *config.WebConfig + Logger *logging.Logger + maxGoroutinueNum = 5 + GoroutinueChan = make(chan bool, maxGoroutinueNum) + //encoderPool *sync.Pool +) + +func FileMD5(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + Logger.With( + zap.String("获取文件MD5错误", filePath), + ).Error(err.Error()) + return "", err + } + hash := md5.New() + _, _ = io.Copy(hash, file) + return hex.EncodeToString(hash.Sum(nil)), nil +} +func deepCopy(s string) string { + b := make([]byte, len(s)) + copy(b, s) + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2158fef --- /dev/null +++ b/go.mod @@ -0,0 +1,72 @@ +module hpds_annotation + +go 1.19 + +require ( + git.hpds.cc/Component/gin_valid v0.0.0-20230104142509-f956bce255b6 + git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b + git.hpds.cc/Component/network v0.0.0-20230421024959-bf7300c92a95 + git.hpds.cc/pavement/hpds_node v0.0.0-20230421025304-47b7490878f0 + github.com/gin-contrib/zap v0.1.0 + github.com/gin-gonic/gin v1.9.0 + github.com/goccy/go-json v0.10.0 + github.com/klauspost/compress v1.16.5 + github.com/shirou/gopsutil/v3 v3.23.4 + github.com/spf13/cobra v1.7.0 + go.uber.org/zap v1.24.0 + golang.org/x/image v0.1.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 // indirect + github.com/bytedance/sonic v1.8.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.11.2 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/matoous/go-nanoid/v2 v2.0.0 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/onsi/ginkgo/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/quic-go/qtls-go1-19 v0.2.1 // indirect + github.com/quic-go/qtls-go1-20 v0.1.1 // indirect + github.com/quic-go/quic-go v0.33.0 // indirect + github.com/shoenig/go-m1cpu v0.1.5 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.9 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.5.0 // indirect + golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect + golang.org/x/mod v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/tools v0.2.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/internal/handler/index.go b/internal/handler/index.go new file mode 100644 index 0000000..ca04d4e --- /dev/null +++ b/internal/handler/index.go @@ -0,0 +1,55 @@ +package handler + +import ( + "git.hpds.cc/Component/logging" + "github.com/gin-gonic/gin" + "hpds_annotation/config" + "hpds_annotation/internal/proto" + "hpds_annotation/internal/service" + e "hpds_annotation/pkg/err" +) + +type HandlerService struct { + AppConfig *config.WebConfig + Logger *logging.Logger +} + +func NewHandlerService(cfg *config.WebConfig, logger *logging.Logger) *HandlerService { + return &HandlerService{ + AppConfig: cfg, + Logger: logger, + } +} + +func (s HandlerService) GetList(c *gin.Context) (data interface{}, err error) { + repo := service.NewEdgeService(s.Logger) + var req proto.ListRequest + err = c.ShouldBindJSON(&req) + if err != nil { + return nil, e.NewValidErr(err) + } + data, err = repo.GetList(c, req) + return +} + +func (s HandlerService) GetInfo(c *gin.Context) (data interface{}, err error) { + repo := service.NewEdgeService(s.Logger) + var req proto.ListRequest + err = c.ShouldBindJSON(&req) + if err != nil { + return nil, e.NewValidErr(err) + } + data, err = repo.GetInfo(c, req) + return +} + +func (s HandlerService) LabelSubmit(c *gin.Context) (data interface{}, err error) { + repo := service.NewEdgeService(s.Logger) + var req proto.LabelRequest + err = c.ShouldBindJSON(&req) + if err != nil { + return nil, e.NewValidErr(err) + } + data, err = repo.LabelSubmit(c, req) + return +} diff --git a/internal/middleware/cors.go b/internal/middleware/cors.go new file mode 100644 index 0000000..ca8b337 --- /dev/null +++ b/internal/middleware/cors.go @@ -0,0 +1,24 @@ +package middleware + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func Cors() gin.HandlerFunc { + return func(c *gin.Context) { + method := c.Request.Method + c.Header("Access-Control-Allow-Origin", "*") + c.Header("Access-Control-Allow-Headers", "Content-Type,AccessToken,X-CSRF-Token, Authorization, Token") + c.Header("Access-Control-Allow-Methods", "POST, GET,DELETE,PUT,OPTIONS") + c.Header("Access-Control-Expose-Headers", "Content-Length, Access-Control-Allow-Origin, Access-Control-Allow-Headers, Content-Type") + c.Header("Access-Control-Allow-Credentials", "true") + // 放行所有OPTIONS方法 + if method == "OPTIONS" { + c.AbortWithStatus(http.StatusNoContent) + } + // 处理请求 + c.Next() + } +} diff --git a/internal/proto/request.go b/internal/proto/request.go new file mode 100644 index 0000000..878ea16 --- /dev/null +++ b/internal/proto/request.go @@ -0,0 +1,13 @@ +package proto + +type ListRequest struct { + Path string `json:"path"` + IsIncludeSubdirectories bool `json:"isIncludeSubdirectories"` +} + +type LabelRequest struct { + FileList []string `json:"fileList"` + LabelStatus bool `json:"labelStatus"` + TrainingSet string `json:"trainingSet"` + BizType int `json:"bizType"` +} diff --git a/internal/proto/response.go b/internal/proto/response.go new file mode 100644 index 0000000..aee27b7 --- /dev/null +++ b/internal/proto/response.go @@ -0,0 +1,28 @@ +package proto + +// BaseResponse 基础返回结构 +type BaseResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"result,omitempty"` + Status string `json:"type,omitempty"` + Err error `json:"error,omitempty"` // 错误堆栈 + //Page int64 `json:"page,omitempty"` //当前页码 + //PageSize int64 `json:"pageSize,omitempty"` // 单页显示记录数--前端参数2 + //PageCount int64 `json:"totalPage,omitempty"` // 总页数 + //TotalSize int64 `json:"total,omitempty"` // 总记录数 +} + +type FileBaseInfo struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"isDir"` + Size int64 `json:"size"` + ModTime int64 `json:"modTime"` +} + +type FileContent struct { + FileBaseInfo + ContentBase string `json:"contentBase"` + LabelStatus int `json:"labelStatus"` //0:未标注;1:有病害;2:无病害 +} diff --git a/internal/router/router.go b/internal/router/router.go new file mode 100644 index 0000000..8f908e1 --- /dev/null +++ b/internal/router/router.go @@ -0,0 +1,33 @@ +package router + +import ( + "git.hpds.cc/Component/logging" + ginzap "github.com/gin-contrib/zap" + "github.com/gin-gonic/gin" + "hpds_annotation/config" + "hpds_annotation/internal/handler" + "hpds_annotation/internal/middleware" + + e "hpds_annotation/pkg/err" +) + +func InitRouter(cfg *config.WebConfig, logger *logging.Logger) *gin.Engine { + hs := handler.NewHandlerService(cfg, logger) + gin.SetMode(gin.ReleaseMode) + root := gin.New() + root.Use(ginzap.Ginzap(logger.Logger, "2006-01-02 15:04:05.000", true)) + root.Use(middleware.Cors()) + r := root.Group("/api") + { + dir := r.Group("/directory") + { + dir.POST("/list", e.ErrorWrapper(hs.GetList)) + dir.POST("/info", e.ErrorWrapper(hs.GetInfo)) + } + label := r.Group("/label") + { + label.POST("/submit", e.ErrorWrapper(hs.LabelSubmit)) + } + } + return root +} diff --git a/internal/service/index.go b/internal/service/index.go new file mode 100644 index 0000000..4b6774d --- /dev/null +++ b/internal/service/index.go @@ -0,0 +1,220 @@ +package service + +import ( + "context" + "encoding/base64" + "fmt" + "git.hpds.cc/Component/logging" + "hpds_annotation/config" + "hpds_annotation/global" + "hpds_annotation/internal/proto" + "hpds_annotation/mq" + "hpds_annotation/pkg/utils" + "hpds_annotation/store" + "net/http" + "os" + "path" +) + +type PagingStruct struct { + List interface{} `json:"list"` + Total int64 `json:"total"` +} + +// FillPaging 填充分页数据 +func FillPaging(count, pageNum, pageSize int64, list interface{}, data *proto.BaseResponse) *proto.BaseResponse { + _ = fmt.Sprintf("%d, %d", pageNum, pageSize) + ps := new(PagingStruct) + ps.List = list + ps.Total = count + data.Data = ps + //data.PageSize = pageSize + //data.Data = list + //data.Page = pageNum + //data.PageCount = tp + //data.TotalSize = count + return data +} + +type repo struct { + AppConfig *config.WebConfig + logger *logging.Logger +} + +type EdgeService interface { + GetList(ctx context.Context, req proto.ListRequest) (rsp *proto.BaseResponse, err error) + GetInfo(ctx context.Context, req proto.ListRequest) (rsp *proto.BaseResponse, err error) + LabelSubmit(ctx context.Context, req proto.LabelRequest) (rsp *proto.BaseResponse, err error) +} + +func NewEdgeService(logger *logging.Logger) EdgeService { + return &repo{ + logger: logger, + } +} + +func (rp *repo) GetList(ctx context.Context, req proto.ListRequest) (rsp *proto.BaseResponse, err error) { + rsp = new(proto.BaseResponse) + select { + case <-ctx.Done(): + err = fmt.Errorf("超时/取消") + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Message = "超时/取消" + rsp.Err = ctx.Err() + return rsp, ctx.Err() + default: + if req.IsIncludeSubdirectories { + + } else { + var files []os.DirEntry + files, err = os.ReadDir(req.Path) + if err != nil { + goto ReturnPoint + } + list := make([]proto.FileBaseInfo, len(files)) + for k, v := range files { + info, _ := v.Info() + item := proto.FileBaseInfo{ + Name: v.Name(), + Path: path.Join(req.Path, v.Name()), + IsDir: v.IsDir(), + Size: info.Size(), + ModTime: info.ModTime().Unix(), + } + list[k] = item + } + + rsp.Code = http.StatusOK + rsp.Status = http.StatusText(http.StatusOK) + rsp.Message = "成功" + rsp.Data = list + rsp.Err = err + return rsp, err + } + } + +ReturnPoint: + if err != nil { + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Err = err + rsp.Message = "失败" + } + return rsp, err +} + +func (rp *repo) GetInfo(ctx context.Context, req proto.ListRequest) (rsp *proto.BaseResponse, err error) { + rsp = new(proto.BaseResponse) + select { + case <-ctx.Done(): + err = fmt.Errorf("超时/取消") + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Message = "超时/取消" + rsp.Err = ctx.Err() + return rsp, ctx.Err() + default: + var ( + fileInfo os.FileInfo + ) + fileInfo, err = os.Stat(req.Path) + if err != nil { + goto ReturnPoint + } + if fileInfo.IsDir() { + err = fmt.Errorf("不能获取文件夹的详细信息") + goto ReturnPoint + } + buff := utils.ReadFile(req.Path) + res := new(proto.FileContent) + res.Name = fileInfo.Name() + res.IsDir = false + res.Path = req.Path + res.Size = fileInfo.Size() + res.ModTime = fileInfo.ModTime().Unix() + res.ContentBase = base64.StdEncoding.EncodeToString(buff) + b, ok := global.FileLabelList[req.Path] + if ok { + if b { + res.LabelStatus = 1 + } else { + res.LabelStatus = 2 + } + } else { + res.LabelStatus = 0 + } + + rsp.Code = http.StatusOK + rsp.Status = http.StatusText(http.StatusOK) + rsp.Message = "成功" + rsp.Data = res + rsp.Err = err + return rsp, err + } + +ReturnPoint: + if err != nil { + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Err = err + rsp.Message = "失败" + } + return rsp, err +} + +func (rp *repo) LabelSubmit(ctx context.Context, req proto.LabelRequest) (rsp *proto.BaseResponse, err error) { + rsp = new(proto.BaseResponse) + select { + case <-ctx.Done(): + err = fmt.Errorf("超时/取消") + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Message = "超时/取消" + rsp.Err = ctx.Err() + return rsp, ctx.Err() + default: + list := make([]proto.FileContent, len(req.FileList)) + var ( + fileInfo os.FileInfo + ) + for k, v := range req.FileList { + global.FileLabelList[v] = req.LabelStatus + fileInfo, err = os.Stat(v) + if err != nil { + goto ReturnPoint + } + buff := utils.ReadFile(v) + status := 0 + if req.LabelStatus { + status = 1 + } else { + status = 2 + } + dstContent := store.Compress(buff) + list[k] = proto.FileContent{ + FileBaseInfo: proto.FileBaseInfo{ + Name: fileInfo.Name(), + Path: v, + Size: fileInfo.Size(), + ModTime: fileInfo.ModTime().Unix(), + }, + ContentBase: base64.StdEncoding.EncodeToString(dstContent), + LabelStatus: status, + } + } + go mq.SendLabelData(list, rp.logger) + rsp.Code = http.StatusOK + rsp.Status = http.StatusText(http.StatusOK) + rsp.Message = "成功" + rsp.Err = err + ReturnPoint: + if err != nil { + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Err = err + rsp.Message = "失败" + } + return rsp, err + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..314ab3d --- /dev/null +++ b/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "github.com/spf13/cobra" + "hpds_annotation/cmd" + "os" +) + +var ( + rootCmd = &cobra.Command{ + Use: "hpds_edge_web", + Long: "hpds_edge_web is a IoT WEB UI", + Version: "0.1", + } +) + +func init() { + rootCmd.AddCommand(cmd.NewStartCmd()) +} +func main() { + if err := rootCmd.Execute(); err != nil { + _, _ = fmt.Fprint(os.Stderr, err.Error()) + os.Exit(1) + } +} diff --git a/mq/handler.go b/mq/handler.go new file mode 100644 index 0000000..a22d44f --- /dev/null +++ b/mq/handler.go @@ -0,0 +1,103 @@ +package mq + +import ( + "encoding/base64" + "fmt" + "hpds_annotation/global" + "hpds_annotation/internal/proto" + "hpds_annotation/pkg/utils" + "net/http" + "os" + "path" +) + +func GetList(req proto.ListRequest) (rsp *proto.BaseResponse, err error) { + rsp = new(proto.BaseResponse) + + var ( + files []os.DirEntry + list []proto.FileBaseInfo + ) + files, err = os.ReadDir(req.Path) + if err != nil { + goto ReturnPoint + } + list = make([]proto.FileBaseInfo, len(files)) + for k, v := range files { + info, _ := v.Info() + item := proto.FileBaseInfo{ + Name: v.Name(), + Path: path.Join(req.Path, v.Name()), + IsDir: v.IsDir(), + Size: info.Size(), + ModTime: info.ModTime().Unix(), + } + list[k] = item + } + + rsp.Code = http.StatusOK + rsp.Status = http.StatusText(http.StatusOK) + rsp.Message = "成功" + rsp.Data = list + rsp.Err = err + return rsp, err +ReturnPoint: + if err != nil { + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Err = err + rsp.Message = "失败" + } + return rsp, err +} + +func GetFileInfo(req proto.ListRequest) (rsp *proto.BaseResponse, err error) { + rsp = new(proto.BaseResponse) + var ( + fileInfo os.FileInfo + b, ok bool + buff []byte + ) + res := new(proto.FileContent) + fileInfo, err = os.Stat(req.Path) + if err != nil { + goto ReturnPoint + } + if fileInfo.IsDir() { + err = fmt.Errorf("不能获取文件夹的详细信息") + goto ReturnPoint + } + buff = utils.ReadFile(req.Path) + res.Name = fileInfo.Name() + res.IsDir = false + res.Path = req.Path + res.Size = fileInfo.Size() + res.ModTime = fileInfo.ModTime().Unix() + res.ContentBase = base64.StdEncoding.EncodeToString(buff) + b, ok = global.FileLabelList[req.Path] + if ok { + if b { + res.LabelStatus = 1 + } else { + res.LabelStatus = 2 + } + } else { + res.LabelStatus = 0 + } + + rsp.Code = http.StatusOK + rsp.Status = http.StatusText(http.StatusOK) + rsp.Message = "成功" + rsp.Data = res + rsp.Err = err + return rsp, err + +ReturnPoint: + if err != nil { + rsp.Code = http.StatusInternalServerError + rsp.Status = http.StatusText(http.StatusInternalServerError) + rsp.Err = err + rsp.Message = "失败" + } + return rsp, err +} diff --git a/mq/index.go b/mq/index.go new file mode 100644 index 0000000..873eef4 --- /dev/null +++ b/mq/index.go @@ -0,0 +1,199 @@ +package mq + +import ( + "encoding/json" + "fmt" + "go.uber.org/zap" + "hpds_annotation/config" + "hpds_annotation/global" + "hpds_annotation/internal/proto" + "net/http" + "os" + "strings" + "time" + + "git.hpds.cc/Component/logging" + "git.hpds.cc/Component/network/frame" + "git.hpds.cc/pavement/hpds_node" + "github.com/shirou/gopsutil/v3/host" +) + +var MqList []HpdsMqNode + +type HpdsMqNode struct { + MqType uint + Topic string + Node config.HpdsNode + EndPoint interface{} +} + +var ifChannelsMapInit = false + +var ChannelsMap = map[string]chan string{} + +func initChannelsMap() { + ChannelsMap = make(map[string]chan string) +} +func AddChannel(userId string) { + if !ifChannelsMapInit { + initChannelsMap() + ifChannelsMapInit = true + } + var newChannel = make(chan string) + ChannelsMap[userId] = newChannel + logging.L().Info("Build SSE connection for user = " + userId) +} + +func must(logger *logging.Logger, err error) { + if err != nil { + if logger != nil { + logger.With(zap.String("web节点", "错误信息")).Error("启动错误", zap.Error(err)) + } else { + _, _ = fmt.Fprint(os.Stderr, err) + } + os.Exit(1) + } +} + +func NewMqClient(funcs []config.FuncConfig, node config.HpdsNode, logger *logging.Logger) (mqList []HpdsMqNode, err error) { + mqList = make([]HpdsMqNode, 0) + for _, v := range funcs { + switch v.MqType { + case 2: + sf := hpds_node.NewStreamFunction( + v.Name, + hpds_node.WithMqAddr(fmt.Sprintf("%s:%d", node.Host, node.Port)), + hpds_node.WithObserveDataTags(frame.Tag(v.DataTag)), + hpds_node.WithCredential(node.Token), + ) + err = sf.Connect() + nodeInfo := HpdsMqNode{ + MqType: 2, + Topic: v.Name, + Node: node, + EndPoint: sf, + } + must(logger, err) + switch v.Name { + case "edge-cmd-request": + _ = sf.SetHandler(EdgeCmdHandle) + default: + + } + mqList = append(mqList, nodeInfo) + default: + ap := hpds_node.NewAccessPoint( + v.Name, + hpds_node.WithMqAddr(fmt.Sprintf("%s:%d", node.Host, node.Port)), + hpds_node.WithCredential(node.Token), + ) + err = ap.Connect() + nodeInfo := HpdsMqNode{ + MqType: 1, + Topic: v.Name, + Node: node, + EndPoint: ap, + } + must(logger, err) + ap.SetDataTag(frame.Tag(v.DataTag)) + mqList = append(mqList, nodeInfo) + } + + } + return mqList, err +} + +func GetMqClient(topic string, mqType uint) *HpdsMqNode { + for _, v := range MqList { + if v.Topic == topic && v.MqType == mqType { + return &v + } + } + return nil +} + +func GenerateAndSendData(stream hpds_node.AccessPoint, data []byte, logger *logging.Logger) error { + logger.With(zap.String("web节点", "发送消息")).Info("数据", zap.String("发送的数据", string(data))) + _, err := stream.Write(data) + if err != nil { + return err + } + time.Sleep(1000 * time.Millisecond) + return nil +} + +func EdgeCmdHandle(data []byte) (frame.Tag, []byte) { + global.Logger.Info("任务日志", zap.String("接收数据", string(data))) + cmd := new(InstructionReq) + err := json.Unmarshal(data, cmd) + if err != nil { + return 0x0B, []byte(err.Error()) + } + switch cmd.Command { + case DataLabelRequest: + hi, _ := host.Info() + payload := cmd.Payload.(map[string]interface{}) + if payload["nodeGuid"] == hi.HostID { + //currTime := time.Now().Unix() + var ( + req proto.ListRequest + res *proto.BaseResponse + ) + if v, ok := payload["path"]; ok { + req.Path = v.(string) + } + resPayload := new(DataLabelRes) + resPayload.NodeGuid = hi.HostID + + switch strings.ToLower(payload["cmd"].(string)) { + case "get-list": + res, err = GetList(req) + if err != nil { + goto ErrorPoint + } + case "get-file-info": + res, err = GetFileInfo(req) + if err != nil { + goto ErrorPoint + } + } + ErrorPoint: + if err != nil { + res.Code = http.StatusInternalServerError + res.Status = http.StatusText(http.StatusInternalServerError) + res.Err = err + res.Message = "失败" + } + resPayload.Body = res + rsp := new(InstructionReq) + rsp.Command = DataLabelResponse + rsp.Payload = resPayload + str, _ := json.Marshal(rsp) + cli := GetMqClient("edge-cmd-response", 1) + if cli != nil { + _ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), str, global.Logger) + } + } + } + return 0x0B, nil +} + +func SendLabelData(list []proto.FileContent, logger *logging.Logger) { + cli := GetMqClient("edge-cmd-response", 1) + if cli != nil { + for _, v := range list { + payload := InstructionReq{ + Command: DataLabelResponse, + Payload: v, + } + s, _ := json.Marshal(payload) + err := GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), s, logger) + if err != nil { + logger.With( + zap.String("文件名称", v.Name), + zap.String("存储路径", v.Path), + ).Error("文件传输", zap.Error(err)) + } + } + } +} diff --git a/mq/instruction.go b/mq/instruction.go new file mode 100644 index 0000000..4b07dc8 --- /dev/null +++ b/mq/instruction.go @@ -0,0 +1,16 @@ +package mq + +const ( + DataLabelRequest = iota + 12 + DataLabelResponse +) + +type InstructionReq struct { + Command int `json:"command"` + Payload interface{} `json:"payload"` +} + +type DataLabelRes struct { + NodeGuid string `json:"nodeGuid"` + Body interface{} `json:"body"` +} diff --git a/pkg/err/err_code.go b/pkg/err/err_code.go new file mode 100644 index 0000000..4e9e85d --- /dev/null +++ b/pkg/err/err_code.go @@ -0,0 +1,83 @@ +package e + +import "github.com/goccy/go-json" + +const ( + UnKnow = 1 + ValidErr = 1000 + DbErr = 2020 + IllegalPhone = 2001 + NoAuth = 401 + NoUser = 2003 + ExistingUserName = 2004 + UnmarshalReqErr = 3000 + Ordinary = 6666 +) + +func init() { + m := map[int]string{ + UnKnow: "未定义错误", + ValidErr: "无效的请求参数", + DbErr: "数据库错误", + IllegalPhone: "请输入正确的手机号码", + NoAuth: "未授权", + NoUser: "未找到对应用户", + ExistingUserName: "已经存在的用户", + UnmarshalReqErr: "参数类型错误", + } + + errMap[UnKnow] = &ApiError{ + Status: 500, + Code: UnKnow, + Message: "未定义错误", + } + + for k, v := range m { + errMap[k] = &ApiError{ + Status: 200, + Code: k, + Message: v, + } + } +} + +var errMap = map[int]*ApiError{} + +func NewCode(code int) *ApiError { + if e, ok := errMap[code]; ok { + return e + } + return errMap[UnKnow] +} +func NewString(msg string) *ApiError { + return &ApiError{ + Status: 200, + Code: Ordinary, + Message: msg, + } +} +func newErr(err *ApiError) *ApiError { + return &(*err) +} +func NewValidErr(err error) error { + if _, ok := err.(*json.UnmarshalTypeError); ok { + return newErr(errMap[UnmarshalReqErr]) + } + return err +} + +func (e *ApiError) Set(msg string) *ApiError { + r := *e + r.Message = msg + return &r +} + +type ApiError struct { + Status int `json:"-"` + Code int `json:"code"` + Message string `json:"msg"` +} + +func (e *ApiError) Error() string { + return e.Message +} diff --git a/pkg/err/err_encode.go b/pkg/err/err_encode.go new file mode 100644 index 0000000..8c76ee7 --- /dev/null +++ b/pkg/err/err_encode.go @@ -0,0 +1,58 @@ +package e + +import ( + "git.hpds.cc/Component/gin_valid/gin/binding" + validator "git.hpds.cc/Component/gin_valid/go-playground/validator/v10" + "github.com/gin-gonic/gin" + "net/http" +) + +type WrapperHandle func(c *gin.Context) (interface{}, error) + +func ErrorWrapper(handle WrapperHandle) gin.HandlerFunc { + return func(c *gin.Context) { + data, err := handle(c) + if err != nil { + switch r := err.(type) { + case *ApiError: + c.JSON(r.Status, r) + case validator.ValidationErrors: + msg := r.Translate(binding.ValidTrans).Error() + c.JSON(http.StatusOK, gin.H{"code": "", "msg": msg}) + default: + er := NewValidErr(err) + c.JSON(http.StatusOK, gin.H{"code": 500, "msg": er.Error()}) + } + return + } + if data != nil { + c.JSON(http.StatusOK, data) + } else { + c.JSON(http.StatusOK, gin.H{"code": 200, "data": data}) + } + + } +} +func ErrorWeChatWrapper(handle WrapperHandle) gin.HandlerFunc { + return func(c *gin.Context) { + data, err := handle(c) + if err != nil { + switch r := err.(type) { + case *ApiError: + c.JSON(r.Status, r) + case validator.ValidationErrors: + msg := r.Translate(binding.ValidTrans).Error() + c.JSON(http.StatusOK, gin.H{"code": "", "msg": msg}) + default: + er := NewValidErr(err) + c.JSON(http.StatusOK, gin.H{"code": 500, "msg": er.Error()}) + } + return + } + if data != nil { + c.JSON(http.StatusOK, gin.H{"code": "SUCCESS", "data": data}) + } else { + c.JSON(http.StatusOK, gin.H{"code": "SUCCESS", "message": "成功"}) + } + } +} diff --git a/pkg/utils/file.go b/pkg/utils/file.go new file mode 100644 index 0000000..5c7e24d --- /dev/null +++ b/pkg/utils/file.go @@ -0,0 +1,88 @@ +package utils + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "git.hpds.cc/Component/logging" + "go.uber.org/zap" + "io" + "os" + "path" + "path/filepath" + "strings" +) + +func CopyFile(src, dst string) error { + sourceFileStat, err := os.Stat(src) + if err != nil { + return err + } + + if !sourceFileStat.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer func(source *os.File) { + _ = source.Close() + }(source) + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer func(destination *os.File) { + _ = destination.Close() + }(destination) + _, err = io.Copy(destination, source) + return err +} + +func PathExists(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return false +} + +// ReadFile 读取到file中，再利用ioutil将file直接读取到[]byte中, 这是最优 +func ReadFile(fn string) []byte { + f, err := os.Open(fn) + if err != nil { + logging.L().Error("Read File", zap.String("File Name", fn), zap.Error(err)) + return nil + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + fd, err := io.ReadAll(f) + if err != nil { + logging.L().Error("Read File To buff", zap.String("File Name", fn), zap.Error(err)) + return nil + } + + return fd +} + +func GetFileName(fn string) string { + fileType := path.Ext(fn) + return strings.TrimSuffix(fn, fileType) +} +func GetFileNameAndExt(fn string) string { + _, fileName := filepath.Split(fn) + return fileName +} +func GetFileMd5(data []byte) string { + hash := md5.New() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/pkg/utils/http.go b/pkg/utils/http.go new file mode 100644 index 0000000..02a5a7d --- /dev/null +++ b/pkg/utils/http.go @@ -0,0 +1,126 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "path/filepath" + "strings" +) + +func HttpDo(reqUrl, method string, params map[string]string, header map[string]string) (data []byte, err error) { + var paramStr string = "" + if contentType, ok := header["ContentType"]; ok && strings.Contains(contentType, "json") { + bytesData, _ := json.Marshal(params) + paramStr = string(bytesData) + } else { + for k, v := range params { + if len(paramStr) == 0 { + paramStr = fmt.Sprintf("%s=%s", k, url.QueryEscape(v)) + } else { + paramStr = fmt.Sprintf("%s&%s=%s", paramStr, k, url.QueryEscape(v)) + } + } + } + + client := &http.Client{} + req, err := http.NewRequest(strings.ToUpper(method), reqUrl, strings.NewReader(paramStr)) + if err != nil { + return nil, err + } + for k, v := range header { + req.Header.Set(k, v) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + defer func() { + if resp.Body != nil { + err = resp.Body.Close() + if err != nil { + return + } + } + }() + var body []byte + if resp.Body != nil { + body, err = io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + } + return body, nil +} + +type UploadFile struct { + // 表单名称 + Name string + Filepath string + // 文件全路径 + File *bytes.Buffer +} + +func PostFile(reqUrl string, reqParams map[string]string, contentType string, files []UploadFile, headers map[string]string) string { + requestBody, realContentType := getReader(reqParams, contentType, files) + httpRequest, _ := http.NewRequest("POST", reqUrl, requestBody) + // 添加请求头 + httpRequest.Header.Add("Content-Type", realContentType) + if headers != nil { + for k, v := range headers { + httpRequest.Header.Add(k, v) + } + } + httpClient := &http.Client{} + // 发送请求 + resp, err := httpClient.Do(httpRequest) + if err != nil { + panic(err) + } + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + response, _ := io.ReadAll(resp.Body) + return string(response) +} + +func getReader(reqParams map[string]string, contentType string, files []UploadFile) (io.Reader, string) { + if strings.Index(contentType, "json") > -1 { + bytesData, _ := json.Marshal(reqParams) + return bytes.NewReader(bytesData), contentType + } else if files != nil { + body := &bytes.Buffer{} + // 文件写入 body + writer := multipart.NewWriter(body) + for _, uploadFile := range files { + part, err := writer.CreateFormFile(uploadFile.Name, filepath.Base(uploadFile.Filepath)) + if err != nil { + panic(err) + } + _, err = io.Copy(part, uploadFile.File) + } + // 其他参数列表写入 body + for k, v := range reqParams { + if err := writer.WriteField(k, v); err != nil { + panic(err) + } + } + if err := writer.Close(); err != nil { + panic(err) + } + // 上传文件需要自己专用的contentType + return body, writer.FormDataContentType() + } else { + urlValues := url.Values{} + for key, val := range reqParams { + urlValues.Set(key, val) + } + reqBody := urlValues.Encode() + return strings.NewReader(reqBody), contentType + } +} diff --git a/pkg/utils/image.go b/pkg/utils/image.go new file mode 100644 index 0000000..5c7975c --- /dev/null +++ b/pkg/utils/image.go @@ -0,0 +1,216 @@ +package utils + +import ( + "bytes" + "encoding/base64" + "fmt" + "golang.org/x/image/bmp" + "golang.org/x/image/tiff" + "image" + "image/color" + "image/draw" + "image/jpeg" + "image/png" + "math" +) + +func BuffToImage(in []byte) image.Image { + buff := bytes.NewBuffer(in) + m, _, _ := image.Decode(buff) + return m +} + +// Clip 图片裁剪 +func Clip(buff *bytes.Buffer, wi, hi int, equalProportion bool) (out image.Image, imageType string, err error) { + //buff := bytes.NewBuffer(in) + m, imgType, err := image.Decode(buff) + if err != nil { + return nil, "", err + } + if equalProportion { + w := m.Bounds().Max.X + h := m.Bounds().Max.Y + if w > 0 && h > 0 && wi > 0 && hi > 0 { + wi, hi = fixSize(w, h, wi, hi) + } + } + switch imgType { + case "jpeg", "jpg": + rgbImg := m.(*image.YCbCr) + return rgbImg.SubImage(image.Rect(0, 0, wi, hi)), imgType, nil + case "bmp": + img := m.(*image.RGBA) + if equalProportion { + + } + return img.SubImage(image.Rect(0, 0, wi, hi)).(*image.RGBA), imageType, nil + case "png": + switch m.(type) { + case *image.NRGBA: + img := m.(*image.NRGBA) + subImg := img.SubImage(image.Rect(0, 0, wi, hi)).(*image.NRGBA) + + return subImg, imageType, nil + case *image.RGBA: + img := m.(*image.RGBA) + subImg := img.SubImage(image.Rect(0, 0, wi, hi)).(*image.RGBA) + return subImg, imageType, nil + } + case "gif": + img := m.(*image.Paletted) + subImg := img.SubImage(image.Rect(0, 0, wi, hi)).(*image.Paletted) + return subImg, imageType, nil + } + return nil, "", fmt.Errorf("未知的图片格式") + +} + +func fixSize(img1W, img2H, wi, hi int) (new1W, new2W int) { + var ( //为了方便计算，将图片的宽转为 float64 + imgWidth, imgHeight = float64(img1W), float64(img2H) + ratio float64 + ) + if imgWidth >= imgHeight { + ratio = imgWidth / float64(wi) + return int(imgWidth * ratio), int(imgHeight * ratio) + } + ratio = imgHeight / float64(hi) + return int(imgWidth * ratio), int(imgHeight * ratio) +} + +func Gray(buff *bytes.Buffer) (out image.Image, err error) { + m, _, _ := image.Decode(buff) + bounds := m.Bounds() + dx := bounds.Dx() + dy := bounds.Dy() + newRgba := image.NewRGBA(bounds) + for i := 0; i < dx; i++ { + for j := 0; j < dy; j++ { + colorRgb := m.At(i, j) + _, g, _, a := colorRgb.RGBA() + gUint8 := uint8(g >> 8) + aUint8 := uint8(a >> 8) + newRgba.SetRGBA(i, j, color.RGBA{R: gUint8, G: gUint8, B: gUint8, A: aUint8}) + } + } + r := image.Rect(0, 0, dx, dy) + return newRgba.SubImage(r), nil +} + +func Rotate90(buff *bytes.Buffer) image.Image { + m, _, _ := image.Decode(buff) + rotate90 := image.NewRGBA(image.Rect(0, 0, m.Bounds().Dy(), m.Bounds().Dx())) + // 矩阵旋转 + for x := m.Bounds().Min.Y; x < m.Bounds().Max.Y; x++ { + for y := m.Bounds().Max.X - 1; y >= m.Bounds().Min.X; y-- { + // 设置像素点 + rotate90.Set(m.Bounds().Max.Y-x, y, m.At(y, x)) + } + } + return rotate90 +} + +// Rotate180 旋转180度 +func Rotate180(buff *bytes.Buffer) image.Image { + m, _, _ := image.Decode(buff) + rotate180 := image.NewRGBA(image.Rect(0, 0, m.Bounds().Dx(), m.Bounds().Dy())) + // 矩阵旋转 + for x := m.Bounds().Min.X; x < m.Bounds().Max.X; x++ { + for y := m.Bounds().Min.Y; y < m.Bounds().Max.Y; y++ { + // 设置像素点 + rotate180.Set(m.Bounds().Max.X-x, m.Bounds().Max.Y-y, m.At(x, y)) + } + } + return rotate180 +} + +// Rotate270 旋转270度 +func Rotate270(buff *bytes.Buffer) image.Image { + m, _, _ := image.Decode(buff) + rotate270 := image.NewRGBA(image.Rect(0, 0, m.Bounds().Dy(), m.Bounds().Dx())) + // 矩阵旋转 + for x := m.Bounds().Min.Y; x < m.Bounds().Max.Y; x++ { + for y := m.Bounds().Max.X - 1; y >= m.Bounds().Min.X; y-- { + // 设置像素点 + rotate270.Set(x, m.Bounds().Max.X-y, m.At(y, x)) + } + } + return rotate270 + +} + +func ImageToBase64(img image.Image, imgType string) string { + buff := ImageToBuff(img, imgType) + return base64.StdEncoding.EncodeToString(buff.Bytes()) +} + +func ImageToBuff(img image.Image, imgType string) *bytes.Buffer { + buff := bytes.NewBuffer(nil) + switch imgType { + case "bmp": + imgType = "bmp" + _ = bmp.Encode(buff, img) + case "png": + imgType = "png" + _ = png.Encode(buff, img) + case "tiff": + imgType = "tiff" + _ = tiff.Encode(buff, img, nil) + default: + imgType = "jpeg" + _ = jpeg.Encode(buff, img, &jpeg.Options{ + Quality: 100, + }) + } + return buff +} +func SplitImage(buff *bytes.Buffer, w, h int) []image.Image { + img, _, _ := image.Decode(buff) + list := make([]image.Image, 0) + newWidth := int(math.Ceil(float64(img.Bounds().Dx()) / float64(w))) + newHeight := int(math.Ceil(float64(img.Bounds().Dy()) / float64(h))) + rect := image.Rect(0, 0, newWidth*w, newHeight*h) + newImage := image.NewRGBA(rect) + black := color.RGBA{A: 255} + draw.Draw(newImage, newImage.Bounds(), &image.Uniform{C: black}, image.Point{}, draw.Src) + draw.Draw(newImage, img.Bounds(), img, newImage.Bounds().Min, draw.Over) + sp := splitPattern(newImage, w, h) + spLen := len(sp) + var ( + square image.Rectangle + ) + for i := 0; i < spLen; i++ { + square = image.Rect(sp[i][0], sp[i][1], sp[i][2], sp[i][3]) + imgPart := img.(interface { + SubImage(r image.Rectangle) image.Image + }).SubImage(square) + list = append(list, imgPart) + } + return list +} +func splitPattern(img image.Image, w, h int) [][]int { + ret := make([][]int, 0) + vOffSet := width(img) / w + hOffSet := height(img) / h + for r := 0; r < hOffSet; r++ { + for c := 0; c < vOffSet; c++ { + //行偏移,仅应用在x + x1 := w * c + y1 := h * r + + x2 := w * (c + 1) + y2 := (r + 1) * h + el := []int{x1, y1, x2, y2} + ret = append(ret, el) + } + } + return ret +} + +func width(i image.Image) int { + return i.Bounds().Max.X - i.Bounds().Min.X +} + +func height(i image.Image) int { + return i.Bounds().Max.Y - i.Bounds().Min.Y +} diff --git a/pkg/utils/network.go b/pkg/utils/network.go new file mode 100644 index 0000000..c619faf --- /dev/null +++ b/pkg/utils/network.go @@ -0,0 +1,40 @@ +package utils + +import ( + "fmt" + "git.hpds.cc/Component/network/log" + "net" +) + +// GetAvailablePort 获取可用端口 +func GetAvailablePort() (int, error) { + address, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:0", "0.0.0.0")) + if err != nil { + return 0, err + } + + listener, err := net.ListenTCP("tcp", address) + if err != nil { + return 0, err + } + + defer func(listener *net.TCPListener) { + _ = listener.Close() + }(listener) + return listener.Addr().(*net.TCPAddr).Port, nil +} + +// IsPortAvailable 判断端口是否可以（未被占用） +func IsPortAvailable(port int) bool { + address := fmt.Sprintf("%s:%d", "0.0.0.0", port) + listener, err := net.Listen("tcp", address) + if err != nil { + log.Infof("port %s is taken: %s", address, err) + return false + } + + defer func(listener net.Listener) { + _ = listener.Close() + }(listener) + return true +} diff --git a/store/index.go b/store/index.go new file mode 100644 index 0000000..7001c1c --- /dev/null +++ b/store/index.go @@ -0,0 +1,91 @@ +package store + +import ( + "encoding/json" + "fmt" + "hpds_annotation/pkg/utils" + "io" + "os" + "path" + + "github.com/klauspost/compress/zstd" +) + +func Load(storePath string) map[string]bool { + if !utils.PathExists(storePath) { + _ = os.MkdirAll(storePath, os.ModePerm) + } + fileName := "store" + storeFile := path.Join(storePath, fmt.Sprintf("%s.hdb", fileName)) + if !utils.PathExists(storeFile) { + NewFile(storeFile) + return make(map[string]bool) + } + list := make(map[string]bool) + f, _ := os.OpenFile(storeFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + defer func(f *os.File) { + _ = f.Close() + }(f) + buff, err := io.ReadAll(f) + if err != nil { + fmt.Println(err) + return nil + } + if len(buff) > 0 { + str, err := UnCompress(buff) + if err != nil { + return nil + } + err = json.Unmarshal(str, &list) + if err != nil { + return nil + } + } + return list +} + +func Save(storePath string, list map[string]bool) { + if !utils.PathExists(storePath) { + _ = os.MkdirAll(storePath, os.ModePerm) + } + fileName := "store" + storeFile := path.Join(storePath, fmt.Sprintf("%s.hdb", fileName)) + if !utils.PathExists(storeFile) { + NewFile(storeFile) + } + f, _ := os.OpenFile(storeFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + defer func() { + _ = f.Close() + }() + str, _ := json.Marshal(list) + c := Compress(str) + _, _ = f.Write(c) +} +func NewFile(fileName string) { + f, _ := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + defer func(f *os.File) { + _ = f.Close() + }(f) +} + +// Compress 压缩 +func Compress(src []byte) []byte { + encoder, _ := zstd.NewWriter(nil) + zstd.WithEncoderConcurrency(3) + return encoder.EncodeAll(src, make([]byte, 0, len(src))) +} + +func UnCompress(src []byte) ([]byte, error) { + d, err := zstd.NewReader(nil) + if err != nil { + return nil, err + } + defer d.Close() + + uncompressed, err := d.DecodeAll(src, nil) + if err != nil { + return nil, err + } + + return uncompressed, nil +}