初始化代码

cmd/server.go

@@ -0,0 +1,131 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"folder_monitoring/config"
+	"folder_monitoring/global"
+	"folder_monitoring/mq"
+	"folder_monitoring/store"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+)
+
+var (
+	ConfigFileFlag string = "./config/config.yaml"
+	NodeName       string = "main-node"
+	Mode           string = "dev"
+	Path           string = "" //监控文件夹
+)
+
+func must(err error) {
+	if err != nil {
+		_, _ = fmt.Fprint(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func NewStartCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "start",
+		Short: "Start folder monitor",
+		Run: func(cmd *cobra.Command, args []string) {
+			var (
+				cfg *config.FolderMonitorConfig
+				err error
+			)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			must(err)
+			configFileFlag, err := cmd.Flags().GetString("c")
+			if err != nil {
+				fmt.Println("get local config err: ", err)
+				return
+			}
+
+			NodeName, err = cmd.Flags().GetString("n")
+			if err != nil {
+				fmt.Println("get remote path config err: ", err)
+				return
+			}
+			Mode, err = cmd.Flags().GetString("m")
+			if err != nil {
+				fmt.Println("get remote path config err: ", err)
+				return
+			}
+			if len(configFileFlag) > 1 {
+				cfg, err = config.ParseConfigByFile(configFileFlag)
+				must(err)
+				cfg.MonitorPath, _ = filepath.Abs(cfg.MonitorPath)
+				cfg.TempPath, _ = filepath.Abs(cfg.TempPath)
+				ConfigFileFlag = configFileFlag
+			}
+			global.Cfg = cfg
+			global.Logger = config.LoadLoggerConfig(cfg.Logging)
+			//加载历史文件的传输情况
+			err = store.Load(cfg.TempPath, global.FileHistoryMap)
+			must(err)
+			//创建消息连接点
+			mq.MqList, err = mq.NewMqClient(cfg.Funcs, cfg.Node, global.Logger)
+			must(err)
+			if b, _ := store.PathExists(cfg.MonitorPath); !b {
+				_ = os.MkdirAll(cfg.MonitorPath, os.ModePerm)
+			}
+			//初始化全局
+			global.InitWatchFolder(cfg.MonitorPath)
+			must(err)
+			// 退出channel
+			exitChannel := make(chan os.Signal)
+			defer close(exitChannel)
+			// 退出信号监听
+			go func(c chan os.Signal) {
+				signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
+			}(exitChannel)
+			//开始监控
+			go global.WatchEvent()
+			//go func() {
+			//	for {
+			//		select {
+			//		case ev := <-global.Watch.Events:
+			//			if ev.Op&fsnotify.Create == fsnotify.Create {
+			//				logging.L().Info("创建文件", zap.String("文件名", ev.Name))
+			//				global.FileList.Add(ev.Name)
+			//			}
+			//		case <-t1.C:
+			//			{
+			//				if global.FileList.Size() > 0 {
+			//				}
+			//			}
+			//		case err := <-global.Watch.Errors:
+			//			{
+			//				log.Println("error : ", err)
+			//				return
+			//			}
+			//	}
+			//}()
+			select {
+			case <-ctx.Done():
+				global.Logger.With(
+					zap.String(cfg.Name, "exit"),
+				).Error(ctx.Err().Error())
+				return
+			case errs := <-exitChannel:
+				store.Save(cfg.TempPath, global.FileHistoryMap)
+				global.Logger.With(
+					zap.String(cfg.Name, "服务退出"),
+				).Info(errs.String())
+				os.Exit(-1)
+				return
+			}
+		},
+	}
+	cmd.Flags().StringVar(&ConfigFileFlag, "c", "./config/config.yaml", "The configuration file path")
+	cmd.Flags().StringVar(&NodeName, "n", "main-node", "The configuration name")
+	cmd.Flags().StringVar(&Mode, "m", "dev", "run mode : dev | test | releases")
+	cmd.Flags().StringVar(&Path, "f", "", "monitor folder path")
+	return cmd
+}

config/config.go

@@ -0,0 +1,77 @@
+package config
+
+import (
+	"os"
+
+	"git.hpds.cc/Component/logging"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type FolderMonitorConfig struct {
+	Name        string       `yaml:"name" json:"name" toml:"name"`
+	MonitorPath string       `json:"monitorPath" yaml:"monitorPath" toml:"monitorPath"`
+	TempPath    string       `json:"tempPath" yaml:"tempPath" toml:"tempPath"`
+	DatasetId   int          `yaml:"datasetId" json:"datasetId" toml:"datasetId"`
+	Logging     LogOptions   `yaml:"logging" json:"logging" tom:"logging"`
+	Node        HpdsNode     `yaml:"node,omitempty" json:"node,omitempty" toml:"node,omitempty"`
+	Funcs       []FuncConfig `yaml:"functions,omitempty" json:"funcs,omitempty" toml:"funcs,omitempty"`
+}
+
+type LogOptions struct {
+	Path            string `yaml:"path" json:"path" toml:"path"`                                  // 文件保存地方
+	Prefix          string `yaml:"prefix" json:"prefix" toml:"prefix"`                            // 日志文件前缀
+	ErrorFileSuffix string `yaml:"errorFileSuffix" json:"errorFileSuffix" toml:"errorFileSuffix"` // error日志文件后缀
+	WarnFileSuffix  string `yaml:"warnFileSuffix" json:"warnFileSuffix" toml:"warnFileSuffix"`    // warn日志文件后缀
+	InfoFileSuffix  string `yaml:"infoFileSuffix" json:"infoFileSuffix" toml:"infoFileSuffix"`    // info日志文件后缀
+	DebugFileSuffix string `yaml:"debugFileSuffix" json:"debugFileSuffix" toml:"debugFileSuffix"` // debug日志文件后缀
+	Level           string `yaml:"level" json:"level" toml:"level"`                               // 日志等级
+	MaxSize         int    `yaml:"maxSize" json:"maxSize" toml:"maxSize"`                         // 日志文件大小（M）
+	MaxBackups      int    `yaml:"maxBackups" json:"maxBackups" toml:"maxBackups"`                // 最多存在多少个切片文件
+	MaxAge          int    `yaml:"maxAge" json:"maxAge" toml:"maxAge"`                            // 保存的最大天数
+	Development     bool   `yaml:"development" json:"development" toml:"development"`             // 是否是开发模式
+}
+
+type HpdsNode struct {
+	Host  string `yaml:"host"`
+	Port  int    `yaml:"port"`
+	Token string `yaml:"token,omitempty"`
+}
+
+type FuncConfig struct {
+	Name    string `yaml:"name"`
+	DataTag uint8  `yaml:"dataTag"`
+	MqType  uint   `yaml:"mqType"` //消息类型， 发布，1；订阅；2
+}
+
+func ParseConfigByFile(path string) (cfg *FolderMonitorConfig, err error) {
+	buffer, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return load(buffer)
+}
+
+func load(buf []byte) (cfg *FolderMonitorConfig, err error) {
+	//cViper := viper.New()
+	//cViper.SetConfigType("yaml")
+	cfg = new(FolderMonitorConfig)
+	cfg.Funcs = make([]FuncConfig, 0)
+	err = yaml.Unmarshal(buf, cfg)
+	return
+}
+
+func LoadLoggerConfig(opt LogOptions) *logging.Logger {
+	return logging.NewLogger(
+		logging.SetPath(opt.Path),
+		logging.SetPrefix(opt.Prefix),
+		logging.SetDevelopment(opt.Development),
+		logging.SetDebugFileSuffix(opt.DebugFileSuffix),
+		logging.SetWarnFileSuffix(opt.WarnFileSuffix),
+		logging.SetErrorFileSuffix(opt.ErrorFileSuffix),
+		logging.SetInfoFileSuffix(opt.InfoFileSuffix),
+		logging.SetMaxAge(opt.MaxAge),
+		logging.SetMaxBackups(opt.MaxBackups),
+		logging.SetMaxSize(opt.MaxSize),
+		logging.SetLevel(logging.LogLevel["debug"]),
+	)
+}

config/config.yaml

@@ -0,0 +1,23 @@
+name: "folder_monitoring"
+monitorPath: "./src/"
+tempPath: "./tmp/"
+datasetId: 1
+node:
+  host: 127.0.0.1
+  port: 27188
+  token: 06d36c6f5705507dae778fdce90d0767
+functions:
+  - name: dataset-request
+    dataTag : 10
+    mqType: 1
+logging:
+  path: ./logs
+  prefix: folder_monitor
+  errorFileSuffix: error.log
+  warnFileSuffix: warn.log
+  infoFileSuffix: info.log
+  debugFileSuffix: debug.log
+  maxSize: 100
+  maxBackups: 3000
+  maxAge: 30
+  development: true

global/global.go

@@ -0,0 +1,219 @@
+package global
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"folder_monitoring/config"
+	"folder_monitoring/mq"
+	"folder_monitoring/store"
+	"git.hpds.cc/Component/logging"
+	"git.hpds.cc/pavement/hpds_node"
+	"github.com/emirpasic/gods/lists/arraylist"
+	"github.com/fsnotify/fsnotify"
+	"go.uber.org/zap"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var (
+	FileList       *arraylist.List
+	Watch          *fsnotify.Watcher
+	FileHistoryMap map[string]*store.FileStatus
+	Cfg            *config.FolderMonitorConfig
+	Logger         *logging.Logger
+	//encoderPool    *sync.Pool
+)
+
+func init() {
+	FileHistoryMap = make(map[string]*store.FileStatus)
+	FileList = arraylist.New()
+	Watch, _ = fsnotify.NewWatcher()
+	//encoderPool = syncpool.NewEncoderPool(zstd.WithEncoderConcurrency(1), zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
+}
+
+func InitWatchFolder(folderPath string) {
+	err := filepath.Walk(folderPath, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			Logger.With(
+				zap.String("监控的文件夹", path),
+			).Error(err.Error())
+			return err
+		}
+
+		if !fi.IsDir() {
+			item, ok := FileHistoryMap[fi.Name()]
+			if ok {
+				if !item.TransferStatus {
+					FileList.Add(fi.Name())
+				}
+			} else {
+				mdStr, _ := FileMD5(path)
+				item = &store.FileStatus{
+					FileName:   fi.Name(),
+					FilePath:   path,
+					CreateTime: fi.ModTime(),
+					FileMd5:    mdStr,
+				}
+				FileHistoryMap[path] = item
+				FileList.Add(path)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		Logger.With(
+			zap.String("文件夹遍历出错", "InitWatchFolder"),
+		).Error(err.Error())
+		return
+	}
+	_ = Watch.Add(folderPath)
+}
+
+func WatchEvent() {
+	t1 := time.NewTicker(1 * time.Second)
+	for {
+		select {
+		case ev := <-Watch.Events:
+			{
+				//判断事件发生的类型，如下5种
+				// Create 创建
+				// Write 写入
+				// Remove 删除
+				// Rename 重命名
+				// Chmod 修改权限
+				if ev.Op&fsnotify.Create == fsnotify.Create {
+					Logger.Info("创建文件", zap.String("文件名", ev.Name))
+					FileHistoryMap[ev.Name] = &store.FileStatus{
+						FileName:       ev.Name,
+						FilePath:       ev.Name,
+						CreateTime:     time.Now(),
+						FileMd5:        "",
+						TransferStatus: false,
+					}
+					FileList.Add(ev.Name)
+				}
+			}
+		case <-t1.C:
+			{
+				if FileList.Size() > 0 {
+					transferData(FileList)
+				}
+			}
+		case err := <-Watch.Errors:
+			{
+				Logger.With(
+					zap.String("文件夹监控出错", "WatchEvent"),
+				).Error(err.Error())
+				return
+			}
+		}
+	}
+}
+
+func PathExists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	if os.IsNotExist(err) { //如果返回的错误类型使用os.isNotExist()判断为true，说明文件或者文件夹不存在
+		return false
+	}
+	return false
+}
+
+// ReadFile 读取到file中，再利用ioutil将file直接读取到[]byte中, 这是最优
+func ReadFile(fn string) []byte {
+	f, err := os.Open(fn)
+	if err != nil {
+		logging.L().Error("Read File", zap.String("File Name", fn), zap.Error(err))
+		return nil
+	}
+	defer func(f *os.File) {
+		_ = f.Close()
+	}(f)
+
+	fd, err := io.ReadAll(f)
+	if err != nil {
+		logging.L().Error("Read File To buff", zap.String("File Name", fn), zap.Error(err))
+		return nil
+	}
+
+	return fd
+}
+
+func transferData(list *arraylist.List) {
+	var (
+		mu            sync.RWMutex
+		wg            sync.WaitGroup
+		compressRatio float64 = 0
+		compressTime  time.Duration
+	)
+	mu.Lock()
+	defer mu.Unlock()
+	startTime := time.Now()
+	cli := mq.GetMqClient("dataset-request", 1)
+	for i := 0; i < list.Size(); i++ {
+		if fn, ok := list.Get(0); ok {
+			if PathExists(fn.(string)) {
+				wg.Add(1)
+				go func() {
+					fileName := fn.(string)
+					src := ReadFile(fileName)
+					if src != nil {
+						dstContent := store.Compress(src)
+						item := mq.FileTransferInfo{
+							FileName:   fileName,
+							FilePath:   FileHistoryMap[fileName].FilePath,
+							DatasetId:  Cfg.DatasetId,
+							File:       string(dstContent),
+							IsCompress: true,
+							FileMd5:    FileHistoryMap[fileName].FileMd5,
+						}
+						payload := mq.InstructionReq{
+							Command: mq.DatasetRequest,
+							Payload: item,
+						}
+						s, _ := json.Marshal(payload)
+						list.Remove(0)
+						err := mq.GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), s, Logger)
+						if err != nil {
+							Logger.With(
+								zap.String("文件名称", fileName),
+							).Error("文件传输", zap.Error(err))
+							list.Add(fileName)
+						}
+						FileHistoryMap[fileName].TransferStatus = true
+						Logger.With(
+							zap.String("文件名称", fileName),
+						).Info("文件传输")
+						compressRatio += float64(len(dstContent)) * 100 / float64(len(src))
+					}
+				}()
+			}
+		}
+		wg.Wait()
+		compressTime = time.Since(startTime)
+		Logger.Info("批量任务完成",
+			zap.Int("压缩数量", list.Size()),
+			zap.String("平均压缩率", fmt.Sprintf("%.2f%%", compressRatio/float64(list.Size()))),
+			zap.Duration("运行时间", compressTime),
+		)
+	}
+}
+func FileMD5(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		Logger.With(
+			zap.String("获取文件MD5错误", filePath),
+		).Error(err.Error())
+		return "", err
+	}
+	hash := md5.New()
+	_, _ = io.Copy(hash, file)
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}

go.mod

@@ -0,0 +1,41 @@
+module folder_monitoring
+
+go 1.19
+
+require (
+	git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b
+	git.hpds.cc/pavement/hpds_node v0.0.0-20230326152949-a1c0ad2f7052
+	github.com/emirpasic/gods v1.18.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/klauspost/compress v1.16.3
+	// github.com/mostynb/zstdpool-syncpool v0.0.12
+	github.com/spf13/cobra v1.6.1
+	go.uber.org/zap v1.23.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 // indirect
+	git.hpds.cc/Component/network v0.0.0-20230326151855-3c157f531d86 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/lucas-clemente/quic-go v0.29.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
+	github.com/matoous/go-nanoid/v2 v2.0.0 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+)

main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"folder_monitoring/cmd"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:     "file_monitoring",
+		Long:    "file_monitoring is a file monitoring and transfer tool",
+		Version: "v 1.0",
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(cmd.NewStartCmd())
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		_, _ = fmt.Fprint(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}

mq/index.go

@@ -0,0 +1,93 @@
+package mq
+
+import (
+	"fmt"
+	"folder_monitoring/config"
+	"git.hpds.cc/Component/logging"
+	"go.uber.org/zap"
+	"os"
+	"time"
+
+	"git.hpds.cc/pavement/hpds_node"
+)
+
+var MqList []HpdsMqNode
+
+type HpdsMqNode struct {
+	MqType   uint
+	Topic    string
+	Node     config.HpdsNode
+	EndPoint interface{}
+}
+
+func must(logger *logging.Logger, err error) {
+	if err != nil {
+		if logger != nil {
+			logger.With(zap.String("web节点", "错误信息")).Error("启动错误", zap.Error(err))
+		} else {
+			_, _ = fmt.Fprint(os.Stderr, err)
+		}
+		os.Exit(1)
+	}
+}
+
+func NewMqClient(funcs []config.FuncConfig, node config.HpdsNode, logger *logging.Logger) (mqList []HpdsMqNode, err error) {
+	mqList = make([]HpdsMqNode, 0)
+	for _, v := range funcs {
+		switch v.MqType {
+		case 2:
+			sf := hpds_node.NewStreamFunction(
+				v.Name,
+				hpds_node.WithMqAddr(fmt.Sprintf("%s:%d", node.Host, node.Port)),
+				hpds_node.WithObserveDataTags(v.DataTag),
+				hpds_node.WithCredential(node.Token),
+			)
+			err = sf.Connect()
+			nodeInfo := HpdsMqNode{
+				MqType:   2,
+				Topic:    v.Name,
+				Node:     node,
+				EndPoint: sf,
+			}
+			must(logger, err)
+			mqList = append(mqList, nodeInfo)
+		default:
+			ap := hpds_node.NewAccessPoint(
+				v.Name,
+				hpds_node.WithMqAddr(fmt.Sprintf("%s:%d", node.Host, node.Port)),
+				hpds_node.WithCredential(node.Token),
+			)
+			err = ap.Connect()
+			nodeInfo := HpdsMqNode{
+				MqType:   1,
+				Topic:    v.Name,
+				Node:     node,
+				EndPoint: ap,
+			}
+			must(logger, err)
+			ap.SetDataTag(v.DataTag)
+			mqList = append(mqList, nodeInfo)
+		}
+
+	}
+	return mqList, err
+}
+
+func GetMqClient(topic string, mqType uint) *HpdsMqNode {
+	for _, v := range MqList {
+		if v.Topic == topic && v.MqType == mqType {
+			return &v
+		}
+	}
+	return nil
+}
+
+func GenerateAndSendData(stream hpds_node.AccessPoint, data []byte, logger *logging.Logger) error {
+	logger.With(zap.String("web节点", "发送消息")).Info("数据", zap.String("发送的数据", string(data)))
+	_, err := stream.Write(data)
+	if err != nil {
+		return err
+	}
+	time.Sleep(1000 * time.Millisecond)
+	return nil
+}

mq/instruction.go

@@ -0,0 +1,20 @@
+package mq
+
+const (
+	DatasetRequest = iota + 10
+	DatasetResponse
+)
+
+type InstructionReq struct {
+	Command int         `json:"command"`
+	Payload interface{} `json:"payload"`
+}
+
+type FileTransferInfo struct {
+	FileName   string
+	FilePath   string
+	DatasetId  int
+	File       string
+	IsCompress bool
+	FileMd5    string
+}

store/index.go

@@ -0,0 +1,116 @@
+package store
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+type FileStatus struct {
+	FileName       string    `json:"fileName"`
+	FilePath       string    `json:"filePath"`
+	CreateTime     time.Time `json:"createTime"`
+	FileMd5        string    `json:"fileMd5"`
+	TransferStatus bool      `json:"transferStatus"`
+}
+
+// PathExists 判断所给路径文件/文件夹是否存在
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	//IsNotExist来判断，是不是不存在的错误
+	if os.IsNotExist(err) { //如果返回的错误类型使用os.isNotExist()判断为true，说明文件或者文件夹不存在
+		return false, nil
+	}
+	return false, err //如果有错误了，但是不是不存在的错误，所以把这个错误原封不动的返回
+}
+
+func Load(storePath string, list map[string]*FileStatus) error {
+	if b, _ := PathExists(storePath); !b {
+		_ = os.MkdirAll(storePath, os.ModePerm)
+	}
+	fileName := strings.Replace(storePath, ":\\", "_", -1)
+	fileName = strings.Replace(storePath, "\\", "_", -1)
+	fileName = strings.Replace(storePath, "/", "_", -1)
+	storeFile := path.Join(storePath, fmt.Sprintf("%s.hdb", fileName))
+	if b, _ := PathExists(storeFile); !b {
+		NewFile(storeFile)
+		return nil
+	}
+	f, _ := os.OpenFile(storeFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	defer func(f *os.File) {
+		_ = f.Close()
+	}(f)
+	buff, err := io.ReadAll(f)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+	if len(buff) > 0 {
+		str, err := UnCompress(buff)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(str, &list)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func Save(storePath string, list map[string]*FileStatus) {
+	if b, _ := PathExists(storePath); !b {
+		_ = os.MkdirAll(storePath, os.ModePerm)
+	}
+	fileName := strings.Replace(storePath, ":\\", "_", -1)
+	fileName = strings.Replace(storePath, "\\", "_", -1)
+	fileName = strings.Replace(storePath, "/", "_", -1)
+	storeFile := path.Join(storePath, fmt.Sprintf("%s.hdb", fileName))
+	if b, _ := PathExists(storeFile); !b {
+		NewFile(storeFile)
+	}
+	f, _ := os.OpenFile(storeFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	defer func() {
+		_ = f.Close()
+	}()
+	str, _ := json.Marshal(list)
+	c := Compress(str)
+	_, _ = f.Write(c)
+}
+func NewFile(fileName string) {
+	f, _ := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	defer func(f *os.File) {
+		_ = f.Close()
+	}(f)
+}
+
+// Compress 压缩
+func Compress(src []byte) []byte {
+	encoder, _ := zstd.NewWriter(nil)
+	zstd.WithEncoderConcurrency(3)
+	return encoder.EncodeAll(src, make([]byte, 0, len(src)))
+}
+
+func UnCompress(src []byte) ([]byte, error) {
+	d, err := zstd.NewReader(nil)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	uncompressed, err := d.DecodeAll(src, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return uncompressed, nil
+}