493 lines
18 KiB
Go
493 lines
18 KiB
Go
|
package download
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bytes"
|
|||
|
|
"encoding/hex"
|
|||
|
|
"errors"
|
|||
|
|
"fmt"
|
|||
|
|
"git.hpds.cc/Component/logging"
|
|||
|
|
"go.uber.org/zap"
|
|||
|
|
"io"
|
|||
|
|
"net/http"
|
|||
|
|
"os"
|
|||
|
|
"strconv"
|
|||
|
|
"strings"
|
|||
|
|
"sync"
|
|||
|
|
"time"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
OneThreadDownloadSize = 1024 * 1024 * 2 // 一个线程下载文件的大小
|
|||
|
|
ThreadCount = 6
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
type Task struct {
|
|||
|
|
customFunc func(params interface{}) // 执行方法
|
|||
|
|
paramsInfo interface{} // 执行方法参数
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type ThreadController struct {
|
|||
|
|
TaskQueue chan Task // 用于接收下载任务
|
|||
|
|
TaskCount chan int // 用于记载当前任务数量
|
|||
|
|
Exit chan int // 用于记载当前任务数量
|
|||
|
|
ThreadCount int // 最大协程数
|
|||
|
|
WaitGroup sync.WaitGroup // 等待协程完成
|
|||
|
|
RangeStrs map[int]string // 所有需要下载的文件名
|
|||
|
|
FileUrl string // 下载链接
|
|||
|
|
DownloadResultInfoChan chan DownFileParams // 下载任务响应通道
|
|||
|
|
DownloadFolder string // 下载文件保存文件夹
|
|||
|
|
DownloadFileName string // 下载文件保存文件名
|
|||
|
|
Filenames []string // 子文件名，有序
|
|||
|
|
Logger *logging.Logger //日志
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type DownFileParams struct {
|
|||
|
|
UrlStr string
|
|||
|
|
RangeStr string
|
|||
|
|
RangeIndex int
|
|||
|
|
TempFilename string
|
|||
|
|
Succeed bool
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var fileTypeMap sync.Map
|
|||
|
|
|
|||
|
|
func init() { //用于判断文件名的后缀
|
|||
|
|
fileTypeMap.Store("ffd8ffe000104a464946", "jpg") //JPEG (jpg)
|
|||
|
|
fileTypeMap.Store("89504e470d0a1a0a0000", "png") //PNG (png)
|
|||
|
|
fileTypeMap.Store("47494638396126026f01", "gif") //GIF (gif)
|
|||
|
|
fileTypeMap.Store("49492a00227105008037", "tif") //TIFF (tif)
|
|||
|
|
fileTypeMap.Store("424d228c010000000000", "bmp") //16色位图(bmp)
|
|||
|
|
fileTypeMap.Store("424d8240090000000000", "bmp") //24位位图(bmp)
|
|||
|
|
fileTypeMap.Store("424d8e1b030000000000", "bmp") //256色位图(bmp)
|
|||
|
|
fileTypeMap.Store("41433130313500000000", "dwg") //CAD (dwg)
|
|||
|
|
fileTypeMap.Store("3c21444f435459504520", "html") //HTML (html) 3c68746d6c3e0 3c68746d6c3e0
|
|||
|
|
fileTypeMap.Store("3c68746d6c3e0", "html") //HTML (html) 3c68746d6c3e0 3c68746d6c3e0
|
|||
|
|
fileTypeMap.Store("3c21646f637479706520", "htm") //HTM (htm)
|
|||
|
|
fileTypeMap.Store("48544d4c207b0d0a0942", "css") //css
|
|||
|
|
fileTypeMap.Store("696b2e71623d696b2e71", "js") //js
|
|||
|
|
fileTypeMap.Store("7b5c727466315c616e73", "rtf") //Rich Text Format (rtf)
|
|||
|
|
fileTypeMap.Store("38425053000100000000", "psd") //Photoshop (psd)
|
|||
|
|
fileTypeMap.Store("46726f6d3a203d3f6762", "eml") //Email [Outlook Express 6] (eml)
|
|||
|
|
fileTypeMap.Store("d0cf11e0a1b11ae10000", "doc") //MS Excel 注意：word、msi 和 excel的文件头一样
|
|||
|
|
fileTypeMap.Store("d0cf11e0a1b11ae10000", "vsd") //Visio 绘图
|
|||
|
|
fileTypeMap.Store("5374616E64617264204A", "mdb") //MS Access (mdb)
|
|||
|
|
fileTypeMap.Store("252150532D41646F6265", "ps")
|
|||
|
|
fileTypeMap.Store("255044462d312e350d0a", "pdf") //Adobe Acrobat (pdf)
|
|||
|
|
fileTypeMap.Store("2e524d46000000120001", "rmvb") //rmvb/rm相同
|
|||
|
|
fileTypeMap.Store("464c5601050000000900", "flv") //flv与f4v相同
|
|||
|
|
fileTypeMap.Store("00000020667479706d70", "mp4")
|
|||
|
|
fileTypeMap.Store("49443303000000002176", "mp3")
|
|||
|
|
fileTypeMap.Store("000001ba210001000180", "mpg") //
|
|||
|
|
fileTypeMap.Store("3026b2758e66cf11a6d9", "wmv") //wmv与asf相同
|
|||
|
|
fileTypeMap.Store("52494646e27807005741", "wav") //Wave (wav)
|
|||
|
|
fileTypeMap.Store("52494646d07d60074156", "avi")
|
|||
|
|
fileTypeMap.Store("4d546864000000060001", "mid") //MIDI (mid)
|
|||
|
|
fileTypeMap.Store("504b0304140000000800", "zip")
|
|||
|
|
fileTypeMap.Store("526172211a0700cf9073", "rar")
|
|||
|
|
fileTypeMap.Store("235468697320636f6e66", "ini")
|
|||
|
|
fileTypeMap.Store("504b03040a0000000000", "jar")
|
|||
|
|
fileTypeMap.Store("4d5a9000030000000400", "exe") //可执行文件
|
|||
|
|
fileTypeMap.Store("3c25402070616765206c", "jsp") //jsp文件
|
|||
|
|
fileTypeMap.Store("4d616e69666573742d56", "mf") //MF文件
|
|||
|
|
fileTypeMap.Store("3c3f786d6c2076657273", "xml") //xml文件
|
|||
|
|
fileTypeMap.Store("494e5345525420494e54", "sql") //xml文件
|
|||
|
|
fileTypeMap.Store("7061636b616765207765", "java") //java文件
|
|||
|
|
fileTypeMap.Store("406563686f206f66660d", "bat") //bat文件
|
|||
|
|
fileTypeMap.Store("1f8b0800000000000000", "gz") //gz文件
|
|||
|
|
fileTypeMap.Store("6c6f67346a2e726f6f74", "properties") //bat文件
|
|||
|
|
fileTypeMap.Store("cafebabe0000002e0041", "class") //bat文件
|
|||
|
|
fileTypeMap.Store("49545346030000006000", "chm") //bat文件
|
|||
|
|
fileTypeMap.Store("04000000010000001300", "mxp") //bat文件
|
|||
|
|
fileTypeMap.Store("504b0304140006000800", "docx") //docx文件
|
|||
|
|
fileTypeMap.Store("d0cf11e0a1b11ae10000", "wps") //WPS文字wps、表格et、演示dps都是一样的
|
|||
|
|
fileTypeMap.Store("6431303a637265617465", "torrent")
|
|||
|
|
fileTypeMap.Store("6D6F6F76", "mov") //Quicktime (mov)
|
|||
|
|
fileTypeMap.Store("FF575043", "wpd") //WordPerfect (wpd)
|
|||
|
|
fileTypeMap.Store("CFAD12FEC5FD746F", "dbx") //Outlook Express (dbx)
|
|||
|
|
fileTypeMap.Store("2142444E", "pst") //Outlook (pst)
|
|||
|
|
fileTypeMap.Store("AC9EBD8F", "qdf") //Quicken (qdf)
|
|||
|
|
fileTypeMap.Store("E3828596", "pwl") //Windows Password (pwl)
|
|||
|
|
fileTypeMap.Store("2E7261FD", "ram") //Real Audio (ram)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 获取前面结果字节的二进制
|
|||
|
|
func bytesToHexString(src []byte) string {
|
|||
|
|
res := bytes.Buffer{}
|
|||
|
|
if src == nil || len(src) <= 0 {
|
|||
|
|
return ""
|
|||
|
|
}
|
|||
|
|
temp := make([]byte, 0)
|
|||
|
|
for _, v := range src {
|
|||
|
|
sub := v & 0xFF
|
|||
|
|
hv := hex.EncodeToString(append(temp, sub))
|
|||
|
|
if len(hv) < 2 {
|
|||
|
|
res.WriteString(strconv.FormatInt(int64(0), 10))
|
|||
|
|
}
|
|||
|
|
res.WriteString(hv)
|
|||
|
|
}
|
|||
|
|
return res.String()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func SafeMkdir(folder string) {
|
|||
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|||
|
|
_ = os.MkdirAll(folder, os.ModePerm)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// GetFileType 用文件前面几个字节来判断
|
|||
|
|
// fSrc: 文件字节流（就用前面几个字节）
|
|||
|
|
func GetFileType(fSrc []byte) string {
|
|||
|
|
var fileType string
|
|||
|
|
fileCode := bytesToHexString(fSrc)
|
|||
|
|
|
|||
|
|
fileTypeMap.Range(func(key, value interface{}) bool {
|
|||
|
|
k := key.(string)
|
|||
|
|
v := value.(string)
|
|||
|
|
if strings.HasPrefix(fileCode, strings.ToLower(k)) ||
|
|||
|
|
strings.HasPrefix(k, strings.ToLower(fileCode)) {
|
|||
|
|
fileType = v
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
return true
|
|||
|
|
})
|
|||
|
|
return fileType
|
|||
|
|
}
|
|||
|
|
func GetBytesFile(filename string, bufferSize int) []byte {
|
|||
|
|
file, err := os.Open(filename)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println(err)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
defer func() {
|
|||
|
|
_ = file.Close()
|
|||
|
|
}()
|
|||
|
|
|
|||
|
|
buffer := make([]byte, bufferSize)
|
|||
|
|
_, err = file.Read(buffer)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println(err)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
return buffer
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) GetSuffix(contentType string) string {
|
|||
|
|
suffix := ""
|
|||
|
|
contentTypes := map[string]string{
|
|||
|
|
"image/gif": "gif",
|
|||
|
|
"image/jpeg": "jpg",
|
|||
|
|
"application/x-img": "img",
|
|||
|
|
"image/png": "png",
|
|||
|
|
"application/json": "json",
|
|||
|
|
"application/pdf": "pdf",
|
|||
|
|
"application/msword": "word",
|
|||
|
|
"application/octet-stream": "rar",
|
|||
|
|
"application/x-zip-compressed": "zip",
|
|||
|
|
"application/x-msdownload": "exe",
|
|||
|
|
"video/mpeg4": "mp4",
|
|||
|
|
"video/avi": "avi",
|
|||
|
|
"audio/mp3": "mp3",
|
|||
|
|
"text/css": "css",
|
|||
|
|
"application/x-javascript": "js",
|
|||
|
|
"application/vnd.android.package-archive": "apk",
|
|||
|
|
}
|
|||
|
|
for key, value := range contentTypes {
|
|||
|
|
if strings.Contains(contentType, key) {
|
|||
|
|
suffix = value
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return suffix
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) Put(task Task) {
|
|||
|
|
// 用于开启单个协程任务，下载文件的部分内容
|
|||
|
|
defer func() {
|
|||
|
|
err := recover() //内置函数，可以捕捉到函数异常
|
|||
|
|
if err != nil {
|
|||
|
|
controller.Logger.With(zap.String("文件下载", "错误信息")).Error("recover 错误", zap.Any("错误信息", err))
|
|||
|
|
}
|
|||
|
|
}()
|
|||
|
|
controller.WaitGroup.Add(1) // 每插入一个任务，就需要计数
|
|||
|
|
controller.TaskCount <- 1 // 含缓冲区的通道，用于控制下载器的协程最大数量
|
|||
|
|
controller.TaskQueue <- task // 插入下载任务
|
|||
|
|
//go task.customFunc(task.paramsInfo)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) DownloadFile(paramsInfo interface{}) {
|
|||
|
|
// 下载任务，接收对应的参数，负责从网页中下载对应部分的文件资源
|
|||
|
|
defer func() {
|
|||
|
|
controller.WaitGroup.Done() // 下载任务完成，协程结束
|
|||
|
|
}()
|
|||
|
|
switch paramsInfo.(type) {
|
|||
|
|
case DownFileParams:
|
|||
|
|
params := paramsInfo.(DownFileParams)
|
|||
|
|
params.Succeed = false
|
|||
|
|
defer func() {
|
|||
|
|
err := recover() //内置函数，可以捕捉到函数异常
|
|||
|
|
if err != nil {
|
|||
|
|
// 如果任意环节出错，表明下载流程未成功完成，标记下载失败
|
|||
|
|
controller.Logger.With(zap.String("文件下载", "错误信息")).Error("recover 错误", zap.Any("错误信息", err))
|
|||
|
|
params.Succeed = false
|
|||
|
|
}
|
|||
|
|
}()
|
|||
|
|
//fmt.Println("Start to down load " + params.UrlStr + ", Content-type: " + params.RangeStr + " , save to file: " + params.TempFilename)
|
|||
|
|
urlStr := params.UrlStr
|
|||
|
|
rangeStr := params.RangeStr
|
|||
|
|
tempFilename := params.TempFilename
|
|||
|
|
_ = os.Remove(tempFilename) // 删除已有的文件, 避免下载的数据被污染
|
|||
|
|
// 发起文件下载请求
|
|||
|
|
req, _ := http.NewRequest("GET", urlStr, nil)
|
|||
|
|
req.Header.Add("Range", rangeStr) // 测试下载部分内容
|
|||
|
|
res, err := http.DefaultClient.Do(req) // 发出下载请求，等待回应
|
|||
|
|
if err != nil {
|
|||
|
|
controller.Logger.With(zap.String("文件下载", "错误信息")).Error("连接失败", zap.Error(err))
|
|||
|
|
params.Succeed = false // 无法连接, 标记下载失败
|
|||
|
|
} else if res.StatusCode != 206 {
|
|||
|
|
params.Succeed = false
|
|||
|
|
} else { // 能正常发起请求
|
|||
|
|
// 打开文件，写入文件
|
|||
|
|
fileObj, err := os.OpenFile(tempFilename, os.O_RDONLY|os.O_CREATE|os.O_APPEND, 0666)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println("Failed to open file " + tempFilename)
|
|||
|
|
controller.Logger.With(zap.String("文件下载", "错误信息")).
|
|||
|
|
With(zap.String("文件名", tempFilename)).
|
|||
|
|
Error("打开文件失败", zap.Error(err))
|
|||
|
|
|
|||
|
|
params.Succeed = false // 无法打开文件, 标记下载失败
|
|||
|
|
} else {
|
|||
|
|
defer func() {
|
|||
|
|
_ = fileObj.Close()
|
|||
|
|
}() // 关闭文件流
|
|||
|
|
body, err := io.ReadAll(res.Body) // 读取响应体的所有内容
|
|||
|
|
if err != nil {
|
|||
|
|
controller.Logger.With(zap.String("文件下载", "错误信息")).
|
|||
|
|
Error("读取返回值错误", zap.Error(err))
|
|||
|
|
params.Succeed = false
|
|||
|
|
} else {
|
|||
|
|
defer func() {
|
|||
|
|
_ = res.Body.Close()
|
|||
|
|
}() // 关闭连接流
|
|||
|
|
_, _ = fileObj.Write(body) // 写入字节数据到文件
|
|||
|
|
params.Succeed = true // 成功执行到最后一步，则表示下载成功
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
controller.DownloadResultInfoChan <- params // 将下载结果传入
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) Run() {
|
|||
|
|
// 只需要将待下载的请求发送一次即可，成功了会直接剔除，不成功则由接收方重试
|
|||
|
|
for rangeIndex, rangeStr := range controller.RangeStrs {
|
|||
|
|
params := DownFileParams{
|
|||
|
|
UrlStr: controller.FileUrl,
|
|||
|
|
RangeStr: rangeStr,
|
|||
|
|
TempFilename: controller.DownloadFolder + "/" + rangeStr,
|
|||
|
|
RangeIndex: rangeIndex,
|
|||
|
|
Succeed: true,
|
|||
|
|
} // 下载参数初始化
|
|||
|
|
task := Task{controller.DownloadFile, params}
|
|||
|
|
controller.Put(task) // 若通道满了会阻塞，等待空闲时再下载
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) ResultProcess(trunkSize int) string {
|
|||
|
|
// 负责处理各个协程下载资源的结果， 若成功则从下载列表中剔除，否则重新将该任务Put到任务列表中；超过5秒便会停止
|
|||
|
|
MaxRetryTime := 100
|
|||
|
|
nowRetryTime := 0
|
|||
|
|
resultMsg := ""
|
|||
|
|
for {
|
|||
|
|
select {
|
|||
|
|
case resultInfo := <-controller.DownloadResultInfoChan:
|
|||
|
|
<-controller.TaskCount // 取出一个计数器，表示一个协程已经完成
|
|||
|
|
if resultInfo.Succeed { // 成功下载该文件，清除文件名列表中的信息
|
|||
|
|
delete(controller.RangeStrs, resultInfo.RangeIndex) // 删除任务队列中的该任务（rangeStr队列）
|
|||
|
|
fmt.Println("Download progress -> " + strconv.FormatFloat((1.0-float64(len(controller.RangeStrs))/float64(trunkSize))*100, 'f', 2, 64) + "%")
|
|||
|
|
if len(controller.RangeStrs) == 0 {
|
|||
|
|
resultMsg = "SUCCESSED"
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
nowRetryTime += 1
|
|||
|
|
if nowRetryTime > MaxRetryTime { // 超过最大的重试次数退出下载
|
|||
|
|
resultMsg = "MAX_RETRY"
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
task := Task{
|
|||
|
|
customFunc: controller.DownloadFile,
|
|||
|
|
paramsInfo: resultInfo,
|
|||
|
|
} // 重新加载该任务
|
|||
|
|
go controller.Put(task)
|
|||
|
|
}
|
|||
|
|
case task := <-controller.TaskQueue:
|
|||
|
|
function := task.customFunc
|
|||
|
|
go function(task.paramsInfo)
|
|||
|
|
case <-time.After(5 * time.Second):
|
|||
|
|
resultMsg = "TIMEOUT"
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
if resultMsg == "MAX_RETRY" {
|
|||
|
|
fmt.Println("The network is unstable, exceeding the maximum number of downloads.")
|
|||
|
|
break
|
|||
|
|
} else if resultMsg == "SUCCESSED" {
|
|||
|
|
fmt.Println("Download file success!")
|
|||
|
|
break
|
|||
|
|
} else if resultMsg == "TIMEOUT" {
|
|||
|
|
fmt.Println("Download timeout!")
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
close(controller.TaskCount)
|
|||
|
|
close(controller.TaskQueue)
|
|||
|
|
close(controller.DownloadResultInfoChan)
|
|||
|
|
return resultMsg
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) Download(oneThreadDownloadSize int) bool {
|
|||
|
|
fmt.Println("Try to parse the object file...")
|
|||
|
|
length, rangeMaps, tempFilenames, contentType, err := TryDownload(controller.FileUrl, oneThreadDownloadSize)
|
|||
|
|
fmt.Println("File total size -> " + strconv.FormatFloat(float64(length)/(1024.0*1024.0), 'f', 2, 64) + "M")
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println("The file does not support multi-threaded download.")
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
fmt.Println("Parse the target file successfully, start downloading the target file...")
|
|||
|
|
controller.Init() // 初始化通道、分片等配置
|
|||
|
|
//oneThreadDownloadSize := 1024 * 1024 * 2 // 1024字节 = 1024bite = 1kb -> 2M
|
|||
|
|
oneThreadDownloadSize = 1024 * 1024 * 4 // 1024字节 = 1024bite = 1kb -> 4M
|
|||
|
|
filenames := make([]string, 0)
|
|||
|
|
for _, value := range tempFilenames {
|
|||
|
|
filenames = append(filenames, controller.DownloadFolder+"/"+value)
|
|||
|
|
}
|
|||
|
|
fileSuffix := controller.GetSuffix(contentType)
|
|||
|
|
filename := controller.DownloadFileName // 获取文件下载名
|
|||
|
|
controller.Filenames = filenames //下载文件的切片列表
|
|||
|
|
controller.RangeStrs = rangeMaps // 下载文件的Range范围
|
|||
|
|
go controller.Run() // 开始下载文件
|
|||
|
|
processResult := controller.ResultProcess(len(rangeMaps))
|
|||
|
|
downloadResult := false // 定义下载结果标记
|
|||
|
|
if processResult == "SUCCESSED" {
|
|||
|
|
absoluteFilename := controller.DownloadFolder + "/" + filename + "." + fileSuffix
|
|||
|
|
downloadResult = controller.CombineFiles(filename + "." + fileSuffix)
|
|||
|
|
if downloadResult {
|
|||
|
|
newSuffix := GetFileType(GetBytesFile(absoluteFilename, 10))
|
|||
|
|
err = os.Rename(absoluteFilename, controller.DownloadFolder+"/"+filename+"."+newSuffix)
|
|||
|
|
if err != nil {
|
|||
|
|
downloadResult = false
|
|||
|
|
fmt.Println("Combine file successes, Rename file failed " + absoluteFilename)
|
|||
|
|
} else {
|
|||
|
|
fmt.Println("Combine file successes, rename successes, new file name is -> " + controller.DownloadFolder + "/" + filename + "." + newSuffix)
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
fmt.Println("Failed to download file.")
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
fmt.Println("Failed to download file. Reason -> " + processResult)
|
|||
|
|
downloadResult = false
|
|||
|
|
}
|
|||
|
|
return downloadResult
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) CombineFiles(filename string) bool {
|
|||
|
|
_ = os.Remove(controller.DownloadFolder + "/" + filename)
|
|||
|
|
goalFile, err := os.OpenFile(controller.DownloadFolder+"/"+filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, os.ModePerm)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println("Failed to open file ")
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 正确的话应按照初始计算的文件名顺序合并，并且无缺失
|
|||
|
|
for _, value := range controller.Filenames {
|
|||
|
|
retryTime := 3
|
|||
|
|
tempFileBytes := make([]byte, 0)
|
|||
|
|
for retryTime > 0 {
|
|||
|
|
tempFileBytes = ReadFile(value)
|
|||
|
|
time.Sleep(100) // 休眠100毫秒，看看是不是文件加载错误
|
|||
|
|
if tempFileBytes != nil {
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
retryTime = retryTime - 1
|
|||
|
|
}
|
|||
|
|
_, _ = goalFile.Write(tempFileBytes)
|
|||
|
|
_ = os.Remove(value)
|
|||
|
|
}
|
|||
|
|
_ = goalFile.Close()
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func ReadFile(filename string) []byte {
|
|||
|
|
tempFile, err := os.OpenFile(filename, os.O_RDONLY, os.ModePerm)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println("Failed to open file " + filename)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
tempFileBytes, err := io.ReadAll(tempFile)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Println("Failed to read file data " + filename)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
_ = tempFile.Close()
|
|||
|
|
return tempFileBytes
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TryDownload(urlStr string, perThreadSize int) (int, map[int]string, []string, string, error) {
|
|||
|
|
// 尝试连接目标资源，目标资源是否可以使用多线程下载
|
|||
|
|
length := 0
|
|||
|
|
rangeMaps := make(map[int]string)
|
|||
|
|
req, _ := http.NewRequest("GET", urlStr, nil)
|
|||
|
|
req.Header.Add("Range", "bytes=0-1") // 测试下载部分内容
|
|||
|
|
res, err := http.DefaultClient.Do(req)
|
|||
|
|
contentType := ""
|
|||
|
|
rangeIndex := 1
|
|||
|
|
filenames := make([]string, 0)
|
|||
|
|
if err != nil {
|
|||
|
|
rangeMaps[rangeIndex] = urlStr
|
|||
|
|
return length, rangeMaps, filenames, contentType, errors.New("Failed to connect " + urlStr)
|
|||
|
|
}
|
|||
|
|
if res.StatusCode != 206 {
|
|||
|
|
rangeMaps[rangeIndex] = urlStr
|
|||
|
|
return length, rangeMaps, filenames, contentType, errors.New("Http status is not equal to 206! ")
|
|||
|
|
}
|
|||
|
|
// 206表示响应成功，仅仅返回部分内容
|
|||
|
|
contentLength := res.Header.Get("Content-Range")
|
|||
|
|
contentType = res.Header.Get("Content-Type")
|
|||
|
|
totalLength, err := strconv.Atoi(strings.Split(contentLength, "/")[1])
|
|||
|
|
if err != nil {
|
|||
|
|
return length, rangeMaps, filenames, contentType, errors.New("Can't calculate the content-length form server " + urlStr)
|
|||
|
|
}
|
|||
|
|
nowLength := 0 // 记录byte偏移量
|
|||
|
|
for {
|
|||
|
|
if nowLength >= totalLength {
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
var tempRangeStr string // 记录临时文件名
|
|||
|
|
if nowLength+perThreadSize >= totalLength {
|
|||
|
|
tempRangeStr = "bytes=" + strconv.Itoa(nowLength) + "-" + strconv.Itoa(totalLength-1)
|
|||
|
|
nowLength = totalLength
|
|||
|
|
} else {
|
|||
|
|
tempRangeStr = "bytes=" + strconv.Itoa(nowLength) + "-" + strconv.Itoa(nowLength+perThreadSize-1)
|
|||
|
|
nowLength = nowLength + perThreadSize
|
|||
|
|
}
|
|||
|
|
rangeMaps[rangeIndex] = tempRangeStr
|
|||
|
|
filenames = append(filenames, tempRangeStr)
|
|||
|
|
rangeIndex = rangeIndex + 1
|
|||
|
|
}
|
|||
|
|
return totalLength, rangeMaps, filenames, contentType, nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (controller *ThreadController) Init() {
|
|||
|
|
taskQueue := make(chan Task, controller.ThreadCount)
|
|||
|
|
taskCount := make(chan int, controller.ThreadCount+1)
|
|||
|
|
exit := make(chan int)
|
|||
|
|
downloadResultInfoChan := make(chan DownFileParams)
|
|||
|
|
controller.TaskQueue = taskQueue
|
|||
|
|
controller.TaskCount = taskCount
|
|||
|
|
controller.Exit = exit
|
|||
|
|
controller.DownloadResultInfoChan = downloadResultInfoChan
|
|||
|
|
controller.WaitGroup = sync.WaitGroup{}
|
|||
|
|
controller.RangeStrs = make(map[int]string)
|
|||
|
|
SafeMkdir(controller.DownloadFolder)
|
|||
|
|
}
|