You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
458 lines
11 KiB
458 lines
11 KiB
package util
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"scagent/config"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/schollz/progressbar/v3"
|
|
"go.uber.org/zap"
|
|
"gopkg.in/ini.v1"
|
|
)
|
|
|
|
// 遍历目录下的文件
|
|
// 是否只扫相对路径下
|
|
func GetDirFilePaths(dirPath string, relativeOnly bool) ([]string, error) {
|
|
var filePaths []string
|
|
err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.IsDir() {
|
|
if relativeOnly {
|
|
fileName := filepath.Base(path)
|
|
relativePath := filepath.ToSlash(filepath.Join(filepath.Base(dirPath), fileName))
|
|
filePaths = append(filePaths, relativePath)
|
|
} else {
|
|
filePaths = append(filePaths, filepath.ToSlash(path))
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return filePaths, nil
|
|
}
|
|
|
|
// base64 encode
|
|
// url safe
|
|
func Bas64end(str string) string {
|
|
// bdata:=
|
|
return base64.URLEncoding.EncodeToString([]byte(str))
|
|
}
|
|
|
|
// base64 url safe uneconde
|
|
func Base64dec(bsstr string) string {
|
|
dedc, _ := base64.URLEncoding.DecodeString(bsstr)
|
|
return string(dedc)
|
|
}
|
|
|
|
// 计算文件的hash
|
|
func CalacHash(rfile string) string {
|
|
// 获取到真实地址
|
|
//
|
|
file, err := os.Open(rfile)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer file.Close()
|
|
|
|
// initlize hash object
|
|
hash := sha256.New()
|
|
|
|
// write file into hash object
|
|
if _, err := io.Copy(hash, file); err != nil {
|
|
panic(err)
|
|
}
|
|
// get hash value
|
|
hashBytes := hash.Sum(nil)
|
|
//converto to hash string
|
|
hastString := fmt.Sprintf("%x", hashBytes)
|
|
return hastString
|
|
}
|
|
|
|
// 判断文件是否存在
|
|
func IsFileExist(filename string) bool {
|
|
if _, err := os.Stat(filename); err == nil {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// 发送文件
|
|
func SendFiles(filePath string, url string) error {
|
|
filePath = filepath.ToSlash(filePath)
|
|
fileInfo, err := os.Stat(filePath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return fmt.Errorf("file [%s] not exist", filePath)
|
|
}
|
|
return fmt.Errorf("file [%s] error: %w", filePath, err)
|
|
}
|
|
|
|
if fileInfo.IsDir() {
|
|
return postDirectory(filePath, url)
|
|
}
|
|
|
|
return postFile(filePath, path.Base(filePath), url)
|
|
}
|
|
|
|
// 传送文件夹
|
|
func postDirectory(dirPath string, url string) error {
|
|
files, err := GetDirFilePaths(dirPath, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("\nAll files in folder:")
|
|
for _, file := range files {
|
|
fmt.Println(file)
|
|
}
|
|
|
|
var confirm string
|
|
fmt.Print("\nTransfer all files? [Y/N] ")
|
|
fmt.Scanln(&confirm)
|
|
if strings.ToLower(confirm) != "y" {
|
|
fmt.Print("\nCancel send all files ")
|
|
return nil
|
|
}
|
|
|
|
for _, file := range files {
|
|
fileName, _ := filepath.Rel(dirPath, file)
|
|
fileName = filepath.Join(filepath.Base(dirPath), fileName)
|
|
err := postFile(file, fileName, url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
fmt.Printf("Send folder %s success.\n", dirPath)
|
|
return nil
|
|
}
|
|
|
|
// 传送文件
|
|
func postFile(wfpath string, filename string, url string) error {
|
|
payload := &bytes.Buffer{}
|
|
writer := multipart.NewWriter(payload)
|
|
|
|
file, err := os.Open(wfpath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
part, err := writer.CreateFormFile("file", filepath.ToSlash(filename))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fileInfo, _ := file.Stat()
|
|
bar := progressbar.DefaultBytes(
|
|
fileInfo.Size(),
|
|
fmt.Sprintf("Uploading [%s]", filename),
|
|
)
|
|
|
|
_, err = io.Copy(io.MultiWriter(part, bar), file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = writer.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodPost, url, payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
// 在头里面加上路径,去除前缀
|
|
rlpath := strings.TrimPrefix(wfpath, config.G.FilePath)
|
|
// fmt.Printf("wfpath:%s,rel path is:%s", wfpath, rlpath)
|
|
req.Header.Set("Fpath", Bas64end(rlpath))
|
|
|
|
client := &http.Client{}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("upload failed with status code: %d", resp.StatusCode)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
/*
|
|
* add file to zip
|
|
*/
|
|
func ZipFiles(zipname string, files []string, zippath string) error {
|
|
fmt.Printf("zipname:%s\n", zipname)
|
|
// 创建zip文件
|
|
newZipFile, err := os.Create(zipname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer newZipFile.Close()
|
|
// zip写头
|
|
zipWriter := zip.NewWriter(newZipFile)
|
|
defer zipWriter.Close()
|
|
|
|
// 把files添加到zip中
|
|
for _, file := range files {
|
|
//
|
|
fmt.Printf("zipfiles in function :%s\n", file)
|
|
|
|
//
|
|
zipfile, err := os.Open(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zipfile.Close()
|
|
|
|
//
|
|
info, err := zipfile.Stat()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
//
|
|
header, err := zip.FileInfoHeader(info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 在zip存档中设置文件的相对路径
|
|
header.Name, err = filepath.Rel(filepath.Dir(config.G.FilePath), file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 转换为Unix风格的路径分隔符
|
|
header.Name = filepath.ToSlash(header.Name)
|
|
// 目录需要拼上一个 "/" ，否则会出现一个和目录一样的文件在压缩包中
|
|
if info.IsDir() {
|
|
header.Name += "/"
|
|
} else {
|
|
// 将压缩方法设置为deflate
|
|
header.Method = zip.Deflate
|
|
}
|
|
//
|
|
fmt.Printf("header.name is %s\n", header.Name)
|
|
|
|
writer, err := zipWriter.CreateHeader(header)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err = io.Copy(writer, zipfile); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// 解压缩zip文件
|
|
// 这个方法必须是首字母大写的，才能被注册
|
|
// zpFname: 需要解压的zip文件
|
|
// dirpath: 解压到的目录
|
|
func DecompressZip(zpFname string, dirpath string) error {
|
|
// 下载文件
|
|
archive, err := zip.OpenReader(zpFname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 取zip文件的绝对路径的文件夹。会默认解压到files下，暂停使用
|
|
// dir := filepath.Dir(zpFname)
|
|
defer archive.Close()
|
|
// 遍历目录
|
|
for _, f := range archive.File {
|
|
// zip 解压的时候，路径为监听的路径
|
|
filePath := filepath.Join(dirpath, f.Name)
|
|
// 暂停原因 屏蔽解压到files 目录下
|
|
//filePath := filepath.Join(dir, f.Name)
|
|
if f.FileInfo().IsDir() {
|
|
os.MkdirAll(filePath, os.ModePerm)
|
|
continue
|
|
}
|
|
// 父文件夹开始创建目录
|
|
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
|
|
return fmt.Errorf("failed to make directory (%v)", err)
|
|
}
|
|
// 文件原有模式
|
|
dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create file (%v)", err)
|
|
}
|
|
defer dstFile.Close()
|
|
// 打开zip文件中的文件
|
|
fileInArchive, err := f.Open()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to open file in zip (%v)", err)
|
|
}
|
|
defer fileInArchive.Close()
|
|
// 复制文件到目标路径
|
|
if _, err := io.Copy(dstFile, fileInArchive); err != nil {
|
|
return fmt.Errorf("failed to copy file in zip (%v)", err)
|
|
}
|
|
// 保留原始文件的修改时间
|
|
if err := os.Chtimes(filePath, f.ModTime(), f.ModTime()); err != nil {
|
|
return fmt.Errorf("failed to set file times (%v)", err)
|
|
}
|
|
// nothing
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// dest：目的地址以及压缩文件名 eg:/data/logZip/2022-12-12-log.zip
|
|
// paths：需要压缩的所有文件所组成的集合
|
|
func CompressToZip(dest string, currentPath string, paths []string) error {
|
|
|
|
// 打开files 目录
|
|
filesPath := ""
|
|
if dir, err := os.Getwd(); err == nil {
|
|
filesPath = dir + "/sync_zips/"
|
|
}
|
|
// 目录格式 2025/10
|
|
dirpath := fmt.Sprintf("%d/%02d", time.Now().Year(), time.Now().Month())
|
|
// 组织目录路径
|
|
filesPath = filepath.Join(filesPath, dirpath)
|
|
// 创建目录
|
|
if err := os.MkdirAll(filesPath, os.ModePerm); err != nil {
|
|
fmt.Println(err.Error())
|
|
}
|
|
// ToSlash 过滤windows的斜杠引起的bug
|
|
zfile, err := os.Create(path.Join(filesPath, dest))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zfile.Close()
|
|
zipWriter := zip.NewWriter(zfile)
|
|
defer zipWriter.Close()
|
|
|
|
// 遍历带压缩的目录信息
|
|
for _, src := range paths {
|
|
// 替换文件名，并且去除前后 "\" 或 "/"
|
|
// path = strings.Trim(path, string(filepath.Separator))
|
|
// 从配置中读取到源目录
|
|
src = path.Join(currentPath, "/", src)
|
|
// 删除尾随路径（如果它是目录）
|
|
src = strings.TrimSuffix(src, string(os.PathSeparator))
|
|
// fmt.Printf("zip src:%s\n", src)
|
|
// 文件遍历
|
|
err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
|
// fmt.Printf("zip path:%s\n", path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 创建本地文件头
|
|
header, err := zip.FileInfoHeader(info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 监听的工作目录作为文件的结尾，然后取相对路径
|
|
// 在zip存档中设置文件的相对路径
|
|
// 将压缩包中的绝对路径替换为相对路径
|
|
header.Name, err = filepath.Rel(filepath.Dir(src), path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// 转换为Unix风格的路径分隔符
|
|
header.Name = filepath.ToSlash(header.Name)
|
|
|
|
// 目录需要拼上一个 "/" ，否则会出现一个和目录一样的文件在压缩包中
|
|
if info.IsDir() {
|
|
header.Name += "/"
|
|
} else {
|
|
// 将压缩方法设置为deflate
|
|
header.Method = zip.Deflate
|
|
}
|
|
// fmt.Printf("zip header:%v\n", header.Name)
|
|
// 创建写入头的writer
|
|
headerWriter, err := zipWriter.CreateHeader(header)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
// 保留文件的原有的时间
|
|
header.Modified = info.ModTime()
|
|
// 保留文件的原有的权限
|
|
header.SetMode(info.Mode())
|
|
_, err = io.Copy(headerWriter, f)
|
|
return err
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// 存到指定位置
|
|
os.Rename(dest, path.Join("./sync_zips/", dest))
|
|
return nil
|
|
}
|
|
|
|
// 读取配置文件
|
|
func ReadConfig() {
|
|
logger := NewProductionLogger()
|
|
defer logger.Sync()
|
|
// 加载文件
|
|
cfg, err := ini.Load("app.conf")
|
|
if err != nil {
|
|
logger.Error("读取配置文件失败", zap.Error(err))
|
|
return
|
|
}
|
|
//
|
|
iport, err := cfg.Section("rpc").Key("port").Int()
|
|
if err != nil {
|
|
logger.Error("读取配置文件失败", zap.Error(err))
|
|
return
|
|
}
|
|
config.G.Port = fmt.Sprintf("%d", iport)
|
|
}
|
|
|
|
// 删除文件
|
|
func DeleteFile(filePath string) error {
|
|
return os.Remove(filePath)
|
|
}
|
|
|
|
// 删除过期文件
|
|
func DeleteExpiredFiles(dirPath string, expiredTime time.Duration) error {
|
|
// 遍历目录下的所有文件
|
|
err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// 检查文件是否过期
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
if time.Since(info.ModTime()) > expiredTime {
|
|
// 删除过期文件
|
|
return DeleteFile(path)
|
|
}
|
|
return nil
|
|
})
|
|
return err
|
|
}
|
|
|