From e526f9c9311bdf92aced6379aebf79536fe1230a Mon Sep 17 00:00:00 2001 From: xc Date: Tue, 2 Sep 2025 09:51:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=87=E4=BB=B6=E5=B7=A5?= =?UTF-8?q?=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scagent/go.mod | 6 +- scagent/go.sum | 20 ++- scagent/util/fsutil.go | 391 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 scagent/util/fsutil.go diff --git a/scagent/go.mod b/scagent/go.mod index 991b575..ca82daf 100644 --- a/scagent/go.mod +++ b/scagent/go.mod @@ -4,12 +4,16 @@ go 1.22.1 require ( github.com/fsnotify/fsnotify v1.9.0 + github.com/schollz/progressbar/v3 v3.18.0 go.uber.org/zap v1.27.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect ) diff --git a/scagent/go.sum b/scagent/go.sum index 7a6ca35..ddf180e 100644 --- a/scagent/go.sum +++ b/scagent/go.sum @@ -1,19 +1,31 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/scagent/util/fsutil.go b/scagent/util/fsutil.go new file mode 100644 index 0000000..c1b5c3d --- /dev/null +++ b/scagent/util/fsutil.go @@ -0,0 +1,391 @@ +package util + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path" + "path/filepath" + "scagnet/config" + "strings" + + "github.com/schollz/progressbar/v3" +) + +// 遍历目录下的文件 +// 是否只扫相对路径下 +func GetDirFilePaths(dirPath string, relativeOnly bool) ([]string, error) { + var filePaths []string + err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + if relativeOnly { + fileName := filepath.Base(path) + relativePath := filepath.ToSlash(filepath.Join(filepath.Base(dirPath), fileName)) + filePaths = append(filePaths, relativePath) + } else { + filePaths = append(filePaths, filepath.ToSlash(path)) + } + } + return nil + }) + if err != nil { + return nil, err + } + return filePaths, nil +} + +// base64 encode +// url safe +func Bas64end(str string) string { + // bdata:= + return base64.URLEncoding.EncodeToString([]byte(str)) +} + +// base64 url safe uneconde +func Base64dec(bsstr string) string { + dedc, _ := base64.URLEncoding.DecodeString(bsstr) + return string(dedc) +} + +// 计算文件的hash +func CalacHash(rfile string) string { + // 获取到真实地址 + // + file, err := os.Open(rfile) + if err != nil { + panic(err) + } + defer file.Close() + + // initlize hash object + hash := sha256.New() + + // write file into hash object + if _, err := io.Copy(hash, file); err != nil { + panic(err) + } + // get hash value + hashBytes := hash.Sum(nil) + //converto to hash string + hastString := fmt.Sprintf("%x", hashBytes) + return hastString +} + +// 判断文件是否存在 +func IsFileExist(filename string) bool { + if _, err := os.Stat(filename); err == nil { + return true + } + return false +} + +// 发送文件 +func SendFiles(filePath string, url string) error { + filePath = filepath.ToSlash(filePath) + fileInfo, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("file [%s] not exist", filePath) + } + return fmt.Errorf("file [%s] error: %w", filePath, err) + } + + if fileInfo.IsDir() { + return postDirectory(filePath, url) + } + + return postFile(filePath, path.Base(filePath), url) +} + +// 传送文件夹 +func postDirectory(dirPath string, url string) error { + files, err := GetDirFilePaths(dirPath, false) + if err != nil { + return err + } + + fmt.Println("\nAll files in folder:") + for _, file := range files { + fmt.Println(file) + } + + var confirm string + fmt.Print("\nTransfer all files? [Y/N] ") + fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + fmt.Print("\nCancel send all files ") + return nil + } + + for _, file := range files { + fileName, _ := filepath.Rel(dirPath, file) + fileName = filepath.Join(filepath.Base(dirPath), fileName) + err := postFile(file, fileName, url) + if err != nil { + return err + } + } + fmt.Printf("Send folder %s success.\n", dirPath) + return nil +} + +// 传送文件 +func postFile(wfpath string, filename string, url string) error { + payload := &bytes.Buffer{} + writer := multipart.NewWriter(payload) + + file, err := os.Open(wfpath) + if err != nil { + return err + } + defer file.Close() + + part, err := writer.CreateFormFile("file", filepath.ToSlash(filename)) + if err != nil { + return err + } + + fileInfo, _ := file.Stat() + bar := progressbar.DefaultBytes( + fileInfo.Size(), + fmt.Sprintf("Uploading [%s]", filename), + ) + + _, err = io.Copy(io.MultiWriter(part, bar), file) + if err != nil { + return err + } + + err = writer.Close() + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, url, payload) + if err != nil { + return err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + // 在头里面加上路径,去除前缀 + rlpath := strings.TrimPrefix(wfpath, config.G.FilePath) + // fmt.Printf("wfpath:%s,rel path is:%s", wfpath, rlpath) + req.Header.Set("Fpath", Bas64end(rlpath)) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("upload failed with status code: %d", resp.StatusCode) + } + + return nil +} + +/* +* add file to zip + */ +func ZipFiles(zipname string, files []string, zippath string) error { + fmt.Printf("zipname:%s\n", zipname) + // 创建zip文件 + newZipFile, err := os.Create(zipname) + if err != nil { + return err + } + + defer newZipFile.Close() + // zip写头 + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + + // 把files添加到zip中 + for _, file := range files { + // + fmt.Printf("zipfiles in function :%s\n", file) + + // + zipfile, err := os.Open(file) + if err != nil { + return err + } + defer zipfile.Close() + + // + info, err := zipfile.Stat() + if err != nil { + return err + } + // + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + // 在zip存档中设置文件的相对路径 + header.Name, err = filepath.Rel(filepath.Dir(config.G.FilePath), file) + if err != nil { + return err + } + // 转换为Unix风格的路径分隔符 + header.Name = filepath.ToSlash(header.Name) + // 目录需要拼上一个 "/" ，否则会出现一个和目录一样的文件在压缩包中 + if info.IsDir() { + header.Name += "/" + } else { + // 将压缩方法设置为deflate + header.Method = zip.Deflate + } + // + fmt.Printf("header.name is %s\n", header.Name) + + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + + if _, err = io.Copy(writer, zipfile); err != nil { + return err + } + } + return nil +} + +// 解压缩zip文件 +// 这个方法必须是首字母大写的，才能被注册 +// src: 需要解压的zip文件 +func DecompressZip(zpFname string) error { + // 下载文件 + archive, err := zip.OpenReader(zpFname) + if err != nil { + return err + } + // 取zip文件的绝对路径的文件夹。会默认解压到files下，暂停使用 + // dir := filepath.Dir(zpFname) + defer archive.Close() + // 遍历目录 + for _, f := range archive.File { + // zip 解压的时候，路径为监听的路径 + filePath := filepath.Join(config.G.FilePath, f.Name) + // 暂停原因 屏蔽解压到files 目录下 + //filePath := filepath.Join(dir, f.Name) + if f.FileInfo().IsDir() { + os.MkdirAll(filePath, os.ModePerm) + continue + } + // 父文件夹开始创建目录 + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return fmt.Errorf("failed to make directory (%v)", err) + } + // 文件原有模式 + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return fmt.Errorf("failed to create file (%v)", err) + } + fileInArchive, err := f.Open() + if err != nil { + return fmt.Errorf("failed to open file in zip (%v)", err) + } + if _, err := io.Copy(dstFile, fileInArchive); err != nil { + return fmt.Errorf("failed to copy file in zip (%v)", err) + } + dstFile.Close() + fileInArchive.Close() + } + return nil +} + +// dest：目的地址以及压缩文件名 eg:/data/logZip/2022-12-12-log.zip +// paths：需要压缩的所有文件所组成的集合 +func CompressToZip(dest string, currentPath string, paths []string) error { + // fmt.Println("real path", currentPath) + // 打开files 目录 + filesPath := "" + if dir, err := os.Getwd(); err == nil { + filesPath = dir + "/sync_zips/" + } + if err := os.MkdirAll(filesPath, os.ModePerm); err != nil { + fmt.Println(err.Error()) + } + // ToSlash 过滤windows的斜杠引起的bug + zfile, err := os.Create(path.Join("./sync_zips", "/", dest)) + if err != nil { + return err + } + defer zfile.Close() + zipWriter := zip.NewWriter(zfile) + defer zipWriter.Close() + + // 遍历带压缩的目录信息 + for _, src := range paths { + // 替换文件名，并且去除前后 "\" 或 "/" + // path = strings.Trim(path, string(filepath.Separator)) + // 从配置中读取到源目录 + src = path.Join(currentPath, "/", src) + // 删除尾随路径（如果它是目录） + src := strings.TrimSuffix(src, string(os.PathSeparator)) + // 文件遍历 + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // 创建本地文件头 + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + + // 监听的工作目录作为文件的结尾，然后取相对路径 + // 在zip存档中设置文件的相对路径 + header.Name, err = filepath.Rel(filepath.Dir(config.G.FilePath), path) + if err != nil { + return err + } + + // 转换为Unix风格的路径分隔符 + header.Name = filepath.ToSlash(header.Name) + + // 目录需要拼上一个 "/" ，否则会出现一个和目录一样的文件在压缩包中 + if info.IsDir() { + header.Name += "/" + } else { + // 将压缩方法设置为deflate + header.Method = zip.Deflate + } + // fmt.Printf("zip header:%v\n", header.Name) + // 创建写入头的writer + headerWriter, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(headerWriter, f) + return err + }) + + if err != nil { + return err + } + } + // 存到指定位置 + os.Rename(dest, path.Join("./sync_zips/", dest)) + return nil +}