From f5227bcac9820470a549b0121422f7e91c0b480d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Mon, 30 Mar 2026 13:20:13 +0200 Subject: [PATCH] added file checker, completation/size/digest --- app/fileoper/check.go | 88 +++++++++++++++++++++++++++++++ app/fileoper/listfiles.go | 90 +++++++++++++++++--------------- app/handler/file.go | 38 ++++++++++++++ app/service/service.go | 3 ++ app/storage/file.go | 38 ++++++++++++++ cmd/mstorectl/filecmd/filecmd.go | 12 +++++ pkg/filecli/checkfiles.go | 60 +++++++++++++++++++++ pkg/filecli/referer.go | 5 ++ 8 files changed, 291 insertions(+), 43 deletions(-) create mode 100644 app/fileoper/check.go create mode 100644 pkg/filecli/checkfiles.go diff --git a/app/fileoper/check.go b/app/fileoper/check.go new file mode 100644 index 0000000..7d3bcb8 --- /dev/null +++ b/app/fileoper/check.go @@ -0,0 +1,88 @@ +/* + * Copyright 2026 Oleg Borodin + */ +package fileoper + +import ( + "context" + "net/http" + + "mstore/pkg/descr" +) + +// Check files +type CheckFilesParams struct { + Path string + PathType string `param:"pathType"` +} +type CheckFilesResult struct { + Files []descr.File `json:"files,omitempty"` +} + +func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params *CheckFilesParams) (int, *CheckFilesResult, error) { + var code int + res := &CheckFilesResult{} + var err error + + // Check existing and size + files, err := oper.listFiles(ctx, params.PathType, params.Path) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + for _, file := range files { + exists, size, err := oper.store.FileExists(file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + if !exists { + res.Files = append(res.Files, file) + err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + if size != file.Size { + res.Files = append(res.Files, file) + err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + err = oper.store.DeleteFile(file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + } + // Check hashs + files, err = oper.listFiles(ctx, params.PathType, params.Path) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + for _, file := range files { + sum, err := oper.store.GetFileCheksum(file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + if sum != file.Checksum { + res.Files = append(res.Files, file) + err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + err = oper.store.DeleteFile(file.Collection, file.Name) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + } + return code, res, err +} diff --git a/app/fileoper/listfiles.go b/app/fileoper/listfiles.go index 332f82d..c58e1de 100644 --- a/app/fileoper/listfiles.go +++ b/app/fileoper/listfiles.go @@ -26,51 +26,13 @@ type ListFilesResult struct { func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) { var err error res := &ListFilesResult{} - switch params.PathType { - case filecli.PathTypeRegexp: - files, err := oper.listFilesWithRegex(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - case filecli.PathTypePrefix: - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - files, err := oper.listFilesWithPrefix(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - default: - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - files, err := oper.listFilesInCollection(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files + files, err := oper.listFiles(ctx, params.PathType, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err } + res.Files = files code := http.StatusOK return code, res, err } @@ -122,3 +84,45 @@ func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]d } return res, err } + +func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string) ([]descr.File, error) { + res := make([]descr.File, 0) + var err error + switch pathType { + case filecli.PathTypeRegexp: + files, err := oper.listFilesWithRegex(ctx, filepath) + if err != nil { + return res, err + } + res = files + case filecli.PathTypePrefix: + filepath, err = cleanFilepath(filepath) + if err != nil { + return res, err + } + filepath, err = cleanFilepath(filepath) + if err != nil { + return res, err + } + files, err := oper.listFilesWithPrefix(ctx, filepath) + if err != nil { + return res, err + } + res = files + default: + filepath, err = cleanFilepath(filepath) + if err != nil { + return res, err + } + filepath, err = cleanFilepath(filepath) + if err != nil { + return res, err + } + files, err := oper.listFilesInCollection(ctx, filepath) + if err != nil { + return res, err + } + res = files + } + return res, err +} diff --git a/app/handler/file.go b/app/handler/file.go index ae9458d..fd9b463 100644 --- a/app/handler/file.go +++ b/app/handler/file.go @@ -254,6 +254,44 @@ func (hand *Handler) ListFiles(rctx *router.Context) { rctx.SendJSON(code, res.Files) } +func (hand *Handler) CheckFiles(rctx *router.Context) { + + filepath, _ := rctx.GetSubpath("path") + if filepath == "" { + filepath = "/" + } + params := &fileoper.CheckFilesParams{ + Path: filepath, + } + err := rctx.BindQuery(params) + if err != nil { + hand.logg.Errorf("CheckFiles binding error: %v", err) + rctx.SetStatus(http.StatusInternalServerError) + return + } + + // Rigth checking + operatorID, _ := rctx.GetString(userTag) + opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteFiles, params.Path) + if err != nil { + rctx.SetStatus(http.StatusInternalServerError) + return + } + if !opEnable { + rctx.SetStatus(http.StatusMethodNotAllowed) + return + } + // Execution of the operation + ctx := rctx.GetContext() + code, res, err := hand.fiop.CheckFiles(ctx, operatorID, params) + if err != nil { + hand.logg.Errorf("CheckFiles error: %v", err) + rctx.SetStatus(code) + return + } + rctx.SendJSON(code, res.Files) +} + // GetProperty godoc // // @Summary List collections diff --git a/app/service/service.go b/app/service/service.go index aab294d..16d02e0 100644 --- a/app/service/service.go +++ b/app/service/service.go @@ -60,6 +60,9 @@ func (svc *Service) Build() error { svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles) svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles) + svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles) + svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles) + svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections) svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections) diff --git a/app/storage/file.go b/app/storage/file.go index 2b0f613..9beb5cf 100644 --- a/app/storage/file.go +++ b/app/storage/file.go @@ -38,6 +38,24 @@ func (store *Storage) makeFilesubdir(collection, filename string) string { return filepath.Join(store.basepath, fileSubdir) } +func (store *Storage) FileExists(collection, filename string) (bool, int64, error) { + var err error + var exists bool + var size int64 + filename = store.makeFilepath(collection, filename) + stat, err := os.Stat(filename) + if os.IsNotExist(err) { + err = nil + return exists, size, err + } + if err != nil { + return exists, size, err + } + size = stat.Size() + exists = true + return exists, size, err +} + func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) { var err error var res io.ReadCloser @@ -51,6 +69,26 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, return res, err } +func (store *Storage) GetFileCheksum(collection, filename string) (string, error) { + var err error + var res string + + filename = store.makeFilepath(collection, filename) + file, err := os.OpenFile(filename, os.O_RDONLY, 0) + if err != nil { + return res, err + } + defer file.Close() + + hasher := NewHasher() + _, err = io.Copy(hasher.Writer(), file) + if err != nil { + return res, err + } + res = hasher.Hex() + return res, err +} + func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) { var err error var size int64 diff --git a/cmd/mstorectl/filecmd/filecmd.go b/cmd/mstorectl/filecmd/filecmd.go index 16608ea..8b252e6 100644 --- a/cmd/mstorectl/filecmd/filecmd.go +++ b/cmd/mstorectl/filecmd/filecmd.go @@ -21,6 +21,7 @@ type FileUtil struct { exportFilesParams ExportFilesParams deleteCollectionParams DeleteCollectionParams listCollectionsParams ListCollectionsParams + checkFilesParams CheckFilesParams commonFileParams CommonFileParams } @@ -125,6 +126,17 @@ func (util *FileUtil) MakeFileCmds() *cobra.Command { exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.Regexp, "regex", "R", false, "Use path as collection path prefix") subCmd.AddCommand(exportFilesCmd) + // CkeckFiles + var checkFilesCmd = &cobra.Command{ + Use: "check [user:pass@]hostname[:port]/catalog", + Args: cobra.ExactArgs(1), + Short: "Ckeck files in storage", + Run: util.ListFiles, + } + checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Detail, "detail", "D", false, "Show detail file information") + checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Prefix, "prefix", "P", true, "Use path as collection path prefix") + checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Regexp, "regex", "R", false, "Use path as collection path prefix") + subCmd.AddCommand(checkFilesCmd) return subCmd } diff --git a/pkg/filecli/checkfiles.go b/pkg/filecli/checkfiles.go new file mode 100644 index 0000000..2df8706 --- /dev/null +++ b/pkg/filecli/checkfiles.go @@ -0,0 +1,60 @@ +/* + * Copyright 2026 Oleg Borodin + */ +package filecli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strconv" +) + +func (cli *Client) CheckFiles(ctx context.Context, rawpath string) ([]byte, error) { + var err error + var list []byte + + ref, err := ParsePath(rawpath) + if err != nil { + return list, err + } + uri := ref.FilesEP() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return list, err + } + req.Header.Set("User-Agent", cli.userAgent) + req.Header.Set("Accept", "*/*") + resp, err := cli.httpClient.Do(req) + if err != nil { + return list, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return list, err + } + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("Unexpected response code %s", resp.Status) + return list, err + } + contentLength := resp.Header.Get("Content-Length") + if contentLength == "" { + err := fmt.Errorf("Content-Length header is missing") + return list, err + } + blobSize, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return list, err + } + buffer := bytes.NewBuffer(nil) + recSize, err := Copy(ctx, buffer, resp.Body) + if blobSize != recSize { + err := fmt.Errorf("Mismatch declared and actual body size: %d and %d", blobSize, recSize) + return list, err + } + list = buffer.Bytes() + return list, err +} diff --git a/pkg/filecli/referer.go b/pkg/filecli/referer.go index 3a1f2d1..d2dcb4e 100644 --- a/pkg/filecli/referer.go +++ b/pkg/filecli/referer.go @@ -91,6 +91,11 @@ func (ref *Referer) CollectionsEP() string { return curl.String() } +func (ref *Referer) CheckEP() string { + curl := ref.urlobj.JoinPath("/v3/api/checker/", ref.resource) + return curl.String() +} + func (ref *Referer) Userinfo() (string, string) { return ref.user, ref.pass }