From e06ea41fbb2e8bb146d0181eac4d72d4952bdf15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Thu, 19 Feb 2026 23:51:58 +0200 Subject: [PATCH] working commit --- app/handler/file.go | 13 ++++++ app/operator/file.go | 99 ++++++++++++++++++++++++++++++++++++---- app/router/pathc.go | 2 +- cmd/mstorectl/filecmd.go | 62 ++++++++++++++++++++----- pkg/client/file.go | 27 +++++++++-- pkg/terms/terms.go | 18 ++++---- 6 files changed, 186 insertions(+), 35 deletions(-) diff --git a/app/handler/file.go b/app/handler/file.go index be2c24a..8444ae5 100644 --- a/app/handler/file.go +++ b/app/handler/file.go @@ -177,6 +177,13 @@ func (hand *Handler) ListFiles(rctx *router.Context) { params := &operator.ListFilesParams{ Filepath: filepath, } + err := rctx.BindQuery(params) + if err != nil { + hand.logg.Errorf("ListFiles binding error: %v", err) + rctx.SetStatus(http.StatusInternalServerError) + return + } + // Rigth checking operatorID, _ := rctx.GetString(userTag) opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightReadFiles, "") @@ -208,6 +215,12 @@ func (hand *Handler) ListCollections(rctx *router.Context) { params := &operator.ListCollectionsParams{ Path: cpath, } + err := rctx.BindQuery(params) + if err != nil { + hand.logg.Errorf("ListCollections binding error: %v", err) + rctx.SetStatus(http.StatusInternalServerError) + return + } // Rigth checking operatorID, _ := rctx.GetString(userTag) opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightReadFiles, "") diff --git a/app/operator/file.go b/app/operator/file.go index fc3a459..469bd7e 100644 --- a/app/operator/file.go +++ b/app/operator/file.go @@ -16,6 +16,7 @@ import ( "net/http" "path" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -287,7 +288,7 @@ func (oper *Operator) DeleteFile(ctx context.Context, operID string, param *Dele // ListFiles type ListFilesParams struct { Filepath string - PathAs terms.PathAs `param:"pathAs"` + PathAs terms.PathUsage `param:"pathAs"` } type ListFilesResult struct { Files []descr.File `json:"files,omitempty"` @@ -298,13 +299,20 @@ func (oper *Operator) ListFiles(ctx context.Context, operID string, params *List res := &ListFilesResult{ Files: make([]descr.File, 0), } - + //oper.logg.Debugf("FileList path: %s %s", params.Filepath, params.PathAs) params.Filepath, err = cleanFilepath(params.Filepath) if err != nil { code := http.StatusInternalServerError return code, res, err } switch params.PathAs { + case terms.AsRegexp: + files, err := oper.listFilesWithRegex(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files case terms.AsPrefix: files, err := oper.listFilesWithPrefix(ctx, params.Filepath) if err != nil { @@ -312,7 +320,6 @@ func (oper *Operator) ListFiles(ctx context.Context, operID string, params *List return code, res, err } res.Files = files - default: // Fine files, err := oper.listFilesInOneCollection(ctx, params.Filepath) if err != nil { @@ -353,10 +360,31 @@ func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([ return res, err } +func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + + re, err := regexp.Compile(regex) + if err != nil { + return res, err + } + files, err := oper.mdb.ListAllFiles(ctx) + if err != nil { + return res, err + } + for _, file := range files { + fullpath := filepath.Join(file.Collection, file.Name) + if re.MatchString(fullpath) { + res = append(res, file) + } + } + return res, err +} + // ListCollections type ListCollectionsParams struct { Path string - PathAS terms.PathAs `param:"pathAs"` + PathAS terms.PathUsage `param:"pathAs"` } type ListCollectionsResult struct { Collections []string `json:"collection,omitempty"` @@ -374,6 +402,12 @@ func (oper *Operator) ListCollections(ctx context.Context, operID string, param } collectionList := make([]string, 0) switch param.PathAS { + case terms.AsRegexp: + collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } case terms.AsPrefix: collectionList, err = oper.listCollectionsWithPrefix(ctx, param.Path) if err != nil { @@ -408,7 +442,7 @@ func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix stri collMap[item.Collection] = true } } - res = make([]string, len(collMap)) + res = make([]string, 0) for key, _ := range collMap { if strings.HasPrefix(key, prefix) { res = append(res, key) @@ -418,6 +452,36 @@ func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix stri return res, err } +func (oper *Operator) listCollectionsWithRegexp(ctx context.Context, regex string) ([]string, error) { + var err error + res := make([]string, 0) + + re, err := regexp.Compile(regex) + if err != nil { + return res, err + } + + fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO + if err != nil { + return res, err + } + collMap := make(map[string]bool) + for _, item := range fileDescrs { + _, exists := collMap[item.Collection] + if !exists { + collMap[item.Collection] = true + } + } + res = make([]string, 0) + for key, _ := range collMap { + if re.MatchString(key) { + res = append(res, key) + } + } + slices.Sort(res) + return res, err +} + func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) { var err error res := make([]string, 0) @@ -433,7 +497,7 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) collMap[item.Collection] = true } } - res = make([]string, len(collMap)) + res = make([]string, 0) for key, _ := range collMap { res = append(res, key) } @@ -444,8 +508,8 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) // DeleteColletion type DeleteColletionParams struct { Path string - PathAs string `param:"pathAs"` - DryRun bool `param:"dryRun"` + PathAs terms.PathUsage `param:"pathAs"` + DryRun bool `param:"dryRun"` } type DeleteColletionResult struct { Files []descr.File `json:"files,omitempty"` @@ -462,7 +526,24 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operID string, param return code, res, err } //oper.logg.Debugf("DeleteCollection: Use path as %s", param.PathAs) - switch terms.PathAs(param.PathAs) { + switch param.PathAs { + case terms.AsRegexp: + collections, err := oper.listCollectionsWithRegexp(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles := make([]descr.File, 0) + for _, collection := range collections { + files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles = append(allfiles, files...) + } + res.Files = allfiles + case terms.AsPrefix: collections, err := oper.listCollectionsWithPrefix(ctx, param.Path) if err != nil { diff --git a/app/router/pathc.go b/app/router/pathc.go index b1ca57a..7ddebba 100644 --- a/app/router/pathc.go +++ b/app/router/pathc.go @@ -65,7 +65,7 @@ func pathCompiler(path string) (string, error) { } const ( - defaultRegexp = `[a-zA-Z0-9_\.][/\-\.a-zA-Z0-9_%%=:~]+` + defaultRegexp = `[a-zA-Z0-9_\.][/\-\.,a-zA-Z0-9_%=*\[\]:~]+` ) func convertRegexp(src []byte) []byte { diff --git a/cmd/mstorectl/filecmd.go b/cmd/mstorectl/filecmd.go index 18b45de..72b41b5 100644 --- a/cmd/mstorectl/filecmd.go +++ b/cmd/mstorectl/filecmd.go @@ -88,6 +88,8 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command { Run: util.ListFiles, } listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information") + listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") + listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") subCmd.AddCommand(listFilesCmd) // ImportFiles @@ -123,6 +125,7 @@ func (util *FileUtil) CreateCollectionCmds() *cobra.Command { Short: "List collections in storage", Run: util.ListCollections, } + listCollectionsCmd.Flags().BoolVarP(&util.listCollectionsParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") subCmd.AddCommand(listCollectionsCmd) // DeleteCollection @@ -134,6 +137,7 @@ func (util *FileUtil) CreateCollectionCmds() *cobra.Command { } deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.Detail, "detail", "D", false, "Show detail file information") deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsPrefix, "asprefix", "P", false, "Use path as collection path prefix") + deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.DryRun, "dryrun", "Y", false, "Simulate process, don't delete files") subCmd.AddCommand(deleteCollectionCmd) @@ -287,6 +291,8 @@ func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFilePar type ListFilesParams struct { Filepath string Detail bool + AsPrefix bool + AsRegexp bool } type ListFilesResult struct { @@ -306,9 +312,21 @@ func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParam if err != nil { return res, err } + if params.AsRegexp { + params.AsPrefix = false + } + var pathAs terms.PathUsage + switch { + case params.AsRegexp: + pathAs = terms.AsRegexp + case params.AsPrefix: + pathAs = terms.AsPrefix + default: + pathAs = terms.AsFinePath + } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - files, err := client.NewClient().ListFiles(ctx, params.Filepath) + files, err := client.NewClient().ListFiles(ctx, params.Filepath, pathAs) if err != nil { return res, err } @@ -316,10 +334,7 @@ func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParam if params.Detail { res.Files = files } else { - for _, file := range files { - filename := filepath.Join("/", file.Collection, file.Name) - res.Filenames = append(res.Filenames, filename) - } + res.Filenames = makeFilelistFromFiles(files) } return res, err } @@ -398,7 +413,9 @@ func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesP // ListCollections type ListCollectionsParams struct { - Path string + Path string + AsPrefix bool + AsRegexp bool } type ListCollectionsResult struct { @@ -410,6 +427,7 @@ func (util *FileUtil) ListCollections(cmd *cobra.Command, args []string) { res, err := util.listCollections(&util.commonFileParams, &util.listCollectionsParams) printResponse(res, err) } + func (util *FileUtil) listCollections(common *CommonFileParams, params *ListCollectionsParams) (*ListCollectionsResult, error) { var err error res := &ListCollectionsResult{ @@ -419,9 +437,21 @@ func (util *FileUtil) listCollections(common *CommonFileParams, params *ListColl if err != nil { return res, err } + if params.AsRegexp { + params.AsPrefix = false + } + var pathAs terms.PathUsage + switch { + case params.AsRegexp: + pathAs = terms.AsRegexp + case params.AsPrefix: + pathAs = terms.AsPrefix + default: + pathAs = terms.AsFinePath + } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - collecions, err := client.NewClient().ListCollections(ctx, params.Path) + collecions, err := client.NewClient().ListCollections(ctx, params.Path, pathAs) if err != nil { return res, err } @@ -434,6 +464,7 @@ type DeleteCollectionParams struct { Path string Detail bool AsPrefix bool + AsRegexp bool DryRun bool } @@ -447,6 +478,7 @@ func (util *FileUtil) DeleteCollection(cmd *cobra.Command, args []string) { res, err := util.deleteCollection(&util.commonFileParams, &util.deleteCollectionParams) printResponse(res, err) } + func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteCollectionParams) (*DeleteCollectionResult, error) { var err error res := &DeleteCollectionResult{ @@ -458,7 +490,7 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - var pathAs terms.PathAs + var pathAs terms.PathUsage switch { case params.AsPrefix: pathAs = terms.AsPrefix @@ -472,10 +504,16 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC if params.Detail { res.Files = files } else { - for _, file := range files { - res.Filenames = append(res.Filenames, filepath.Join(file.Collection, file.Name)) - } - slices.Sort(res.Filenames) + res.Filenames = makeFilelistFromFiles(files) } return res, err } + +func makeFilelistFromFiles(files []descr.File) []string { + res := make([]string, 0, len(files)) + for _, file := range files { + res = append(res, filepath.Join(file.Collection, file.Name)) + } + slices.Sort(res) + return res +} diff --git a/pkg/client/file.go b/pkg/client/file.go index fb6d5fc..850d4f3 100644 --- a/pkg/client/file.go +++ b/pkg/client/file.go @@ -209,7 +209,7 @@ func (cli *Client) DeleteFile(ctx context.Context, fileuri string) error { return err } -func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.File, error) { +func (cli *Client) ListFiles(ctx context.Context, catalogURI string, usePathAs terms.PathUsage) ([]descr.File, error) { var err error res := make([]descr.File, 0) @@ -221,6 +221,16 @@ func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.Fi if err != nil { return res, err } + // Add values + values := url.Values{} + if usePathAs != "" { + values.Add("pathAs", string(usePathAs)) + } + encodedValues := values.Encode() + if encodedValues != "" { + catalogURI = catalogURI + "?" + encodedValues + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil) if err != nil { return res, err @@ -267,7 +277,7 @@ func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.Fi return res, err } -func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]string, error) { +func (cli *Client) ListCollections(ctx context.Context, catalogURI string, usePathAs terms.PathUsage) ([]string, error) { var err error res := make([]string, 0) @@ -279,6 +289,17 @@ func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]st if err != nil { return res, err } + + // Add values + values := url.Values{} + if usePathAs != "" { + values.Add("pathAs", string(usePathAs)) + } + encodedValues := values.Encode() + if encodedValues != "" { + catalogURI = catalogURI + "?" + encodedValues + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil) if err != nil { return res, err @@ -325,7 +346,7 @@ func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]st return res, err } -func (cli *Client) DeleteCollection(ctx context.Context, catalogURI string, usePathAs terms.PathAs, dryRun bool) ([]descr.File, error) { +func (cli *Client) DeleteCollection(ctx context.Context, catalogURI string, usePathAs terms.PathUsage, dryRun bool) ([]descr.File, error) { var err error res := make([]descr.File, 0) diff --git a/pkg/terms/terms.go b/pkg/terms/terms.go index 29ec444..2673be3 100644 --- a/pkg/terms/terms.go +++ b/pkg/terms/terms.go @@ -10,12 +10,12 @@ package terms -type PathAs string +type PathUsage string const ( - AsFinePath PathAs = "asFinePath" - AsPrefix PathAs = "asPrefix" - AsRegexp PathAs = "asRegexp" + AsFinePath PathUsage = "asFinePath" + AsPrefix PathUsage = "asPrefix" + AsRegexp PathUsage = "asRegexp" ) const ( @@ -32,10 +32,8 @@ const ( // Accounts, grants RightReadAccounts = "readAccounts" // GetAccount, ListAccounts RightWriteAccounts = "writeAccounts" // CreateAccount, UpdateAccount, DeleteAccount - // Files - RightWriteFiles = "writeFiles" // FileInfo, GetFile, ListFiles - RightReadFiles = "readFiles" // PutFile, DeleteFile - // Images: manifests, layers - RightReadImages = "readImages" // ManifestInfo, GetManifest, BlobInfo, GetBlob - RightWriteImages = "writeImages" // other opearion + RightWriteFiles = "writeFiles" // FileInfo, GetFile, ListFiles + RightReadFiles = "readFiles" // PutFile, DeleteFile + RightReadImages = "readImages" // ManifestInfo, GetManifest, BlobInfo, GetBlob + RightWriteImages = "writeImages" // other opearion )