ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

working commit

Browse Source
This commit is contained in:
Олег Бородин
2026-02-19 23:51:58 +02:00
parent 7f7e4706d6
commit e06ea41fbb
6 changed files with 186 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/handler/file.go
+13
View File
@@ -177,6 +177,13 @@ func (hand *Handler) ListFiles(rctx *router.Context) {
params := &operator.ListFilesParams{
Filepath: filepath,
}
err := rctx.BindQuery(params)
if err != nil {
hand.logg.Errorf("ListFiles binding error: %v", err)
rctx.SetStatus(http.StatusInternalServerError)
return
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightReadFiles, "")
@@ -208,6 +215,12 @@ func (hand *Handler) ListCollections(rctx *router.Context) {
params := &operator.ListCollectionsParams{
Path: cpath,
}
err := rctx.BindQuery(params)
if err != nil {
hand.logg.Errorf("ListCollections binding error: %v", err)
rctx.SetStatus(http.StatusInternalServerError)
return
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightReadFiles, "")
app/operator/file.go
+90 -9
View File
@@ -16,6 +16,7 @@ import (
"net/http"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
@@ -287,7 +288,7 @@ func (oper *Operator) DeleteFile(ctx context.Context, operID string, param *Dele
// ListFiles
type ListFilesParams struct {
Filepath string
PathAs terms.PathAs `param:"pathAs"`
PathAs terms.PathUsage `param:"pathAs"`
}
type ListFilesResult struct {
Files []descr.File `json:"files,omitempty"`
@@ -298,13 +299,20 @@ func (oper *Operator) ListFiles(ctx context.Context, operID string, params *List
res := &ListFilesResult{
Files: make([]descr.File, 0),
}
//oper.logg.Debugf("FileList path: %s %s", params.Filepath, params.PathAs)
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
switch params.PathAs {
case terms.AsRegexp:
files, err := oper.listFilesWithRegex(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
case terms.AsPrefix:
files, err := oper.listFilesWithPrefix(ctx, params.Filepath)
if err != nil {
@@ -312,7 +320,6 @@ func (oper *Operator) ListFiles(ctx context.Context, operID string, params *List
return code, res, err
}
res.Files = files
default: // Fine
files, err := oper.listFilesInOneCollection(ctx, params.Filepath)
if err != nil {
@@ -353,10 +360,31 @@ func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([
return res, err
}
func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
re, err := regexp.Compile(regex)
if err != nil {
return res, err
}
files, err := oper.mdb.ListAllFiles(ctx)
if err != nil {
return res, err
}
for _, file := range files {
fullpath := filepath.Join(file.Collection, file.Name)
if re.MatchString(fullpath) {
res = append(res, file)
}
}
return res, err
}
// ListCollections
type ListCollectionsParams struct {
Path string
PathAS terms.PathAs `param:"pathAs"`
PathAS terms.PathUsage `param:"pathAs"`
}
type ListCollectionsResult struct {
Collections []string `json:"collection,omitempty"`
@@ -374,6 +402,12 @@ func (oper *Operator) ListCollections(ctx context.Context, operID string, param
}
collectionList := make([]string, 0)
switch param.PathAS {
case terms.AsRegexp:
collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
case terms.AsPrefix:
collectionList, err = oper.listCollectionsWithPrefix(ctx, param.Path)
if err != nil {
@@ -408,7 +442,7 @@ func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix stri
collMap[item.Collection] = true
}
}
res = make([]string, len(collMap))
res = make([]string, 0)
for key, _ := range collMap {
if strings.HasPrefix(key, prefix) {
res = append(res, key)
@@ -418,6 +452,36 @@ func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix stri
return res, err
}
func (oper *Operator) listCollectionsWithRegexp(ctx context.Context, regex string) ([]string, error) {
var err error
res := make([]string, 0)
re, err := regexp.Compile(regex)
if err != nil {
return res, err
}
fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO
if err != nil {
return res, err
}
collMap := make(map[string]bool)
for _, item := range fileDescrs {
_, exists := collMap[item.Collection]
if !exists {
collMap[item.Collection] = true
}
}
res = make([]string, 0)
for key, _ := range collMap {
if re.MatchString(key) {
res = append(res, key)
}
}
slices.Sort(res)
return res, err
}
func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) {
var err error
res := make([]string, 0)
@@ -433,7 +497,7 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error)
collMap[item.Collection] = true
}
}
res = make([]string, len(collMap))
res = make([]string, 0)
for key, _ := range collMap {
res = append(res, key)
}
@@ -444,8 +508,8 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error)
// DeleteColletion
type DeleteColletionParams struct {
Path string
PathAs string `param:"pathAs"`
DryRun bool `param:"dryRun"`
PathAs terms.PathUsage `param:"pathAs"`
DryRun bool `param:"dryRun"`
}
type DeleteColletionResult struct {
Files []descr.File `json:"files,omitempty"`
@@ -462,7 +526,24 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operID string, param
return code, res, err
}
//oper.logg.Debugf("DeleteCollection: Use path as %s", param.PathAs)
switch terms.PathAs(param.PathAs) {
switch param.PathAs {
case terms.AsRegexp:
collections, err := oper.listCollectionsWithRegexp(ctx, param.Path)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
allfiles := make([]descr.File, 0)
for _, collection := range collections {
files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
allfiles = append(allfiles, files...)
}
res.Files = allfiles
case terms.AsPrefix:
collections, err := oper.listCollectionsWithPrefix(ctx, param.Path)
if err != nil {
app/router/pathc.go
+1 -1
View File
@@ -65,7 +65,7 @@ func pathCompiler(path string) (string, error) {
}
const (
defaultRegexp = `[a-zA-Z0-9_\.][/\-\.a-zA-Z0-9_%%=:~]+`
defaultRegexp = `[a-zA-Z0-9_\.][/\-\.,a-zA-Z0-9_%=*\[\]:~]+`
)
func convertRegexp(src []byte) []byte {
cmd/mstorectl/filecmd.go
+50 -12
View File
@@ -88,6 +88,8 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
Run: util.ListFiles,
}
listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information")
listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix")
listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix")
subCmd.AddCommand(listFilesCmd)
// ImportFiles
@@ -123,6 +125,7 @@ func (util *FileUtil) CreateCollectionCmds() *cobra.Command {
Short: "List collections in storage",
Run: util.ListCollections,
}
listCollectionsCmd.Flags().BoolVarP(&util.listCollectionsParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix")
subCmd.AddCommand(listCollectionsCmd)
// DeleteCollection
@@ -134,6 +137,7 @@ func (util *FileUtil) CreateCollectionCmds() *cobra.Command {
}
deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.Detail, "detail", "D", false, "Show detail file information")
deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsPrefix, "asprefix", "P", false, "Use path as collection path prefix")
deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix")
deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.DryRun, "dryrun", "Y", false, "Simulate process, don't delete files")
subCmd.AddCommand(deleteCollectionCmd)
@@ -287,6 +291,8 @@ func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFilePar
type ListFilesParams struct {
Filepath string
Detail bool
AsPrefix bool
AsRegexp bool
}
type ListFilesResult struct {
@@ -306,9 +312,21 @@ func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParam
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathAs terms.PathUsage
switch {
case params.AsRegexp:
pathAs = terms.AsRegexp
case params.AsPrefix:
pathAs = terms.AsPrefix
default:
pathAs = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
files, err := client.NewClient().ListFiles(ctx, params.Filepath)
files, err := client.NewClient().ListFiles(ctx, params.Filepath, pathAs)
if err != nil {
return res, err
}
@@ -316,10 +334,7 @@ func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParam
if params.Detail {
res.Files = files
} else {
for _, file := range files {
filename := filepath.Join("/", file.Collection, file.Name)
res.Filenames = append(res.Filenames, filename)
}
res.Filenames = makeFilelistFromFiles(files)
}
return res, err
}
@@ -398,7 +413,9 @@ func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesP
// ListCollections
type ListCollectionsParams struct {
Path string
Path string
AsPrefix bool
AsRegexp bool
}
type ListCollectionsResult struct {
@@ -410,6 +427,7 @@ func (util *FileUtil) ListCollections(cmd *cobra.Command, args []string) {
res, err := util.listCollections(&util.commonFileParams, &util.listCollectionsParams)
printResponse(res, err)
}
func (util *FileUtil) listCollections(common *CommonFileParams, params *ListCollectionsParams) (*ListCollectionsResult, error) {
var err error
res := &ListCollectionsResult{
@@ -419,9 +437,21 @@ func (util *FileUtil) listCollections(common *CommonFileParams, params *ListColl
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathAs terms.PathUsage
switch {
case params.AsRegexp:
pathAs = terms.AsRegexp
case params.AsPrefix:
pathAs = terms.AsPrefix
default:
pathAs = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
collecions, err := client.NewClient().ListCollections(ctx, params.Path)
collecions, err := client.NewClient().ListCollections(ctx, params.Path, pathAs)
if err != nil {
return res, err
}
@@ -434,6 +464,7 @@ type DeleteCollectionParams struct {
Path string
Detail bool
AsPrefix bool
AsRegexp bool
DryRun bool
}
@@ -447,6 +478,7 @@ func (util *FileUtil) DeleteCollection(cmd *cobra.Command, args []string) {
res, err := util.deleteCollection(&util.commonFileParams, &util.deleteCollectionParams)
printResponse(res, err)
}
func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteCollectionParams) (*DeleteCollectionResult, error) {
var err error
res := &DeleteCollectionResult{
@@ -458,7 +490,7 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
var pathAs terms.PathAs
var pathAs terms.PathUsage
switch {
case params.AsPrefix:
pathAs = terms.AsPrefix
@@ -472,10 +504,16 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC
if params.Detail {
res.Files = files
} else {
for _, file := range files {
res.Filenames = append(res.Filenames, filepath.Join(file.Collection, file.Name))
}
slices.Sort(res.Filenames)
res.Filenames = makeFilelistFromFiles(files)
}
return res, err
}
func makeFilelistFromFiles(files []descr.File) []string {
res := make([]string, 0, len(files))
for _, file := range files {
res = append(res, filepath.Join(file.Collection, file.Name))
}
slices.Sort(res)
return res
}
pkg/client/file.go
+24 -3
View File
@@ -209,7 +209,7 @@ func (cli *Client) DeleteFile(ctx context.Context, fileuri string) error {
return err
}
func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.File, error) {
func (cli *Client) ListFiles(ctx context.Context, catalogURI string, usePathAs terms.PathUsage) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
@@ -221,6 +221,16 @@ func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.Fi
if err != nil {
return res, err
}
// Add values
values := url.Values{}
if usePathAs != "" {
values.Add("pathAs", string(usePathAs))
}
encodedValues := values.Encode()
if encodedValues != "" {
catalogURI = catalogURI + "?" + encodedValues
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil)
if err != nil {
return res, err
@@ -267,7 +277,7 @@ func (cli *Client) ListFiles(ctx context.Context, catalogURI string) ([]descr.Fi
return res, err
}
func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]string, error) {
func (cli *Client) ListCollections(ctx context.Context, catalogURI string, usePathAs terms.PathUsage) ([]string, error) {
var err error
res := make([]string, 0)
@@ -279,6 +289,17 @@ func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]st
if err != nil {
return res, err
}
// Add values
values := url.Values{}
if usePathAs != "" {
values.Add("pathAs", string(usePathAs))
}
encodedValues := values.Encode()
if encodedValues != "" {
catalogURI = catalogURI + "?" + encodedValues
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil)
if err != nil {
return res, err
@@ -325,7 +346,7 @@ func (cli *Client) ListCollections(ctx context.Context, catalogURI string) ([]st
return res, err
}
func (cli *Client) DeleteCollection(ctx context.Context, catalogURI string, usePathAs terms.PathAs, dryRun bool) ([]descr.File, error) {
func (cli *Client) DeleteCollection(ctx context.Context, catalogURI string, usePathAs terms.PathUsage, dryRun bool) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
pkg/terms/terms.go
+8 -10
View File
@@ -10,12 +10,12 @@
package terms
type PathAs string
type PathUsage string
const (
AsFinePath PathAs = "asFinePath"
AsPrefix PathAs = "asPrefix"
AsRegexp PathAs = "asRegexp"
AsFinePath PathUsage = "asFinePath"
AsPrefix PathUsage = "asPrefix"
AsRegexp PathUsage = "asRegexp"
)
const (
@@ -32,10 +32,8 @@ const (
// Accounts, grants
RightReadAccounts = "readAccounts" // GetAccount, ListAccounts
RightWriteAccounts = "writeAccounts" // CreateAccount, UpdateAccount, DeleteAccount
// Files
RightWriteFiles = "writeFiles" // FileInfo, GetFile, ListFiles
RightReadFiles = "readFiles" // PutFile, DeleteFile
// Images: manifests, layers
RightReadImages = "readImages" // ManifestInfo, GetManifest, BlobInfo, GetBlob
RightWriteImages = "writeImages" // other opearion
RightWriteFiles = "writeFiles" // FileInfo, GetFile, ListFiles
RightReadFiles = "readFiles" // PutFile, DeleteFile
RightReadImages = "readImages" // ManifestInfo, GetManifest, BlobInfo, GetBlob
RightWriteImages = "writeImages" // other opearion
)