/* * Copyright 2026 Oleg Borodin * * This work is published and licensed under a Creative Commons * Attribution-NonCommercial-NoDerivatives 4.0 International License. * * Distribution of this work is permitted, but commercial use and * modifications are strictly prohibited. */ package command import ( "context" "encoding/json" "errors" "fmt" "io/fs" "net/url" "os" "path" "path/filepath" "strings" "time" "github.com/spf13/cobra" "github.com/spf13/viper" "mstore/pkg/filecli" "mstore/pkg/descr" ) func (util *FileUtil) MakeFileCmds() *cobra.Command { var subCmd = &cobra.Command{ Use: "files", Short: "File operations", Aliases: []string{"file"}, } const defaultTimeout uint64 = 10 subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username") subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password") subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout") subCmd.PersistentFlags().BoolVarP(&util.commonFileParams.SkipTLSVerify, "skipVerify", "S", true, "Skip server certificate verify") subCmd.MarkPersistentFlagRequired("host") subCmd.MarkFlagsRequiredTogether("user", "pass") vi := viper.New() vi.SetEnvPrefix("mstore") vi.BindEnv("user") vi.BindEnv("pass") util.commonFileParams.Username = vi.GetString("user") util.commonFileParams.Password = vi.GetString("pass") // PutFile var putFileCmd = &cobra.Command{ Use: "put filepath [user:pass@]hostname[:port]/collection/name", Args: cobra.ExactArgs(2), Aliases: []string{"push"}, Short: "Put file to storage", Run: util.PutFile, } subCmd.AddCommand(putFileCmd) // GetFile var getFileCmd = &cobra.Command{ Use: "get [user:pass@]hostname[:port]/collection/name filepath", Aliases: []string{"pull"}, Args: cobra.ExactArgs(2), Short: "Get file from storage", Run: util.GetFile, } subCmd.AddCommand(getFileCmd) // FileInfo var fileInfoCmd = &cobra.Command{ Use: "info [user:pass@]hostname[:port]/collection/name", Args: cobra.ExactArgs(1), Short: "Show file information", Run: util.FileInfo, } subCmd.AddCommand(fileInfoCmd) // DeleteFile var deleteFileCmd = &cobra.Command{ Use: "delete [user:pass@]hostname[:port]/collection/name", Args: cobra.ExactArgs(1), Aliases: []string{"remove"}, Short: "Delete file in storage", Run: util.DeleteFile, } subCmd.AddCommand(deleteFileCmd) // ListFiles var listFilesCmd = &cobra.Command{ Use: "list [user:pass@]hostname[:port]/catalog", Args: cobra.ExactArgs(1), Short: "List files in storage", Run: util.ListFiles, } listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information") listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") subCmd.AddCommand(listFilesCmd) // ImportFiles var importFilesCmd = &cobra.Command{ Use: "import directory [user:pass@]hostname[:port]/collection", Args: cobra.ExactArgs(2), Short: "Send file tree to storage as is", Run: util.ImportFiles, } subCmd.AddCommand(importFilesCmd) // ExportFiles var exportFilesCmd = &cobra.Command{ Use: "export directory [user:pass@]hostname[:port]/collection dir", Args: cobra.ExactArgs(2), Short: "Download file tree to storage as is", Run: util.ExportFiles, } exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.Detail, "detail", "D", false, "Show detail file information") exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") subCmd.AddCommand(exportFilesCmd) return subCmd } func (util *FileUtil) MakeCollectionCmds() *cobra.Command { var subCmd = &cobra.Command{ Use: "collections", Short: "Colletion operations", Aliases: []string{"col", "cols", "dir", "dirs"}, } const defaultTimeout uint64 = 10 subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username") subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password") subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout") subCmd.MarkPersistentFlagRequired("host") subCmd.MarkFlagsRequiredTogether("user", "pass") // ListCollections var listCollectionsCmd = &cobra.Command{ Use: "list [user:pass@]hostname[:port]/catalog", Args: cobra.ExactArgs(1), Short: "List collections in storage", Run: util.ListCollections, } listCollectionsCmd.Flags().BoolVarP(&util.listCollectionsParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") subCmd.AddCommand(listCollectionsCmd) // DeleteCollection var deleteCollectionCmd = &cobra.Command{ Use: "delete [user:pass@]hostname[:port]/catalog", Args: cobra.ExactArgs(1), Short: "Delete all files in collection", Run: util.DeleteCollection, } deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.Detail, "detail", "D", false, "Show detail file information") deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsPrefix, "asprefix", "P", false, "Use path as collection path prefix") deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.DryRun, "dryrun", "Y", false, "Simulate process, don't delete files") subCmd.AddCommand(deleteCollectionCmd) return subCmd } type FileUtil struct { fileInfoParams FileInfoParams putFileParams PutFileParams getFileParams GetFileParams deleteFileParams DeleteFileParams listFilesParams ListFilesParams importFilesParams ImportFilesParams exportFilesParams ExportFilesParams deleteCollectionParams DeleteCollectionParams listCollectionsParams ListCollectionsParams commonFileParams CommonFileParams } type CommonFileParams struct { Username string Password string Timeout uint64 SkipTLSVerify bool } // PutFile type PutFileParams struct { Source string Dest string } type PutFileResult struct{} func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) { util.putFileParams.Source = args[0] util.putFileParams.Dest = args[1] res, err := util.putFile(&util.commonFileParams, &util.putFileParams) printResponse(res, err) } func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) { var err error res := &PutFileResult{} file, err := os.OpenFile(params.Dest, os.O_RDONLY, 0) if err != nil { return res, err } defer file.Close() stat, err := file.Stat() if err != nil { return res, err } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref, err := filecli.ParsePath(params.Dest) if err != nil { return res, err } ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) if err != nil { return res, err } return res, err } // Get file type GetFileParams struct { Source string Dest string } func (util *FileUtil) GetFile(cmd *cobra.Command, args []string) { util.getFileParams.Source = args[0] util.getFileParams.Dest = args[1] res, err := util.getFile(&util.commonFileParams, &util.getFileParams) printResponse(res, err) } type GetFileResult struct{} func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) { var err error res := &GetFileResult{} err = os.MkdirAll(filepath.Dir(params.Dest), 0750) if err != nil { return res, err } file, err := os.OpenFile(params.Dest, os.O_WRONLY|os.O_CREATE, 0640) if err != nil { return res, err } defer file.Close() timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref, err := filecli.ParsePath(params.Source) if err != nil { return res, err } ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) _, err = cli.GetFile(ctx, ref.Raw(), file) if err != nil { return res, err } return res, err } // DeleteFile type DeleteFileParams struct { Filepath string } type DeleteFileResult struct{} func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) { util.deleteFileParams.Filepath = args[0] res, err := util.deleteFile(&util.commonFileParams, &util.deleteFileParams) printResponse(res, err) } func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) { var err error res := &DeleteFileResult{} ref, err := filecli.ParsePath(params.Filepath) if err != nil { return res, err } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) _, err = cli.DeleteFile(ctx, ref.Raw()) if err != nil { return res, err } return res, err } // ImportFiles type ImportFilesParams struct { Source string Dest string Progress bool } type ImportFilesResult struct { Files []string `yaml:"files,omitempty"` } func (util *FileUtil) ImportFiles(cmd *cobra.Command, args []string) { util.importFilesParams.Source = args[0] util.importFilesParams.Dest = args[1] res, err := util.importFiles(&util.commonFileParams, &util.importFilesParams) printResponse(res, err) } func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesParams) (*ImportFilesResult, error) { var err error res := &ImportFilesResult{ Files: make([]string, 0), } params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password) if err != nil { return res, err } putErrors := make([]error, 0) walcFunc := func(walkPath string, infoItem fs.FileInfo, err error) error { if err != nil { return err } mode := infoItem.Mode() skip := mode == fs.ModeDevice skip = skip || mode == fs.ModeNamedPipe skip = skip || mode == fs.ModeCharDevice if skip { return nil } if !infoItem.IsDir() { relPath, _ := strings.CutPrefix(walkPath, params.Source) dest, _ := url.JoinPath(params.Dest, relPath) ref, err := filecli.ParsePath(dest) if err != nil { putErrors = append(putErrors, err) return nil } file, err := os.OpenFile(dest, os.O_RDONLY, 0) if err != nil { putErrors = append(putErrors, err) return nil } defer file.Close() stat, err := file.Stat() if err != nil { putErrors = append(putErrors, err) return nil } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref.SetUserinfo(common.Username, common.Password) user, pass := ref.Userinfo() mw := filecli.NewBasicAuthMiddleware(user, pass) cli := filecli.NewClient(nil, mw) err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) if err != nil { putErrors = append(putErrors, err) fmt.Printf("- %s: error: %v \n", walkPath, err) } else { res.Files = append(res.Files, walkPath) fmt.Printf("- %s: ok\n", walkPath) } } return nil } for _, item := range putErrors { err = errors.Join(err, item) } err = filepath.Walk(params.Source, walcFunc) if err != nil { return res, err } return res, err } // ExportFiles type ExportFilesParams struct { Filepath string Detail bool AsPrefix bool AsRegexp bool Dest string } type ExportFilesResult struct { Files []descr.File `yaml:"files,omitempty"` Filenames []string `yaml:"filenames,omitempty"` } func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) { util.exportFilesParams.Filepath = args[0] util.exportFilesParams.Dest = args[1] res, err := util.exportFiles(&util.commonFileParams, &util.exportFilesParams) printResponse(res, err) } func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) { var err error res := &ExportFilesResult{} timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref, err := filecli.ParsePath(params.Filepath) if err != nil { return res, err } ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) list, err := cli.ListFiles(ctx, ref.Raw()) if err != nil { return res, err } files := make([]descr.File, 0) err = json.Unmarshal(list, &files) if err != nil { return res, err } exported := make([]descr.File, 0) for _, descr := range files { destdir := filepath.Join(params.Dest, descr.Collection) err = os.MkdirAll(destdir, 0750) if err != nil { return res, err } destpath := filepath.Join(params.Dest, descr.Collection, descr.Name) destfile, err := os.OpenFile(destpath, os.O_WRONLY|os.O_CREATE, 0640) if err != nil { return res, err } defer destfile.Close() timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref.SetResource(filepath.Join(descr.Collection, descr.Name)) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) _, err = cli.GetFile(ctx, ref.Raw(), destfile) if err != nil { fmt.Printf("- %s: error %v\n", ref.Raw(), err) err = nil } else { fmt.Printf("- %s: ok\n", ref.Raw()) exported = append(exported, descr) } } if params.Detail { res.Files = exported } else { files := descr.NewFiles() files.Set(exported) res.Filenames = files.List() } return res, err } func makeFileURI(hosturi, collection, name string) (string, error) { var err error var res string uri, err := url.Parse(hosturi) if err != nil { return res, err } uri.Path = path.Join(collection, name) res = uri.String() return res, err } // ListCollections type ListCollectionsParams struct { Path string AsPrefix bool AsRegexp bool } type ListCollectionsResult struct { Collections []string `yaml:"collections,omitempty"` } func (util *FileUtil) ListCollections(cmd *cobra.Command, args []string) { util.listCollectionsParams.Path = args[0] res, err := util.listCollections(&util.commonFileParams, &util.listCollectionsParams) printResponse(res, err) } func (util *FileUtil) listCollections(common *CommonFileParams, params *ListCollectionsParams) (*ListCollectionsResult, error) { var err error res := &ListCollectionsResult{ Collections: make([]string, 0), } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref, err := filecli.ParsePath(params.Path) if err != nil { return res, err } ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) list, err := cli.ListCollections(ctx, ref.Raw()) if err != nil { return res, err } collections := make([]string, 0) err = json.Unmarshal(list, collections) if err != nil { return res, err } res.Collections = collections return res, err } // DeleteCollection type DeleteCollectionParams struct { Path string Detail bool AsPrefix bool AsRegexp bool DryRun bool } type DeleteCollectionResult struct { Files []descr.File `yaml:"files,omitempty"` Filenames []string `yaml:"filenames,omitempty"` } func (util *FileUtil) DeleteCollection(cmd *cobra.Command, args []string) { util.deleteCollectionParams.Path = args[0] res, err := util.deleteCollection(&util.commonFileParams, &util.deleteCollectionParams) printResponse(res, err) } func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteCollectionParams) (*DeleteCollectionResult, error) { var err error res := &DeleteCollectionResult{ Filenames: make([]string, 0), } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) ref, err := filecli.ParsePath(params.Path) if err != nil { return res, err } ref.SetUserinfo(common.Username, common.Password) mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) cli := filecli.NewClient(nil, mw) list, err := cli.DeleteCollection(ctx, ref.Raw()) files := descr.NewFiles() err = json.Unmarshal(list, files.ArrayPtr()) if err != nil { return res, err } if !params.Detail { res.Filenames = files.List() } else { res.Files = files.Array() } return res, err }