mstore/cmd/mstorectl/filecmd.go

/*
 * Copyright 2026 Oleg Borodin <onborodin@gmail.com>
 *
 * This work is published and licensed under a Creative Commons
 * Attribution-NonCommercial-NoDerivatives 4.0 International License.
 *
 * Distribution of this work is permitted, but commercial use and
 * modifications are strictly prohibited.
 */
package main

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"mstore/pkg/client"
	"mstore/pkg/descr"
	"mstore/pkg/terms"
)

func (util *FileUtil) CreateFileCmds() *cobra.Command {
	var subCmd = &cobra.Command{
		Use:     "files",
		Short:   "File operations",
		Aliases: []string{"file"},
	}
	const defaultTimeout uint64 = 10

	subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username")
	subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password")
	subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout")
	subCmd.PersistentFlags().BoolVarP(&util.commonFileParams.SkipTLSVerify, "skipVerify", "S", true, "Skip server certificate verify")
	subCmd.MarkPersistentFlagRequired("host")
	subCmd.MarkFlagsRequiredTogether("user", "pass")

	vi := viper.New()
	vi.SetEnvPrefix("mstore")
	vi.BindEnv("user")
	vi.BindEnv("pass")
	util.commonFileParams.Username = vi.GetString("user")
	util.commonFileParams.Password = vi.GetString("pass")

	// PutFile
	var putFileCmd = &cobra.Command{
		Use:     "put filepath [user:pass@]hostname[:port]/collection/name",
		Args:    cobra.ExactArgs(2),
		Aliases: []string{"push"},
		Short:   "Put file to storage",
		Run:     util.PutFile,
	}
	subCmd.AddCommand(putFileCmd)

	// GetFile
	var getFileCmd = &cobra.Command{
		Use:     "get [user:pass@]hostname[:port]/collection/name filepath",
		Aliases: []string{"pull"},
		Args:    cobra.ExactArgs(2),
		Short:   "Get file from storage",
		Run:     util.GetFile,
	}
	subCmd.AddCommand(getFileCmd)

	// FileInfo
	var fileInfoCmd = &cobra.Command{
		Use:   "info [user:pass@]hostname[:port]/collection/name",
		Args:  cobra.ExactArgs(1),
		Short: "Show file information",
		Run:   util.FileInfo,
	}
	subCmd.AddCommand(fileInfoCmd)

	// DeleteFile
	var deleteFileCmd = &cobra.Command{
		Use:     "delete [user:pass@]hostname[:port]/collection/name",
		Args:    cobra.ExactArgs(1),
		Aliases: []string{"remove"},
		Short:   "Delete file in storage",
		Run:     util.DeleteFile,
	}
	subCmd.AddCommand(deleteFileCmd)

	// ListFiles
	var listFilesCmd = &cobra.Command{
		Use:   "list [user:pass@]hostname[:port]/catalog",
		Args:  cobra.ExactArgs(1),
		Short: "List files in storage",
		Run:   util.ListFiles,
	}
	listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information")
	listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix")
	listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix")
	subCmd.AddCommand(listFilesCmd)

	// ImportFiles
	var importFilesCmd = &cobra.Command{
		Use:   "import directory [user:pass@]hostname[:port]/collection",
		Args:  cobra.ExactArgs(2),
		Short: "Send file tree to storage as is",
		Run:   util.ImportFiles,
	}
	subCmd.AddCommand(importFilesCmd)

	// ExportFiles
	var exportFilesCmd = &cobra.Command{
		Use:   "export directory [user:pass@]hostname[:port]/collection dir",
		Args:  cobra.ExactArgs(2),
		Short: "Download file tree to storage as is",
		Run:   util.ExportFiles,
	}
	exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.Detail, "detail", "D", false, "Show detail file information")
	exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix")
	exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix")
	subCmd.AddCommand(exportFilesCmd)

	return subCmd
}

func (util *FileUtil) CreateCollectionCmds() *cobra.Command {
	var subCmd = &cobra.Command{
		Use:     "collections",
		Short:   "Colletion operations",
		Aliases: []string{"col", "cols", "dir", "dirs"},
	}
	const defaultTimeout uint64 = 10

	subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username")
	subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password")
	subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout")
	subCmd.MarkPersistentFlagRequired("host")
	subCmd.MarkFlagsRequiredTogether("user", "pass")

	// ListCollections
	var listCollectionsCmd = &cobra.Command{
		Use:   "list [user:pass@]hostname[:port]/catalog",
		Args:  cobra.ExactArgs(1),
		Short: "List collections in storage",
		Run:   util.ListCollections,
	}
	listCollectionsCmd.Flags().BoolVarP(&util.listCollectionsParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix")
	subCmd.AddCommand(listCollectionsCmd)

	// DeleteCollection
	var deleteCollectionCmd = &cobra.Command{
		Use:   "delete [user:pass@]hostname[:port]/catalog",
		Args:  cobra.ExactArgs(1),
		Short: "Delete all files in collection",
		Run:   util.DeleteCollection,
	}
	deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.Detail, "detail", "D", false, "Show detail file information")
	deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsPrefix, "asprefix", "P", false, "Use path as collection path prefix")
	deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix")
	deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.DryRun, "dryrun", "Y", false, "Simulate process, don't delete files")

	subCmd.AddCommand(deleteCollectionCmd)

	return subCmd
}

type FileUtil struct {
	fileInfoParams         FileInfoParams
	putFileParams          PutFileParams
	getFileParams          GetFileParams
	deleteFileParams       DeleteFileParams
	listFilesParams        ListFilesParams
	importFilesParams      ImportFilesParams
	exportFilesParams      ExportFilesParams
	listCollectionsParams  ListCollectionsParams
	deleteCollectionParams DeleteCollectionParams
	commonFileParams       CommonFileParams
}

type CommonFileParams struct {
	Username      string
	Password      string
	Timeout       uint64
	SkipTLSVerify bool
}

// FileInfo
type FileInfoParams struct {
	Filepath string
}
type FileInfoResult struct {
	File *descr.File `yaml:"file,omitempty"`
}

func (util *FileUtil) FileInfo(cmd *cobra.Command, args []string) {
	util.fileInfoParams.Filepath = args[0]
	res, err := util.fileInfo(&util.commonFileParams, &util.fileInfoParams)
	printResponse(res, err)
}

func (util *FileUtil) fileInfo(common *CommonFileParams, params *FileInfoParams) (*FileInfoResult, error) {
	var err error
	res := &FileInfoResult{}
	params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	exists, opres, err := client.NewClient(common.SkipTLSVerify).FileInfo(ctx, params.Filepath)
	if err != nil {
		return res, err
	}
	if !exists {
		err = fmt.Errorf("File %s not exists", params.Filepath)
		return res, err
	}
	res.File = opres
	return res, err
}

// PutFile
type PutFileParams struct {
	Source string
	Dest   string
}
type PutFileResult struct{}

func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) {
	util.putFileParams.Source = args[0]
	util.putFileParams.Dest = args[1]
	res, err := util.putFile(&util.commonFileParams, &util.putFileParams)
	printResponse(res, err)
}

func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) {
	var err error
	res := &PutFileResult{}
	params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, params.Source, params.Dest)
	if err != nil {
		return res, err
	}
	return res, err
}

// Get file
type GetFileParams struct {
	Source string
	Dest   string
}

func (util *FileUtil) GetFile(cmd *cobra.Command, args []string) {
	util.getFileParams.Source = args[0]
	util.getFileParams.Dest = args[1]
	res, err := util.getFile(&util.commonFileParams, &util.getFileParams)
	printResponse(res, err)
}

type GetFileResult struct{}

func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) {
	var err error
	res := &GetFileResult{}
	params.Dest, err = packUserinfo(params.Source, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, params.Dest, params.Source)
	if err != nil {
		return res, err
	}
	return res, err
}

// DeleteFile
type DeleteFileParams struct {
	Filepath string
}

type DeleteFileResult struct{}

func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) {
	util.deleteFileParams.Filepath = args[0]
	res, err := util.deleteFile(&util.commonFileParams, &util.deleteFileParams)
	printResponse(res, err)
}
func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) {
	var err error
	res := &DeleteFileResult{}
	params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	err = client.NewClient(common.SkipTLSVerify).DeleteFile(ctx, params.Filepath)
	if err != nil {
		return res, err
	}
	return res, err
}

// ListFiles
type ListFilesParams struct {
	Filepath string
	Detail   bool
	AsPrefix bool
	AsRegexp bool
}

type ListFilesResult struct {
	Files     []descr.File `yaml:"files,omitempty"`
	Filenames []string     `yaml:"filenames,omitempty"`
}

func (util *FileUtil) ListFiles(cmd *cobra.Command, args []string) {
	util.listFilesParams.Filepath = args[0]
	res, err := util.listFiles(&util.commonFileParams, &util.listFilesParams)
	printResponse(res, err)
}
func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParams) (*ListFilesResult, error) {
	var err error
	res := &ListFilesResult{}
	params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	if params.AsRegexp {
		params.AsPrefix = false
	}
	var pathUsage string
	switch {
	case params.AsRegexp:
		pathUsage = terms.AsRegexp
	case params.AsPrefix:
		pathUsage = terms.AsPrefix
	default:
		pathUsage = terms.AsFinePath
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
	if err != nil {
		return res, err
	}

	if params.Detail {
		res.Files = files
	} else {
		res.Filenames = makeFilelistFromFiles(files)
	}
	return res, err
}

// ImportFiles
type ImportFilesParams struct {
	Source   string
	Dest     string
	Progress bool
}
type ImportFilesResult struct {
	Files []string `yaml:"files,omitempty"`
}

func (util *FileUtil) ImportFiles(cmd *cobra.Command, args []string) {
	util.importFilesParams.Source = args[0]
	util.importFilesParams.Dest = args[1]

	res, err := util.importFiles(&util.commonFileParams, &util.importFilesParams)
	printResponse(res, err)
}

func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesParams) (*ImportFilesResult, error) {
	var err error
	res := &ImportFilesResult{
		Files: make([]string, 0),
	}
	params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password)
	if err != nil {
		return res, err
	}

	putErrors := make([]error, 0)
	walcFunc := func(walkPath string, infoItem fs.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if infoItem.Mode() == fs.ModeDevice {
			return nil
		}
		if infoItem.Mode() == fs.ModeNamedPipe {
			return nil
		}
		if infoItem.Mode() == fs.ModeCharDevice {
			return nil
		}
		if !infoItem.IsDir() {
			timeout := time.Duration(common.Timeout) * time.Second
			ctx, _ := context.WithTimeout(context.Background(), timeout)
			relPath, _ := strings.CutPrefix(walkPath, params.Source)
			dest, _ := url.JoinPath(params.Dest, relPath)
			if err != nil {
				putErrors = append(putErrors, err)
				return nil
			}
			err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, walkPath, dest)
			if err != nil {
				putErrors = append(putErrors, err)
				fmt.Printf("- %s: error: %v \n", walkPath, err)
			} else {
				res.Files = append(res.Files, walkPath)
				fmt.Printf("- %s: ok\n", walkPath)
			}
		}
		return nil
	}
	for _, item := range putErrors {
		err = errors.Join(err, item)
	}
	err = filepath.Walk(params.Source, walcFunc)
	if err != nil {
		return res, err
	}
	return res, err
}

// ExportFiles
type ExportFilesParams struct {
	Filepath string
	Detail   bool
	AsPrefix bool
	AsRegexp bool
	Dest     string
}

type ExportFilesResult struct {
	Files     []descr.File `yaml:"files,omitempty"`
	Filenames []string     `yaml:"filenames,omitempty"`
}

func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) {
	util.exportFilesParams.Filepath = args[0]
	util.exportFilesParams.Dest = args[1]
	res, err := util.exportFiles(&util.commonFileParams, &util.exportFilesParams)
	printResponse(res, err)
}
func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) {
	var err error
	res := &ExportFilesResult{}
	params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	if params.AsRegexp {
		params.AsPrefix = false
	}
	var pathUsage string
	switch {
	case params.AsRegexp:
		pathUsage = terms.AsRegexp
	case params.AsPrefix:
		pathUsage = terms.AsPrefix
	default:
		pathUsage = terms.AsFinePath
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
	if err != nil {
		return res, err
	}

	exportedFiles := make([]descr.File, 0)
	for _, file := range files {
		destdir := filepath.Join(params.Dest, file.Collection)
		err = os.MkdirAll(destdir, 0750)
		if err != nil {
			return res, err
		}
		srcpath, err := makeFileURI(params.Filepath, file.Collection, file.Name)
		if err != nil {
			return res, err
		}
		destpath := filepath.Join(params.Dest, file.Collection, file.Name)
		timeout := time.Duration(common.Timeout) * time.Second
		ctx, _ := context.WithTimeout(context.Background(), timeout)
		_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, srcpath, destpath)
		if err != nil {
			fmt.Printf("- %s: error %v\n", srcpath, err)
			//return res, err
			err = nil
		} else {
			fmt.Printf("- %s: ok\n", srcpath)
			exportedFiles = append(exportedFiles, file)
		}
	}
	if params.Detail {
		res.Files = exportedFiles
	} else {
		res.Filenames = makeFilelistFromFiles(exportedFiles)
	}
	return res, err
}

func makeFileURI(hosturi, collection, name string) (string, error) {
	var err error
	var res string
	uri, err := url.Parse(hosturi)
	if err != nil {
		return res, err
	}
	uri.Path = path.Join(collection, name)
	res = uri.String()
	return res, err
}

// ListCollections
type ListCollectionsParams struct {
	Path     string
	AsPrefix bool
	AsRegexp bool
}

type ListCollectionsResult struct {
	Collections []string `yaml:"collections,omitempty"`
}

func (util *FileUtil) ListCollections(cmd *cobra.Command, args []string) {
	util.listCollectionsParams.Path = args[0]
	res, err := util.listCollections(&util.commonFileParams, &util.listCollectionsParams)
	printResponse(res, err)
}

func (util *FileUtil) listCollections(common *CommonFileParams, params *ListCollectionsParams) (*ListCollectionsResult, error) {
	var err error
	res := &ListCollectionsResult{
		Collections: make([]string, 0),
	}
	params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	if params.AsRegexp {
		params.AsPrefix = false
	}
	var pathUsage string
	switch {
	case params.AsRegexp:
		pathUsage = terms.AsRegexp
	case params.AsPrefix:
		pathUsage = terms.AsPrefix
	default:
		pathUsage = terms.AsFinePath
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collecions, err := client.NewClient(common.SkipTLSVerify).ListCollections(ctx, params.Path, pathUsage)
	if err != nil {
		return res, err
	}
	res.Collections = collecions
	return res, err
}

// DeleteCollection
type DeleteCollectionParams struct {
	Path     string
	Detail   bool
	AsPrefix bool
	AsRegexp bool
	DryRun   bool
}

type DeleteCollectionResult struct {
	Files     []descr.File `yaml:"files,omitempty"`
	Filenames []string     `yaml:"filenames,omitempty"`
}

func (util *FileUtil) DeleteCollection(cmd *cobra.Command, args []string) {
	util.deleteCollectionParams.Path = args[0]
	res, err := util.deleteCollection(&util.commonFileParams, &util.deleteCollectionParams)
	printResponse(res, err)
}

func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteCollectionParams) (*DeleteCollectionResult, error) {
	var err error
	res := &DeleteCollectionResult{
		Filenames: make([]string, 0),
	}
	params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
	if err != nil {
		return res, err
	}
	timeout := time.Duration(common.Timeout) * time.Second
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	var pathUsage string
	switch {
	case params.AsPrefix:
		pathUsage = terms.AsPrefix
	default:
		pathUsage = terms.AsFinePath
	}
	files, err := client.NewClient(common.SkipTLSVerify).DeleteCollection(ctx, params.Path, pathUsage, params.DryRun)
	if err != nil {
		return res, err
	}
	if params.Detail {
		res.Files = files
	} else {
		res.Filenames = makeFilelistFromFiles(files)
	}
	return res, err
}

func makeFilelistFromFiles(files []descr.File) []string {
	res := make([]string, 0, len(files))
	for _, file := range files {
		res = append(res, filepath.Join(file.Collection, file.Name))
	}
	slices.Sort(res)
	return res
}