working commit

app/operator/file.go

@@ -17,6 +17,7 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"mstore/app/descr"
 	"mstore/pkg/auxtool"
@@ -282,18 +283,16 @@ type ListFilesParams struct {
 }
 type ListFilesResult struct {
 	Files       []descr.File `json:"files,omitempty"`
-	//Catalogs []string  `json:"files,omitempty"`
+	Collections []string     `json:"collection,omitempty"`
 }
 
 func (oper *Operator) ListFiles(ctx context.Context, operID string, param *ListFilesParams) (int, *ListFilesResult, error) {
 	var err error
 	res := &ListFilesResult{
 		Files:       make([]descr.File, 0),
-		//Catalogs: make([]string, 0)
+		Collections: make([]string, 0),
 	}
 
-	// TODO: convert file path to a unified and secure state
-
 	_, err = cleanFilepath(param.Filepath)
 	if err != nil {
 		code := http.StatusInternalServerError
@@ -305,8 +304,14 @@ func (oper *Operator) ListFiles(ctx context.Context, operID string, param *ListF
 		code := http.StatusInternalServerError
 		return code, res, err
 	}
+	rFilepath := filepath.Join("/", param.Filepath)
 	for _, item := range fileDescrs {
+		cFilepath := filepath.Join("/", item.Collection, item.Name)
+		collection := filepath.Join("/", item.Collection)
+		if strings.HasPrefix(cFilepath, rFilepath) {
 			res.Files = append(res.Files, item)
+			res.Collections = append(res.Collections, collection)
+		}
 	}
 	code := http.StatusOK
 	return code, res, err

app/router/context.go

@@ -15,6 +15,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strconv"
 )
 
 type Context struct {
@@ -123,18 +124,28 @@ func (rctx *Context) SetStatus(httpStatus int) {
 }
 
 func (rctx *Context) SendJSON(statusCode int, payload any) {
+
+	buffer := bytes.NewBuffer(nil)
+	json.NewEncoder(buffer).Encode(payload)
 	rctx.Writer.Header().Set("Content-Type", "application/json")
+	size := strconv.FormatInt(int64(len(buffer.Bytes())), 10)
+	rctx.Writer.Header().Set("Content-Length", size)
 	rctx.Writer.WriteHeader(statusCode)
-	json.NewEncoder(rctx.Writer).Encode(payload)
+	rctx.Writer.Write(buffer.Bytes())
 }
 
 func (rctx *Context) SendText(statusCode int, payload string) {
+	size := strconv.FormatInt(int64(len(payload)), 10)
 	rctx.Writer.Header().Set("Content-Type", "text/plain")
+	rctx.Writer.Header().Set("Content-Length", size)
 	rctx.Writer.WriteHeader(statusCode)
 	rctx.Writer.Write([]byte(payload))
 }
 
 func (rctx *Context) SendBytes(statusCode int, payload []byte) {
+	size := strconv.FormatInt(int64(len(payload)), 10)
+	rctx.Writer.Header().Set("Content-Type", "application/octet-stream")
+	rctx.Writer.Header().Set("Content-Length", size)
 	rctx.Writer.WriteHeader(statusCode)
 	rctx.Writer.Write(payload)
 }

cmd/mstorectl/accountcmd.go

@@ -124,8 +124,8 @@ type CreateAccountParams struct {
 	NewPassword string
 }
 type CreateAccountResult struct {
-	AccountID string   `json:"accountId"`
+	AccountID string   `yaml:"accountId"`
-	GrantIDs  []string `json:"grantsIds,omitempty"`
+	GrantIDs  []string `yaml:"grantsIds,omitempty"`
 }
 
 func (util *AccountUtil) CreateAccount(cmd *cobra.Command, args []string) {
@@ -179,7 +179,7 @@ type UpdateAccountParams struct {
 	NewPassword string
 }
 type UpdateAccountResult struct {
-	File *descr.File `json:"file,omitempty"`
+	File *descr.File `yaml:"file,omitempty"`
 }
 
 func (util *AccountUtil) UpdateAccount(cmd *cobra.Command, args []string) {
@@ -224,7 +224,7 @@ func (util *AccountUtil) GetAccount(cmd *cobra.Command, args []string) {
 }
 
 type GetAccountResult struct {
-	Account *descr.AccountShort `json:"account,omitempty"`
+	Account *descr.AccountShort `yaml:"account,omitempty"`
 }
 
 func (util *AccountUtil) getAccount(common *CommonAccountParams, params *GetAccountParams) (*GetAccountResult, error) {
@@ -300,13 +300,13 @@ type ListAccountsParams struct {
 }
 
 type Userinfo struct {
-	Username string   `json:"username,omitempty"`
+	Username string   `yaml:"username,omitempty"`
-	Rights   []string `json:"rights,omitempty"`
+	Rights   []string `yaml:"rights,omitempty"`
 }
 
 type ListAccountsResult struct {
-	Accounts []descr.AccountShort `json:"accounts,omitempty"`
+	Accounts []descr.AccountShort `yaml:"accounts,omitempty"`
-	Users    []Userinfo           `json:"users,omitempty"`
+	Users    []Userinfo           `yaml:"users,omitempty"`
 }
 
 func (util *AccountUtil) ListAccounts(cmd *cobra.Command, args []string) {

cmd/mstorectl/filecmd.go

@@ -12,6 +12,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"io/fs"
+	"net/url"
+	"path/filepath"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -41,8 +44,8 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
 		Short:   "Put file to storage",
 		Run:     util.PutFile,
 	}
-	putFileCmd.Flags().StringVarP(&util.putFileParams.Source, "src", "s", "", "Source path")
+	putFileCmd.Flags().StringVarP(&util.putFileParams.Source, "src", "S", "", "Source path")
-	putFileCmd.Flags().StringVarP(&util.putFileParams.Dest, "dest", "d", "", "Desctination path")
+	putFileCmd.Flags().StringVarP(&util.putFileParams.Dest, "dest", "D", "", "Desctination path")
 	putFileCmd.MarkFlagsRequiredTogether("src", "dest")
 	putFileCmd.MarkFlagFilename("src")
 
@@ -55,8 +58,8 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
 		Short:   "Get file from storage",
 		Run:     util.GetFile,
 	}
-	getFileCmd.Flags().StringVarP(&util.getFileParams.Source, "src", "s", "", "Source path")
+	getFileCmd.Flags().StringVarP(&util.getFileParams.Source, "src", "S", "", "Source path")
-	getFileCmd.Flags().StringVarP(&util.getFileParams.Dest, "dest", "d", "", "Desctination path")
+	getFileCmd.Flags().StringVarP(&util.getFileParams.Dest, "dest", "D", "", "Desctination path")
 	getFileCmd.MarkFlagsRequiredTogether("src", "dest")
 
 	subCmd.AddCommand(getFileCmd)
@@ -67,7 +70,7 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
 		Short: "Show file information",
 		Run:   util.FileInfo,
 	}
-	fileInfoCmd.Flags().StringVarP(&util.fileInfoParams.Filepath, "path", "d", "", "File path")
+	fileInfoCmd.Flags().StringVarP(&util.fileInfoParams.Filepath, "path", "P", "", "File path")
 	fileInfoCmd.MarkFlagRequired("path")
 
 	subCmd.AddCommand(fileInfoCmd)
@@ -79,7 +82,9 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
 		Short:   "Delete file in storage",
 		Run:     util.DeleteFile,
 	}
-	deleteFileCmd.Flags().StringVarP(&util.deleteFileParams.Filepath, "path", "d", "", "File path")
+
+	deleteFileCmd.Flags().StringVarP(&util.deleteFileParams.Filepath, "path", "P", "", "File path")
+
 	deleteFileCmd.MarkFlagRequired("path")
 	subCmd.AddCommand(deleteFileCmd)
 
@@ -89,10 +94,23 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
 		Short: "List file in storage",
 		Run:   util.ListFiles,
 	}
-	listFilesCmd.Flags().StringVarP(&util.listFilesParams.Filepath, "catalog", "c", "", "Catalog path")
+	listFilesCmd.Flags().StringVarP(&util.listFilesParams.Filepath, "catalog", "C", "", "Catalog path")
+	listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information")
 	listFilesCmd.MarkFlagRequired("catalog")
 	subCmd.AddCommand(listFilesCmd)
 
+	// ImportFiles
+	var importFilesCmd = &cobra.Command{
+		Use:   "import",
+		Short: "Send file tree to storage as is",
+		Run:   util.ImportFiles,
+	}
+	importFilesCmd.Flags().StringVarP(&util.importFilesParams.Source, "src", "S", "", "Source base path")
+	importFilesCmd.Flags().StringVarP(&util.importFilesParams.Dest, "dest", "D", "", "Desctination base path")
+	importFilesCmd.MarkFlagsRequiredTogether("src", "dest")
+
+	subCmd.AddCommand(importFilesCmd)
+
 	return subCmd
 }
 
@@ -102,6 +120,7 @@ type FileUtil struct {
 	getFileParams     GetFileParams
 	deleteFileParams  DeleteFileParams
 	listFilesParams   ListFilesParams
+	importFilesParams ImportFilesParams
 	commonFileParams  CommonFileParams
 }
 
@@ -116,7 +135,7 @@ type FileInfoParams struct {
 	Filepath string
 }
 type FileInfoResult struct {
-	File *descr.File `json:"file,omitempty"`
+	File *descr.File `yaml:"file,omitempty"`
 }
 
 func (util *FileUtil) FileInfo(cmd *cobra.Command, args []string) {
@@ -232,10 +251,12 @@ func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFilePar
 // ListFiles
 type ListFilesParams struct {
 	Filepath string
+	Detail   bool
 }
 
 type ListFilesResult struct {
-	Files []descr.File `json:"files"`
+	Files     []descr.File `yaml:"files,omitempty"`
+	Filenames []string     `yaml:"filenames,omitempty"`
 }
 
 func (util *FileUtil) ListFiles(cmd *cobra.Command, args []string) {
@@ -255,6 +276,79 @@ func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParam
 	if err != nil {
 		return res, err
 	}
+
+	if params.Detail {
 		res.Files = files
+	} else {
+		for _, file := range files {
+			filename := filepath.Join("/", file.Collection, file.Name)
+			res.Filenames = append(res.Filenames, filename)
+		}
+	}
+	return res, err
+}
+
+// ImportFiles
+type ImportFilesParams struct {
+	Source   string
+	Dest     string
+	Progress bool
+}
+type ImportFilesResult struct {
+	Files []string `yaml:"files,omitempty"`
+}
+
+func (util *FileUtil) ImportFiles(cmd *cobra.Command, args []string) {
+	res, err := util.importFiles(&util.commonFileParams, &util.importFilesParams)
+	printResponse(res, err)
+}
+
+func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesParams) (*ImportFilesResult, error) {
+	var err error
+	res := &ImportFilesResult{
+		Files: make([]string, 0),
+	}
+	params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password)
+	if err != nil {
+		return res, err
+	}
+
+	putErrors := make([]error, 0)
+	walcFunc := func(walkPath string, infoItem fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if infoItem.Mode() == fs.ModeDevice {
+			return nil
+		}
+		if infoItem.Mode() == fs.ModeNamedPipe {
+			return nil
+		}
+		if infoItem.Mode() == fs.ModeCharDevice {
+			return nil
+		}
+		if !infoItem.IsDir() {
+			timeout := time.Duration(common.Timeout) * time.Second
+			ctx, _ := context.WithTimeout(context.Background(), timeout)
+			dest, _ := url.JoinPath(params.Dest, walkPath)
+			if err != nil {
+				putErrors = append(putErrors, err)
+				return nil
+			}
+			err = client.NewClient().PutFile(ctx, walkPath, dest)
+			if err != nil {
+				putErrors = append(putErrors, err)
+				fmt.Printf("- %s: error\n", walkPath)
+			} else {
+				res.Files = append(res.Files, walkPath)
+				fmt.Printf("- %s: ok\n", walkPath)
+			}
+		}
+		return nil
+	}
+	err = filepath.Walk(params.Source, walcFunc)
+	if err != nil {
+		return res, err
+	}
 	return res, err
 }

cmd/mstorectl/grantcmd.go

@@ -123,7 +123,7 @@ type CreateGrantParams struct {
 	Pattern   string
 }
 type CreateGrantResult struct {
-	GrantID string `json:"grantId"`
+	GrantID string `yaml:"grantId"`
 }
 
 func (util *GrantUtil) CreateGrant(cmd *cobra.Command, args []string) {
@@ -184,7 +184,7 @@ type GetGrantParams struct {
 }
 
 type GetGrantResult struct {
-	Grant *descr.Grant `json:"grant,omitempty"`
+	Grant *descr.Grant `yaml:"grant,omitempty"`
 }
 
 func (util *GrantUtil) GetGrant(cmd *cobra.Command, args []string) {
@@ -247,8 +247,8 @@ type ListGrantsParams struct {
 }
 
 type ListGrantsResult struct {
-	Grants []descr.Grant `json:"grants,omitempty"`
+	Grants []descr.Grant `yaml:"grants,omitempty"`
-	Rights []string      `json:"rights,omitempty"`
+	Rights []string      `yaml:"rights,omitempty"`
 }
 
 func (util *GrantUtil) ListGrants(cmd *cobra.Command, args []string) {

cmd/mstorectl/imagecmd.go

@@ -155,7 +155,7 @@ type ImageInfoParams struct {
 }
 
 type ImageInfoResult struct {
-	ImageInfo *client.ImageDescr `json:"imageInfo"`
+	ImageInfo *client.ImageDescr `yaml:"imageInfo"`
 }
 
 func (util *ImageUtil) ImageInfo(cmd *cobra.Command, args []string) {

cmd/mstorectl/main.go

@@ -15,7 +15,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
-	"sigs.k8s.io/yaml"
+	yaml "go.yaml.in/yaml/v4"
 )
 
 func main() {
@@ -76,9 +76,9 @@ func (util *Util) Hello(cmd *cobra.Command, args []string) {
 
 func printResponse(res any, err error) {
 	type Response struct {
-		Error   bool   `json:"error"             yaml:"error"`
+		Error   bool   `yaml:"error"             yaml:"error"`
-		Message string `json:"message,omitempty" yaml:"message,omitempty"`
+		Message string `yaml:"message,omitempty" yaml:"message,omitempty"`
-		Result  any    `json:"result,omitempty"  yaml:"result,omitempty"`
+		Result  any    `yaml:"result,omitempty"  yaml:"result,omitempty"`
 	}
 	resp := Response{}
 	if err != nil {

go.mod

@@ -1,14 +1,17 @@
 module mstore
 
-go 1.24.4
+go 1.25.6
 
 require (
 	github.com/google/go-containerregistry v0.20.7
 	github.com/google/uuid v1.6.0
 	github.com/jmoiron/sqlx v1.4.0
-	github.com/mattn/go-sqlite3 v1.14.33
+	github.com/mattn/go-sqlite3 v1.14.34
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.1
 	github.com/spf13/cobra v1.10.2
 	github.com/stretchr/testify v1.11.1
+	go.yaml.in/yaml/v4 v4.0.0-rc.4
 	sigs.k8s.io/yaml v1.6.0
 )
 
@@ -21,14 +24,11 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	github.com/vbatts/tar-split v0.12.2 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect
 	golang.org/x/sync v0.18.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -29,8 +29,8 @@ github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdB
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
+github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
-github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=

pkg/client/fileaux.go

@@ -75,7 +75,7 @@ func repackServiceURI(fileuri string) (string, string, string, error) {
 		password, _ = uri.User.Password()
 	}
 	uri.User = nil
-	uri.Scheme = "https"
+	//uri.Scheme = "https"
 	res = uri.String()
 	return res, username, password, err
 }