diff --git a/Makefile.am b/Makefile.am index 56da37c..65f22d8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -13,8 +13,7 @@ EXTRA_mstorectl_SOURCES = \ cmd/mstorectl/command/accountcmd.go \ cmd/mstorectl/command/grantcmd.go \ cmd/mstorectl/command/imagecmd.go \ - cmd/mstorectl/command/util.go \ - cmd/mstorectl/command/filecmd.go + cmd/mstorectl/command/util.go mstored_SOURCES = cmd/mstored/main.go @@ -177,14 +176,14 @@ EXTRA_DIST = vendor/ \ test/file_test.go \ test/image_test.go \ test/test-oci.tar - + format: @dirs=$$($(FIND) $(CWD)/app $(CWD)/cmd $(CWD)/pkg $(CWD)/test \ -name '*.go' | $(XARGS) -n1 dirname | $(SORT) | $(UNIQ)); \ for dir in $$dirs;do \ (echo "====$$dir===="; cd $$dir && $(GO) fmt .); \ - done + done .PHONY: test test: @@ -231,7 +230,7 @@ debian-package: $(DIST_ARCHIVES) cd $(BUILD_DIR)/$(distdir) && $(DBUILDPACKAGE) -nc -us -uc -ui -i -b mkdir -p $(DIST_DIR) $(CP) $(BUILD_DIR)/*.deb $(DIST_DIR) - rm -rf $(BUILD_DIR) + rm -rf $(BUILD_DIR) FREEBSD_LOCALBASE = /usr/local FREEBSD_RCDIR = $(FREEBSD_LOCALBASE)/etc/rc.d diff --git a/Makefile.in b/Makefile.in index 0f33081..4206a84 100644 --- a/Makefile.in +++ b/Makefile.in @@ -359,8 +359,7 @@ EXTRA_mstorectl_SOURCES = \ cmd/mstorectl/command/accountcmd.go \ cmd/mstorectl/command/grantcmd.go \ cmd/mstorectl/command/imagecmd.go \ - cmd/mstorectl/command/util.go \ - cmd/mstorectl/command/filecmd.go + cmd/mstorectl/command/util.go mstored_SOURCES = cmd/mstored/main.go EXTRA_mstored_SOURCES = cmd/mstored/starter/starter.go \ @@ -1069,7 +1068,7 @@ format: -name '*.go' | $(XARGS) -n1 dirname | $(SORT) | $(UNIQ)); \ for dir in $$dirs;do \ (echo "====$$dir===="; cd $$dir && $(GO) fmt .); \ - done + done .PHONY: test test: @@ -1104,7 +1103,7 @@ debian-package: $(DIST_ARCHIVES) cd $(BUILD_DIR)/$(distdir) && $(DBUILDPACKAGE) -nc -us -uc -ui -i -b mkdir -p $(DIST_DIR) $(CP) $(BUILD_DIR)/*.deb $(DIST_DIR) - rm -rf $(BUILD_DIR) + rm -rf $(BUILD_DIR) install-data-local: test -z $(DESTDIR)$(srv_confdir) || $(MKDIR_P) $(DESTDIR)$(srv_confdir) diff --git a/app/operator/file.go b/app/operator/file.go deleted file mode 100644 index 6e81dde..0000000 --- a/app/operator/file.go +++ /dev/null @@ -1,639 +0,0 @@ -/* - * Copyright 2026 Oleg Borodin - * - * This work is published and licensed under a Creative Commons - * Attribution-NonCommercial-NoDerivatives 4.0 International License. - * - * Distribution of this work is permitted, but commercial use and - * modifications are strictly prohibited. - */ -package operator - -import ( - "context" - "fmt" - "io" - "net/http" - "path" - "path/filepath" - "regexp" - "slices" - "strconv" - "strings" - - "mstore/pkg/auxtool" - "mstore/pkg/auxuuid" - "mstore/pkg/descr" - "mstore/pkg/filecli" -) - -// FileInfo -type FileInfoParams struct { - Filepath string - Source string - Dest string -} -type FileInfoResult struct { - ContentCollection string - ContentName string - ContentType string - ContentSize string - ContentDigest string - ContentCreatedAt string - ContentCreatedBy string - ContentUpdatedAt string - ContentUpdatedBy string -} - -func cleanFilepath(filename string) (string, error) { - filename = "/" + filename - return filepath.Clean(filename), nil -} - -func (oper *Operator) FileInfo(ctx context.Context, operatorID string, params *FileInfoParams) (int, *FileInfoResult, error) { - var err error - code := http.StatusOK - res := &FileInfoResult{} - - xfilepath, err := cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - filename := path.Base(xfilepath) - collection := path.Dir(xfilepath) - - resName := params.Filepath - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - exist, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - if !exist { - code = http.StatusNotFound - return code, res, err - } - res = &FileInfoResult{ - ContentCollection: fileDescr.Collection, - ContentName: fileDescr.Name, - ContentSize: strconv.FormatInt(fileDescr.Size, 10), - ContentType: fileDescr.Type, - ContentDigest: fileDescr.Checksum, - - ContentCreatedAt: fileDescr.CreatedAt, - ContentCreatedBy: fileDescr.CreatedBy, - ContentUpdatedAt: fileDescr.UpdatedAt, - ContentUpdatedBy: fileDescr.UpdatedBy, - } - return code, res, err -} - -// PutFile -type PutFileParams struct { - ContentType string - ContentSize string - Filepath string - Source io.ReadCloser -} -type PutFileResult struct{} - -const defaultContentType = "application/octet-stream" - -// TODO: checking catalog and file names conflict -func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *PutFileParams) (int, *PutFileResult, error) { - var err error - res := &PutFileResult{} - - if params.ContentSize == "" { - code := http.StatusLengthRequired - err = fmt.Errorf("Required Content-Size header is empty") - return code, res, err - } - size, err := strconv.ParseInt(params.ContentSize, 10, 64) - if err != nil { - code := http.StatusLengthRequired - return code, res, err - } - contentType := params.ContentType - if contentType == "" { - contentType = defaultContentType - } - - resName := params.Filepath - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - // TODO: convert file path to a unified and secure state - xfilepath, err := cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - filename := path.Base(xfilepath) - collection := path.Dir(xfilepath) - - tmpname, size, checksum, err := oper.store.WriteTempFile(params.Source) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - descrExists, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - now := auxtool.TimeNow() - if descrExists { - fileDescr.Size = size - fileDescr.Checksum = checksum - fileDescr.UpdatedAt = now - fileDescr.Type = contentType - fileDescr.UpdatedBy = operatorID - err = oper.mdb.UpdateFileByID(ctx, fileDescr.ID, fileDescr) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - } else { - fileDescr = &descr.File{ - ID: auxuuid.NewUUID(), - Name: filename, - Collection: collection, - Size: size, - Type: contentType, - Checksum: checksum, - CreatedAt: now, - UpdatedAt: now, - CreatedBy: operatorID, - UpdatedBy: operatorID, - } - err = oper.mdb.InsertFile(ctx, fileDescr) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - } - - err = oper.store.HardlinkFile(tmpname, collection, filename) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - code := http.StatusOK - return code, res, err -} - -// GetFile -type GetFileParams struct { - Filepath string -} -type GetFileResult struct { - ContentType string - ContentSize string - ContentDigest string - Source io.ReadCloser - - ContentCreatedAt string - ContentCreatedBy string - ContentUpdatedAt string - ContentUpdatedBy string -} - -func (oper *Operator) GetFile(ctx context.Context, operatorID string, params *GetFileParams) (int, *GetFileResult, error) { - var err error - res := &GetFileResult{} - - xfilepath, err := cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - filename := path.Base(xfilepath) - collection := path.Dir(xfilepath) - - resName := params.Filepath - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - descrExists, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - if !descrExists { - code := http.StatusNotFound - return code, res, err - } - reader, err := oper.store.GetFileReader(collection, filename) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res = &GetFileResult{ - ContentSize: strconv.FormatInt(fileDescr.Size, 10), - ContentType: fileDescr.Type, - ContentDigest: fileDescr.Checksum, - Source: reader, - - ContentCreatedAt: fileDescr.CreatedAt, - ContentCreatedBy: fileDescr.CreatedBy, - ContentUpdatedAt: fileDescr.UpdatedAt, - ContentUpdatedBy: fileDescr.UpdatedBy, - } - code := http.StatusOK - return code, res, err -} - -// DeleteFile -type DeleteFileParams struct { - Filepath string -} -type DeleteFileResult struct{} - -func (oper *Operator) DeleteFile(ctx context.Context, operatorID string, params *DeleteFileParams) (int, *DeleteFileResult, error) { - var err error - res := &DeleteFileResult{} - code := http.StatusOK - - xfilepath, err := cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - filename := path.Base(xfilepath) - collection := path.Dir(xfilepath) - - resName := params.Filepath - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - descrExists, _, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) - if err != nil { - code = http.StatusInternalServerError - return code, res, err - } - if !descrExists { - code := http.StatusNotFound - return code, res, err - } - - if descrExists { - err = oper.mdb.DeleteFileByCollectionName(ctx, collection, filename) - if err != nil { - code = http.StatusInternalServerError - return code, res, err - } - } - err = oper.store.DeleteFile(collection, filename) - if err != nil { - code = http.StatusInternalServerError - return code, res, err - } - return code, res, err -} - -// ListCollections -type ListCollectionsParams struct { - Path string - PathType string `param:"pathType"` -} -type ListCollectionsResult struct { - Collections []string `json:"collection,omitempty"` -} - -func (oper *Operator) ListCollections(ctx context.Context, operatorID string, param *ListCollectionsParams) (int, *ListCollectionsResult, error) { - var err error - res := &ListCollectionsResult{ - Collections: make([]string, 0), - } - - collectionList := make([]string, 0) - switch param.PathType { - case filecli.PathTypeRegexp: - collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - case filecli.PathTypePrefix: - param.Path, err = cleanFilepath(param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - collectionList, err = oper.listCollectionsWithPrefix(ctx, param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - default: - param.Path, err = cleanFilepath(param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - collectionList, err = oper.listAllCollections(ctx) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - } - res.Collections = collectionList - code := http.StatusOK - return code, res, err -} - -func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix string) ([]string, error) { - var err error - res := make([]string, 0) - - fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO - if err != nil { - return res, err - } - collMap := make(map[string]bool) - for _, item := range fileDescrs { - _, exists := collMap[item.Collection] - if !exists { - collMap[item.Collection] = true - } - } - res = make([]string, 0) - for key, _ := range collMap { - if strings.HasPrefix(key, prefix) { - res = append(res, key) - } - } - slices.Sort(res) - return res, err -} - -func (oper *Operator) listCollectionsWithRegexp(ctx context.Context, regex string) ([]string, error) { - var err error - res := make([]string, 0) - - re, err := regexp.Compile(regex) - if err != nil { - return res, err - } - - fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO - if err != nil { - return res, err - } - collMap := make(map[string]bool) - for _, item := range fileDescrs { - _, exists := collMap[item.Collection] - if !exists { - collMap[item.Collection] = true - } - } - res = make([]string, 0) - for key, _ := range collMap { - if re.MatchString(key) { - res = append(res, key) - } - } - slices.Sort(res) - return res, err -} - -func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) { - var err error - res := make([]string, 0) - - fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO - if err != nil { - return res, err - } - collMap := make(map[string]bool) - for _, item := range fileDescrs { - _, exists := collMap[item.Collection] - if !exists { - collMap[item.Collection] = true - } - } - res = make([]string, 0) - for key, _ := range collMap { - res = append(res, key) - } - slices.Sort(res) - return res, err -} - -// DeleteColletion -type DeleteColletionParams struct { - Path string - PathType string `param:"pathType"` - DryRun bool `param:"dryRun"` -} -type DeleteColletionResult struct { - Files []descr.File `json:"files,omitempty"` -} - -func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, param *DeleteColletionParams) (int, *DeleteColletionResult, error) { - var err error - res := &DeleteColletionResult{ - Files: make([]descr.File, 0), - } - switch param.PathType { - case filecli.PathTypeRegexp: - collections, err := oper.listCollectionsWithRegexp(ctx, param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - allfiles := make([]descr.File, 0) - for _, collection := range collections { - files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - allfiles = append(allfiles, files...) - } - res.Files = allfiles - - case filecli.PathTypePrefix: - param.Path, err = cleanFilepath(param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - collections, err := oper.listCollectionsWithPrefix(ctx, param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - allfiles := make([]descr.File, 0) - for _, collection := range collections { - files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - allfiles = append(allfiles, files...) - } - res.Files = allfiles - default: - param.Path, err = cleanFilepath(param.Path) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - collection := param.Path - files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - } - code := http.StatusOK - return code, res, err -} - -func (oper *Operator) deleteFilesInCollection(ctx context.Context, collection string, dryRun bool) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - files, err := oper.mdb.ListFilesByCollection(ctx, collection) - if err != nil { - return res, err - } - for _, file := range files { - oper.logg.Debugf("Delete file %s/%s", file.Collection, file.Name) - if !dryRun { - err = oper.store.DeleteFile(file.Collection, file.Name) - if err != nil { - oper.logg.Warningf("%v", err) - err = nil - } - err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) - if err != nil { - return res, err - } - } - - res = append(res, file) - } - return res, err -} - -// ListFiles -type ListFilesParams struct { - Filepath string - PathType string `param:"pathType"` -} -type ListFilesResult struct { - Files []descr.File `json:"files,omitempty"` -} - -func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) { - var err error - res := &ListFilesResult{} - switch params.PathType { - case filecli.PathTypeRegexp: - files, err := oper.listFilesWithRegex(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - case filecli.PathTypePrefix: - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - files, err := oper.listFilesWithPrefix(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - default: - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - files, err := oper.listFilesInCollection(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - - } - code := http.StatusOK - return code, res, err -} - -func (oper *Operator) listFilesInCollection(ctx context.Context, collection string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - files, err := oper.mdb.ListFilesByCollection(ctx, collection) - if err != nil { - return res, err - } - res = files - return res, err -} - -func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - files, err := oper.mdb.ListAllFiles(ctx) - if err != nil { - return res, err - } - for _, file := range files { - fullpath := filepath.Join(file.Collection, file.Name) - if strings.HasPrefix(fullpath, prefix) { - res = append(res, file) - } - } - return res, err -} - -func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - - re, err := regexp.Compile(regex) - if err != nil { - return res, err - } - files, err := oper.mdb.ListAllFiles(ctx) - if err != nil { - return res, err - } - for _, file := range files { - fullpath := filepath.Join(file.Collection, file.Name) - if re.MatchString(fullpath) { - res = append(res, file) - } - } - return res, err -} diff --git a/app/operator/fileop_delcats.go b/app/operator/fileop_delcats.go new file mode 100644 index 0000000..b2931fe --- /dev/null +++ b/app/operator/fileop_delcats.go @@ -0,0 +1,120 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "net/http" + + "mstore/pkg/descr" + "mstore/pkg/filecli" +) + + +// DeleteColletion +type DeleteColletionParams struct { + Path string + PathType string `param:"pathType"` + DryRun bool `param:"dryRun"` +} +type DeleteColletionResult struct { + Files []descr.File `json:"files,omitempty"` +} + +func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, param *DeleteColletionParams) (int, *DeleteColletionResult, error) { + var err error + res := &DeleteColletionResult{ + Files: make([]descr.File, 0), + } + switch param.PathType { + case filecli.PathTypeRegexp: + collections, err := oper.listCollectionsWithRegexp(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles := make([]descr.File, 0) + for _, collection := range collections { + files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles = append(allfiles, files...) + } + res.Files = allfiles + + case filecli.PathTypePrefix: + param.Path, err = cleanFilepath(param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + collections, err := oper.listCollectionsWithPrefix(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles := make([]descr.File, 0) + for _, collection := range collections { + files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + allfiles = append(allfiles, files...) + } + res.Files = allfiles + default: + param.Path, err = cleanFilepath(param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + collection := param.Path + files, err := oper.deleteFilesInCollection(ctx, collection, param.DryRun) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + } + code := http.StatusOK + return code, res, err +} + +func (oper *Operator) deleteFilesInCollection(ctx context.Context, collection string, dryRun bool) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + files, err := oper.mdb.ListFilesByCollection(ctx, collection) + if err != nil { + return res, err + } + for _, file := range files { + if !dryRun { + oper.logg.Debugf("Delete file %s/%s", file.Collection, file.Name) + err = oper.store.DeleteFile(file.Collection, file.Name) + if err != nil { + oper.logg.Warningf("%v", err) + err = nil + } + err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) + if err != nil { + return res, err + } + } + + res = append(res, file) + } + return res, err +} + + diff --git a/app/operator/fileop_delfile.go b/app/operator/fileop_delfile.go new file mode 100644 index 0000000..a317300 --- /dev/null +++ b/app/operator/fileop_delfile.go @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "net/http" + "path" +) + + +// DeleteFile +type DeleteFileParams struct { + Filepath string +} +type DeleteFileResult struct{} + +func (oper *Operator) DeleteFile(ctx context.Context, operatorID string, params *DeleteFileParams) (int, *DeleteFileResult, error) { + var err error + res := &DeleteFileResult{} + code := http.StatusOK + + xfilepath, err := cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + filename := path.Base(xfilepath) + collection := path.Dir(xfilepath) + + resName := params.Filepath + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + descrExists, _, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + if !descrExists { + code := http.StatusNotFound + return code, res, err + } + + if descrExists { + err = oper.mdb.DeleteFileByCollectionName(ctx, collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + } + err = oper.store.DeleteFile(collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + return code, res, err +} diff --git a/app/operator/fileop_fileinfo.go b/app/operator/fileop_fileinfo.go new file mode 100644 index 0000000..9421d25 --- /dev/null +++ b/app/operator/fileop_fileinfo.go @@ -0,0 +1,84 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "net/http" + "path" + "path/filepath" + "strconv" +) + +// FileInfo +type FileInfoParams struct { + Filepath string + Source string + Dest string +} +type FileInfoResult struct { + ContentCollection string + ContentName string + ContentType string + ContentSize string + ContentDigest string + ContentCreatedAt string + ContentCreatedBy string + ContentUpdatedAt string + ContentUpdatedBy string +} + +func cleanFilepath(filename string) (string, error) { + filename = "/" + filename + return filepath.Clean(filename), nil +} + +func (oper *Operator) FileInfo(ctx context.Context, operatorID string, params *FileInfoParams) (int, *FileInfoResult, error) { + var err error + code := http.StatusOK + res := &FileInfoResult{} + + xfilepath, err := cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + filename := path.Base(xfilepath) + collection := path.Dir(xfilepath) + + resName := params.Filepath + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + exist, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + if !exist { + code = http.StatusNotFound + return code, res, err + } + res = &FileInfoResult{ + ContentCollection: fileDescr.Collection, + ContentName: fileDescr.Name, + ContentSize: strconv.FormatInt(fileDescr.Size, 10), + ContentType: fileDescr.Type, + ContentDigest: fileDescr.Checksum, + + ContentCreatedAt: fileDescr.CreatedAt, + ContentCreatedBy: fileDescr.CreatedBy, + ContentUpdatedAt: fileDescr.UpdatedAt, + ContentUpdatedBy: fileDescr.UpdatedBy, + } + return code, res, err +} + diff --git a/app/operator/fileop_getfile.go b/app/operator/fileop_getfile.go new file mode 100644 index 0000000..18ed602 --- /dev/null +++ b/app/operator/fileop_getfile.go @@ -0,0 +1,80 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "io" + "net/http" + "path" + "strconv" +) + +// GetFile +type GetFileParams struct { + Filepath string +} +type GetFileResult struct { + ContentType string + ContentSize string + ContentDigest string + Source io.ReadCloser + + ContentCreatedAt string + ContentCreatedBy string + ContentUpdatedAt string + ContentUpdatedBy string +} + +func (oper *Operator) GetFile(ctx context.Context, operatorID string, params *GetFileParams) (int, *GetFileResult, error) { + var err error + res := &GetFileResult{} + + xfilepath, err := cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + filename := path.Base(xfilepath) + collection := path.Dir(xfilepath) + + resName := params.Filepath + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + descrExists, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + if !descrExists { + code := http.StatusNotFound + return code, res, err + } + reader, err := oper.store.GetFileReader(collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res = &GetFileResult{ + ContentSize: strconv.FormatInt(fileDescr.Size, 10), + ContentType: fileDescr.Type, + ContentDigest: fileDescr.Checksum, + Source: reader, + + ContentCreatedAt: fileDescr.CreatedAt, + ContentCreatedBy: fileDescr.CreatedBy, + ContentUpdatedAt: fileDescr.UpdatedAt, + ContentUpdatedBy: fileDescr.UpdatedBy, + } + code := http.StatusOK + return code, res, err +} + diff --git a/app/operator/fileop_listcats.go b/app/operator/fileop_listcats.go new file mode 100644 index 0000000..14831c8 --- /dev/null +++ b/app/operator/fileop_listcats.go @@ -0,0 +1,153 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "net/http" + "regexp" + "slices" + "strings" + + "mstore/pkg/filecli" +) + +// ListCollections +type ListCollectionsParams struct { + Path string + PathType string `param:"pathType"` +} +type ListCollectionsResult struct { + Collections []string `json:"collection,omitempty"` +} + +func (oper *Operator) ListCollections(ctx context.Context, operatorID string, param *ListCollectionsParams) (int, *ListCollectionsResult, error) { + var err error + res := &ListCollectionsResult{ + Collections: make([]string, 0), + } + + collectionList := make([]string, 0) + switch param.PathType { + case filecli.PathTypeRegexp: + collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + case filecli.PathTypePrefix: + param.Path, err = cleanFilepath(param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + collectionList, err = oper.listCollectionsWithPrefix(ctx, param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + default: + param.Path, err = cleanFilepath(param.Path) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + collectionList, err = oper.listAllCollections(ctx) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + } + res.Collections = collectionList + code := http.StatusOK + return code, res, err +} + +func (oper *Operator) listCollectionsWithPrefix(ctx context.Context, prefix string) ([]string, error) { + var err error + res := make([]string, 0) + + fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO + if err != nil { + return res, err + } + collMap := make(map[string]bool) + for _, item := range fileDescrs { + _, exists := collMap[item.Collection] + if !exists { + collMap[item.Collection] = true + } + } + res = make([]string, 0) + for key, _ := range collMap { + if strings.HasPrefix(key, prefix) { + res = append(res, key) + } + } + slices.Sort(res) + return res, err +} + +func (oper *Operator) listCollectionsWithRegexp(ctx context.Context, regex string) ([]string, error) { + var err error + res := make([]string, 0) + + re, err := regexp.Compile(regex) + if err != nil { + return res, err + } + + fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO + if err != nil { + return res, err + } + collMap := make(map[string]bool) + for _, item := range fileDescrs { + _, exists := collMap[item.Collection] + if !exists { + collMap[item.Collection] = true + } + } + res = make([]string, 0) + for key, _ := range collMap { + if re.MatchString(key) { + res = append(res, key) + } + } + slices.Sort(res) + return res, err +} + +func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) { + var err error + res := make([]string, 0) + + fileDescrs, err := oper.mdb.ListAllFiles(ctx) // TODO + if err != nil { + return res, err + } + collMap := make(map[string]bool) + for _, item := range fileDescrs { + _, exists := collMap[item.Collection] + if !exists { + collMap[item.Collection] = true + } + } + res = make([]string, 0) + for key, _ := range collMap { + res = append(res, key) + } + slices.Sort(res) + return res, err +} + diff --git a/app/operator/fileop_listfiles.go b/app/operator/fileop_listfiles.go new file mode 100644 index 0000000..e0f28ae --- /dev/null +++ b/app/operator/fileop_listfiles.go @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "net/http" + "path/filepath" + "regexp" + "strings" + + "mstore/pkg/descr" + "mstore/pkg/filecli" +) + +// ListFiles +type ListFilesParams struct { + Filepath string + PathType string `param:"pathType"` +} +type ListFilesResult struct { + Files []descr.File `json:"files,omitempty"` +} + +func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) { + var err error + res := &ListFilesResult{} + switch params.PathType { + case filecli.PathTypeRegexp: + files, err := oper.listFilesWithRegex(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + case filecli.PathTypePrefix: + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + files, err := oper.listFilesWithPrefix(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + default: + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + files, err := oper.listFilesInCollection(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + + } + code := http.StatusOK + return code, res, err +} + +func (oper *Operator) listFilesInCollection(ctx context.Context, collection string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + files, err := oper.mdb.ListFilesByCollection(ctx, collection) + if err != nil { + return res, err + } + res = files + return res, err +} + +func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + files, err := oper.mdb.ListAllFiles(ctx) + if err != nil { + return res, err + } + for _, file := range files { + fullpath := filepath.Join(file.Collection, file.Name) + if strings.HasPrefix(fullpath, prefix) { + res = append(res, file) + } + } + return res, err +} + +func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + + re, err := regexp.Compile(regex) + if err != nil { + return res, err + } + files, err := oper.mdb.ListAllFiles(ctx) + if err != nil { + return res, err + } + for _, file := range files { + fullpath := filepath.Join(file.Collection, file.Name) + if re.MatchString(fullpath) { + res = append(res, file) + } + } + return res, err +} diff --git a/app/operator/fileop_putfile.go b/app/operator/fileop_putfile.go new file mode 100644 index 0000000..552e29c --- /dev/null +++ b/app/operator/fileop_putfile.go @@ -0,0 +1,121 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package operator + +import ( + "context" + "fmt" + "io" + "net/http" + "path" + "strconv" + + "mstore/pkg/auxtool" + "mstore/pkg/auxuuid" + "mstore/pkg/descr" +) + +// PutFile +type PutFileParams struct { + ContentType string + ContentSize string + Filepath string + Source io.ReadCloser +} +type PutFileResult struct{} + +const defaultContentType = "application/octet-stream" + +// TODO: checking catalog and file names conflict +func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *PutFileParams) (int, *PutFileResult, error) { + var err error + res := &PutFileResult{} + + if params.ContentSize == "" { + code := http.StatusLengthRequired + err = fmt.Errorf("Required Content-Size header is empty") + return code, res, err + } + size, err := strconv.ParseInt(params.ContentSize, 10, 64) + if err != nil { + code := http.StatusLengthRequired + return code, res, err + } + contentType := params.ContentType + if contentType == "" { + contentType = defaultContentType + } + + resName := params.Filepath + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + // TODO: convert file path to a unified and secure state + xfilepath, err := cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + filename := path.Base(xfilepath) + collection := path.Dir(xfilepath) + + tmpname, size, checksum, err := oper.store.WriteTempFile(params.Source) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + descrExists, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + now := auxtool.TimeNow() + if descrExists { + fileDescr.Size = size + fileDescr.Checksum = checksum + fileDescr.UpdatedAt = now + fileDescr.Type = contentType + fileDescr.UpdatedBy = operatorID + err = oper.mdb.UpdateFileByID(ctx, fileDescr.ID, fileDescr) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } else { + fileDescr = &descr.File{ + ID: auxuuid.NewUUID(), + Name: filename, + Collection: collection, + Size: size, + Type: contentType, + Checksum: checksum, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: operatorID, + UpdatedBy: operatorID, + } + err = oper.mdb.InsertFile(ctx, fileDescr) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + + err = oper.store.HardlinkFile(tmpname, collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + code := http.StatusOK + return code, res, err +} + diff --git a/app/storage/storage.go b/app/storage/storage.go index e4f36d6..52c0de3 100644 --- a/app/storage/storage.go +++ b/app/storage/storage.go @@ -124,7 +124,7 @@ func (store *Storage) HardlinkFile(tmpname, collection, filename string) error { } filename = store.makeFilepath(collection, filename) - os.Remove(filename) // TODO + os.Remove(filename) // TODO: safe removing tmpname = store.makeTmppath(tmpname) err = os.Link(tmpname, filename) @@ -145,12 +145,9 @@ func (store *Storage) DeleteFile(collection, filename string) error { if err != nil { return err } - // TODO: safe dir removing + // TODO: clean removing dirname := store.makeCollecionpath(collection) - err = os.RemoveAll(dirname) - if err != nil { - return err - } + os.RemoveAll(dirname) return err } diff --git a/cmd/mstorectl/command/filecmd.go b/cmd/mstorectl/command/filecmd.go deleted file mode 100644 index ade6315..0000000 --- a/cmd/mstorectl/command/filecmd.go +++ /dev/null @@ -1,586 +0,0 @@ -/* - * Copyright 2026 Oleg Borodin - * - * This work is published and licensed under a Creative Commons - * Attribution-NonCommercial-NoDerivatives 4.0 International License. - * - * Distribution of this work is permitted, but commercial use and - * modifications are strictly prohibited. - */ -package command - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "net/url" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/viper" - - "mstore/pkg/filecli" - "mstore/pkg/descr" -) - -func (util *FileUtil) MakeFileCmds() *cobra.Command { - var subCmd = &cobra.Command{ - Use: "files", - Short: "File operations", - Aliases: []string{"file"}, - } - const defaultTimeout uint64 = 10 - - subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username") - subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password") - subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout") - subCmd.PersistentFlags().BoolVarP(&util.commonFileParams.SkipTLSVerify, "skipVerify", "S", true, "Skip server certificate verify") - subCmd.MarkPersistentFlagRequired("host") - subCmd.MarkFlagsRequiredTogether("user", "pass") - - vi := viper.New() - vi.SetEnvPrefix("mstore") - vi.BindEnv("user") - vi.BindEnv("pass") - util.commonFileParams.Username = vi.GetString("user") - util.commonFileParams.Password = vi.GetString("pass") - - // PutFile - var putFileCmd = &cobra.Command{ - Use: "put filepath [user:pass@]hostname[:port]/collection/name", - Args: cobra.ExactArgs(2), - Aliases: []string{"push"}, - Short: "Put file to storage", - Run: util.PutFile, - } - subCmd.AddCommand(putFileCmd) - - // GetFile - var getFileCmd = &cobra.Command{ - Use: "get [user:pass@]hostname[:port]/collection/name filepath", - Aliases: []string{"pull"}, - Args: cobra.ExactArgs(2), - Short: "Get file from storage", - Run: util.GetFile, - } - subCmd.AddCommand(getFileCmd) - - // FileInfo - var fileInfoCmd = &cobra.Command{ - Use: "info [user:pass@]hostname[:port]/collection/name", - Args: cobra.ExactArgs(1), - Short: "Show file information", - Run: util.FileInfo, - } - subCmd.AddCommand(fileInfoCmd) - - // DeleteFile - var deleteFileCmd = &cobra.Command{ - Use: "delete [user:pass@]hostname[:port]/collection/name", - Args: cobra.ExactArgs(1), - Aliases: []string{"remove"}, - Short: "Delete file in storage", - Run: util.DeleteFile, - } - subCmd.AddCommand(deleteFileCmd) - - // ListFiles - var listFilesCmd = &cobra.Command{ - Use: "list [user:pass@]hostname[:port]/catalog", - Args: cobra.ExactArgs(1), - Short: "List files in storage", - Run: util.ListFiles, - } - listFilesCmd.Flags().BoolVarP(&util.listFilesParams.Detail, "detail", "D", false, "Show detail file information") - listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") - listFilesCmd.Flags().BoolVarP(&util.listFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") - subCmd.AddCommand(listFilesCmd) - - // ImportFiles - var importFilesCmd = &cobra.Command{ - Use: "import directory [user:pass@]hostname[:port]/collection", - Args: cobra.ExactArgs(2), - Short: "Send file tree to storage as is", - Run: util.ImportFiles, - } - subCmd.AddCommand(importFilesCmd) - - // ExportFiles - var exportFilesCmd = &cobra.Command{ - Use: "export directory [user:pass@]hostname[:port]/collection dir", - Args: cobra.ExactArgs(2), - Short: "Download file tree to storage as is", - Run: util.ExportFiles, - } - exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.Detail, "detail", "D", false, "Show detail file information") - exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") - exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") - subCmd.AddCommand(exportFilesCmd) - - return subCmd -} - -func (util *FileUtil) MakeCollectionCmds() *cobra.Command { - var subCmd = &cobra.Command{ - Use: "collections", - Short: "Colletion operations", - Aliases: []string{"col", "cols", "dir", "dirs"}, - } - const defaultTimeout uint64 = 10 - - subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Username, "user", "u", "", "Username") - subCmd.PersistentFlags().StringVarP(&util.commonFileParams.Password, "pass", "p", "", "Password") - subCmd.PersistentFlags().Uint64VarP(&util.commonFileParams.Timeout, "timeout", "t", defaultTimeout, "Operation timeout") - subCmd.MarkPersistentFlagRequired("host") - subCmd.MarkFlagsRequiredTogether("user", "pass") - - // ListCollections - var listCollectionsCmd = &cobra.Command{ - Use: "list [user:pass@]hostname[:port]/catalog", - Args: cobra.ExactArgs(1), - Short: "List collections in storage", - Run: util.ListCollections, - } - listCollectionsCmd.Flags().BoolVarP(&util.listCollectionsParams.AsPrefix, "asprefix", "P", true, "Use path as collection path prefix") - subCmd.AddCommand(listCollectionsCmd) - - // DeleteCollection - var deleteCollectionCmd = &cobra.Command{ - Use: "delete [user:pass@]hostname[:port]/catalog", - Args: cobra.ExactArgs(1), - Short: "Delete all files in collection", - Run: util.DeleteCollection, - } - deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.Detail, "detail", "D", false, "Show detail file information") - deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsPrefix, "asprefix", "P", false, "Use path as collection path prefix") - deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.AsRegexp, "asregex", "R", false, "Use path as collection path prefix") - deleteCollectionCmd.Flags().BoolVarP(&util.deleteCollectionParams.DryRun, "dryrun", "Y", false, "Simulate process, don't delete files") - - subCmd.AddCommand(deleteCollectionCmd) - - return subCmd -} - -type FileUtil struct { - fileInfoParams FileInfoParams - putFileParams PutFileParams - getFileParams GetFileParams - deleteFileParams DeleteFileParams - listFilesParams ListFilesParams - importFilesParams ImportFilesParams - exportFilesParams ExportFilesParams - deleteCollectionParams DeleteCollectionParams - listCollectionsParams ListCollectionsParams - commonFileParams CommonFileParams -} - -type CommonFileParams struct { - Username string - Password string - Timeout uint64 - SkipTLSVerify bool -} - -// PutFile -type PutFileParams struct { - Source string - Dest string -} -type PutFileResult struct{} - -func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) { - util.putFileParams.Source = args[0] - util.putFileParams.Dest = args[1] - res, err := util.putFile(&util.commonFileParams, &util.putFileParams) - printResponse(res, err) -} - -func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) { - var err error - res := &PutFileResult{} - - file, err := os.OpenFile(params.Dest, os.O_RDONLY, 0) - if err != nil { - return res, err - } - defer file.Close() - stat, err := file.Stat() - if err != nil { - return res, err - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref, err := filecli.ParsePath(params.Dest) - if err != nil { - return res, err - } - ref.SetUserinfo(common.Username, common.Password) - - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) - if err != nil { - return res, err - } - return res, err -} - -// Get file -type GetFileParams struct { - Source string - Dest string -} - -func (util *FileUtil) GetFile(cmd *cobra.Command, args []string) { - util.getFileParams.Source = args[0] - util.getFileParams.Dest = args[1] - res, err := util.getFile(&util.commonFileParams, &util.getFileParams) - printResponse(res, err) -} - -type GetFileResult struct{} - -func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) { - var err error - res := &GetFileResult{} - - err = os.MkdirAll(filepath.Dir(params.Dest), 0750) - if err != nil { - return res, err - } - file, err := os.OpenFile(params.Dest, os.O_WRONLY|os.O_CREATE, 0640) - if err != nil { - return res, err - } - defer file.Close() - - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref, err := filecli.ParsePath(params.Source) - if err != nil { - return res, err - } - ref.SetUserinfo(common.Username, common.Password) - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - _, err = cli.GetFile(ctx, ref.Raw(), file) - if err != nil { - return res, err - } - return res, err -} - -// DeleteFile -type DeleteFileParams struct { - Filepath string -} - -type DeleteFileResult struct{} - -func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) { - util.deleteFileParams.Filepath = args[0] - res, err := util.deleteFile(&util.commonFileParams, &util.deleteFileParams) - printResponse(res, err) -} -func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) { - var err error - res := &DeleteFileResult{} - ref, err := filecli.ParsePath(params.Filepath) - if err != nil { - return res, err - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref.SetUserinfo(common.Username, common.Password) - - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - _, err = cli.DeleteFile(ctx, ref.Raw()) - if err != nil { - return res, err - } - return res, err -} - -// ImportFiles -type ImportFilesParams struct { - Source string - Dest string - Progress bool -} -type ImportFilesResult struct { - Files []string `yaml:"files,omitempty"` -} - -func (util *FileUtil) ImportFiles(cmd *cobra.Command, args []string) { - util.importFilesParams.Source = args[0] - util.importFilesParams.Dest = args[1] - - res, err := util.importFiles(&util.commonFileParams, &util.importFilesParams) - printResponse(res, err) -} - -func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesParams) (*ImportFilesResult, error) { - var err error - res := &ImportFilesResult{ - Files: make([]string, 0), - } - params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password) - if err != nil { - return res, err - } - - putErrors := make([]error, 0) - walcFunc := func(walkPath string, infoItem fs.FileInfo, err error) error { - if err != nil { - return err - } - mode := infoItem.Mode() - skip := mode == fs.ModeDevice - skip = skip || mode == fs.ModeNamedPipe - skip = skip || mode == fs.ModeCharDevice - if skip { - return nil - } - if !infoItem.IsDir() { - relPath, _ := strings.CutPrefix(walkPath, params.Source) - dest, _ := url.JoinPath(params.Dest, relPath) - ref, err := filecli.ParsePath(dest) - if err != nil { - putErrors = append(putErrors, err) - return nil - } - file, err := os.OpenFile(dest, os.O_RDONLY, 0) - if err != nil { - putErrors = append(putErrors, err) - return nil - } - defer file.Close() - stat, err := file.Stat() - if err != nil { - putErrors = append(putErrors, err) - return nil - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref.SetUserinfo(common.Username, common.Password) - user, pass := ref.Userinfo() - mw := filecli.NewBasicAuthMiddleware(user, pass) - cli := filecli.NewClient(nil, mw) - err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) - if err != nil { - putErrors = append(putErrors, err) - fmt.Printf("- %s: error: %v \n", walkPath, err) - } else { - res.Files = append(res.Files, walkPath) - fmt.Printf("- %s: ok\n", walkPath) - } - } - return nil - } - for _, item := range putErrors { - err = errors.Join(err, item) - } - err = filepath.Walk(params.Source, walcFunc) - if err != nil { - return res, err - } - return res, err -} - -// ExportFiles -type ExportFilesParams struct { - Filepath string - Detail bool - AsPrefix bool - AsRegexp bool - Dest string -} - -type ExportFilesResult struct { - Files []descr.File `yaml:"files,omitempty"` - Filenames []string `yaml:"filenames,omitempty"` -} - -func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) { - util.exportFilesParams.Filepath = args[0] - util.exportFilesParams.Dest = args[1] - res, err := util.exportFiles(&util.commonFileParams, &util.exportFilesParams) - printResponse(res, err) -} -func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) { - var err error - res := &ExportFilesResult{} - - - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref, err := filecli.ParsePath(params.Filepath) - if err != nil { - return res, err - } - ref.SetUserinfo(common.Username, common.Password) - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - list, err := cli.ListFiles(ctx, ref.Raw()) - if err != nil { - return res, err - } - files := make([]descr.File, 0) - err = json.Unmarshal(list, &files) - if err != nil { - return res, err - } - - exported := make([]descr.File, 0) - for _, descr := range files { - destdir := filepath.Join(params.Dest, descr.Collection) - err = os.MkdirAll(destdir, 0750) - if err != nil { - return res, err - } - destpath := filepath.Join(params.Dest, descr.Collection, descr.Name) - destfile, err := os.OpenFile(destpath, os.O_WRONLY|os.O_CREATE, 0640) - if err != nil { - return res, err - } - defer destfile.Close() - - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref.SetResource(filepath.Join(descr.Collection, descr.Name)) - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - _, err = cli.GetFile(ctx, ref.Raw(), destfile) - if err != nil { - - fmt.Printf("- %s: error %v\n", ref.Raw(), err) - err = nil - } else { - fmt.Printf("- %s: ok\n", ref.Raw()) - exported = append(exported, descr) - } - } - if params.Detail { - res.Files = exported - } else { - files := descr.NewFiles() - files.Set(exported) - res.Filenames = files.List() - } - return res, err -} - -func makeFileURI(hosturi, collection, name string) (string, error) { - var err error - var res string - uri, err := url.Parse(hosturi) - if err != nil { - return res, err - } - uri.Path = path.Join(collection, name) - res = uri.String() - return res, err -} - -// ListCollections -type ListCollectionsParams struct { - Path string - AsPrefix bool - AsRegexp bool -} - -type ListCollectionsResult struct { - Collections []string `yaml:"collections,omitempty"` -} - -func (util *FileUtil) ListCollections(cmd *cobra.Command, args []string) { - util.listCollectionsParams.Path = args[0] - res, err := util.listCollections(&util.commonFileParams, &util.listCollectionsParams) - printResponse(res, err) -} - -func (util *FileUtil) listCollections(common *CommonFileParams, params *ListCollectionsParams) (*ListCollectionsResult, error) { - var err error - res := &ListCollectionsResult{ - Collections: make([]string, 0), - } - - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref, err := filecli.ParsePath(params.Path) - if err != nil { - return res, err - } - ref.SetUserinfo(common.Username, common.Password) - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - list, err := cli.ListCollections(ctx, ref.Raw()) - if err != nil { - return res, err - } - collections := make([]string, 0) - err = json.Unmarshal(list, collections) - if err != nil { - return res, err - } - res.Collections = collections - return res, err -} - -// DeleteCollection -type DeleteCollectionParams struct { - Path string - Detail bool - AsPrefix bool - AsRegexp bool - DryRun bool -} - -type DeleteCollectionResult struct { - Files []descr.File `yaml:"files,omitempty"` - Filenames []string `yaml:"filenames,omitempty"` -} - -func (util *FileUtil) DeleteCollection(cmd *cobra.Command, args []string) { - util.deleteCollectionParams.Path = args[0] - res, err := util.deleteCollection(&util.commonFileParams, &util.deleteCollectionParams) - printResponse(res, err) -} - -func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteCollectionParams) (*DeleteCollectionResult, error) { - var err error - res := &DeleteCollectionResult{ - Filenames: make([]string, 0), - } - - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - ref, err := filecli.ParsePath(params.Path) - if err != nil { - return res, err - } - ref.SetUserinfo(common.Username, common.Password) - mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) - cli := filecli.NewClient(nil, mw) - list, err := cli.DeleteCollection(ctx, ref.Raw()) - - files := descr.NewFiles() - err = json.Unmarshal(list, files.ArrayPtr()) - if err != nil { - return res, err - } - if !params.Detail { - res.Filenames = files.List() - } else { - res.Files = files.Array() - } - return res, err -} - diff --git a/pkg/descr/file.go b/pkg/descr/file.go index 9125709..b8498ef 100644 --- a/pkg/descr/file.go +++ b/pkg/descr/file.go @@ -48,17 +48,17 @@ func (fi *Files) Set(files []File) { fi.files = files } -func (fi Files) ArrayPtr() *[]File { +func (fi *Files) ArrayPtr() *[]File { return &fi.files } -func (fi Files) Array() []File { +func (fi *Files) Array() []File { return fi.files } -func (fi Files) List() []string { +func (fi *Files) List() []string { list := make([]string, 0) for _, file := range fi.files { list = append(list, path.Join(file.Collection, file.Name)) diff --git a/pkg/filecli/repo.go b/pkg/filecli/repo.go index 3c75b6f..44c6654 100644 --- a/pkg/filecli/repo.go +++ b/pkg/filecli/repo.go @@ -4,6 +4,7 @@ import ( "net/url" "path" "strings" + "strconv" ) const ( @@ -35,7 +36,7 @@ func ParsePath(rawpath string) (*Repository, error) { repo.pass, _ = urlobj.User.Password() urlobj.User = nil } - repo.resource = urlobj.Path + repo.resource = path.Join("/", urlobj.Path) urlobj.Path = "/" repo.urlobj = urlobj repo.values = urlobj.Query() @@ -52,30 +53,38 @@ func (repo *Repository) Raw() string { } func (repo *Repository) SetResource(resource string) { - repo.resource = resource + repo.resource = path.Join("/", resource) +} + +func (repo *Repository) JoinResource(resource string) { + repo.resource = path.Join("/", repo.resource, resource) } func (repo *Repository) PathType(typ string) { repo.values.Set("pathType", typ) } +func (repo *Repository) DryRun(yesno bool) { + repo.values.Set("dryRun", strconv.FormatBool(yesno)) +} + func (repo *Repository) File() string { - curl := repo.urlobj.JoinPath("/v3/api/file", repo.resource) + curl := repo.urlobj.JoinPath("/v3/api/file/", repo.resource) return curl.String() } func (repo *Repository) Files() string { - curl := repo.urlobj.JoinPath("/v3/api/files", repo.resource) + curl := repo.urlobj.JoinPath("/v3/api/files/", repo.resource) return curl.String() } func (repo *Repository) Collection() string { - curl := repo.urlobj.JoinPath("/v3/api/collection", repo.resource) + curl := repo.urlobj.JoinPath("/v3/api/collection/", repo.resource) return curl.String() } func (repo *Repository) Collections() string { - curl := repo.urlobj.JoinPath("/v3/api/collections", repo.resource) + curl := repo.urlobj.JoinPath("/v3/api/collections/", repo.resource) return curl.String() }