ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

added file checker, completation/size/digest

Browse Source
This commit is contained in:
Олег Бородин
2026-03-30 13:20:13 +02:00
parent 8afe71d925
commit f5227bcac9
8 changed files with 291 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fileoper/check.go
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package fileoper
import (
"context"
"net/http"
"mstore/pkg/descr"
)
// Check files
type CheckFilesParams struct {
Path string
PathType string `param:"pathType"`
}
type CheckFilesResult struct {
Files []descr.File `json:"files,omitempty"`
}
func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params *CheckFilesParams) (int, *CheckFilesResult, error) {
var code int
res := &CheckFilesResult{}
var err error
// Check existing and size
files, err := oper.listFiles(ctx, params.PathType, params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, file := range files {
exists, size, err := oper.store.FileExists(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
if !exists {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
if size != file.Size {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
err = oper.store.DeleteFile(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
// Check hashs
files, err = oper.listFiles(ctx, params.PathType, params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, file := range files {
sum, err := oper.store.GetFileCheksum(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
if sum != file.Checksum {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
err = oper.store.DeleteFile(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
return code, res, err
}
app/fileoper/listfiles.go
+43 -39
View File
@@ -26,51 +26,13 @@ type ListFilesResult struct {
func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) {
var err error
res := &ListFilesResult{}
switch params.PathType {
case filecli.PathTypeRegexp:
files, err := oper.listFilesWithRegex(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
case filecli.PathTypePrefix:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesWithPrefix(ctx, params.Filepath)
files, err := oper.listFiles(ctx, params.PathType, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
default:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesInCollection(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
}
code := http.StatusOK
return code, res, err
}
@@ -122,3 +84,45 @@ func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]d
}
return res, err
}
func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string) ([]descr.File, error) {
res := make([]descr.File, 0)
var err error
switch pathType {
case filecli.PathTypeRegexp:
files, err := oper.listFilesWithRegex(ctx, filepath)
if err != nil {
return res, err
}
res = files
case filecli.PathTypePrefix:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
files, err := oper.listFilesWithPrefix(ctx, filepath)
if err != nil {
return res, err
}
res = files
default:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
files, err := oper.listFilesInCollection(ctx, filepath)
if err != nil {
return res, err
}
res = files
}
return res, err
}
app/handler/file.go
+38
View File
@@ -254,6 +254,44 @@ func (hand *Handler) ListFiles(rctx *router.Context) {
rctx.SendJSON(code, res.Files)
}
func (hand *Handler) CheckFiles(rctx *router.Context) {
filepath, _ := rctx.GetSubpath("path")
if filepath == "" {
filepath = "/"
}
params := &fileoper.CheckFilesParams{
Path: filepath,
}
err := rctx.BindQuery(params)
if err != nil {
hand.logg.Errorf("CheckFiles binding error: %v", err)
rctx.SetStatus(http.StatusInternalServerError)
return
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteFiles, params.Path)
if err != nil {
rctx.SetStatus(http.StatusInternalServerError)
return
}
if !opEnable {
rctx.SetStatus(http.StatusMethodNotAllowed)
return
}
// Execution of the operation
ctx := rctx.GetContext()
code, res, err := hand.fiop.CheckFiles(ctx, operatorID, params)
if err != nil {
hand.logg.Errorf("CheckFiles error: %v", err)
rctx.SetStatus(code)
return
}
rctx.SendJSON(code, res.Files)
}
// GetProperty godoc
//
// @Summary List collections
app/service/service.go
+3
View File
@@ -60,6 +60,9 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections)
svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections)
app/storage/file.go
+38
View File
@@ -38,6 +38,24 @@ func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) FileExists(collection, filename string) (bool, int64, error) {
var err error
var exists bool
var size int64
filename = store.makeFilepath(collection, filename)
stat, err := os.Stat(filename)
if os.IsNotExist(err) {
err = nil
return exists, size, err
}
if err != nil {
return exists, size, err
}
size = stat.Size()
exists = true
return exists, size, err
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
@@ -51,6 +69,26 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser,
return res, err
}
func (store *Storage) GetFileCheksum(collection, filename string) (string, error) {
var err error
var res string
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
hasher := NewHasher()
_, err = io.Copy(hasher.Writer(), file)
if err != nil {
return res, err
}
res = hasher.Hex()
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
cmd/mstorectl/filecmd/filecmd.go
+12
View File
@@ -21,6 +21,7 @@ type FileUtil struct {
exportFilesParams ExportFilesParams
deleteCollectionParams DeleteCollectionParams
listCollectionsParams ListCollectionsParams
checkFilesParams CheckFilesParams
commonFileParams CommonFileParams
}
@@ -125,6 +126,17 @@ func (util *FileUtil) MakeFileCmds() *cobra.Command {
exportFilesCmd.Flags().BoolVarP(&util.exportFilesParams.Regexp, "regex", "R", false, "Use path as collection path prefix")
subCmd.AddCommand(exportFilesCmd)
// CkeckFiles
var checkFilesCmd = &cobra.Command{
Use: "check [user:pass@]hostname[:port]/catalog",
Args: cobra.ExactArgs(1),
Short: "Ckeck files in storage",
Run: util.ListFiles,
}
checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Detail, "detail", "D", false, "Show detail file information")
checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Prefix, "prefix", "P", true, "Use path as collection path prefix")
checkFilesCmd.Flags().BoolVarP(&util.checkFilesParams.Regexp, "regex", "R", false, "Use path as collection path prefix")
subCmd.AddCommand(checkFilesCmd)
return subCmd
}
pkg/filecli/checkfiles.go
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package filecli
import (
"bytes"
"context"
"fmt"
"net/http"
"strconv"
)
func (cli *Client) CheckFiles(ctx context.Context, rawpath string) ([]byte, error) {
var err error
var list []byte
ref, err := ParsePath(rawpath)
if err != nil {
return list, err
}
uri := ref.FilesEP()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return list, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return list, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return list, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unexpected response code %s", resp.Status)
return list, err
}
contentLength := resp.Header.Get("Content-Length")
if contentLength == "" {
err := fmt.Errorf("Content-Length header is missing")
return list, err
}
blobSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return list, err
}
buffer := bytes.NewBuffer(nil)
recSize, err := Copy(ctx, buffer, resp.Body)
if blobSize != recSize {
err := fmt.Errorf("Mismatch declared and actual body size: %d and %d", blobSize, recSize)
return list, err
}
list = buffer.Bytes()
return list, err
}
pkg/filecli/referer.go
+5
View File
@@ -91,6 +91,11 @@ func (ref *Referer) CollectionsEP() string {
return curl.String()
}
func (ref *Referer) CheckEP() string {
curl := ref.urlobj.JoinPath("/v3/api/checker/", ref.resource)
return curl.String()
}
func (ref *Referer) Userinfo() (string, string) {
return ref.user, ref.pass
}