ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

added file checker, completation/size/digest

Browse Source
This commit is contained in:
Олег Бородин
2026-03-30 13:20:13 +02:00
parent 8afe71d925
commit f5227bcac9
8 changed files with 291 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fileoper/check.go
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package fileoper
import (
"context"
"net/http"
"mstore/pkg/descr"
)
// Check files
type CheckFilesParams struct {
Path string
PathType string `param:"pathType"`
}
type CheckFilesResult struct {
Files []descr.File `json:"files,omitempty"`
}
func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params *CheckFilesParams) (int, *CheckFilesResult, error) {
var code int
res := &CheckFilesResult{}
var err error
// Check existing and size
files, err := oper.listFiles(ctx, params.PathType, params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, file := range files {
exists, size, err := oper.store.FileExists(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
if !exists {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
if size != file.Size {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
err = oper.store.DeleteFile(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
// Check hashs
files, err = oper.listFiles(ctx, params.PathType, params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, file := range files {
sum, err := oper.store.GetFileCheksum(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
if sum != file.Checksum {
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
err = oper.store.DeleteFile(file.Collection, file.Name)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
return code, res, err
}
app/fileoper/listfiles.go
+47 -43
View File
@@ -26,51 +26,13 @@ type ListFilesResult struct {
func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) {
var err error
res := &ListFilesResult{}
switch params.PathType {
case filecli.PathTypeRegexp:
files, err := oper.listFilesWithRegex(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
case filecli.PathTypePrefix:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesWithPrefix(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
default:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesInCollection(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
files, err := oper.listFiles(ctx, params.PathType, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
code := http.StatusOK
return code, res, err
}
@@ -122,3 +84,45 @@ func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]d
}
return res, err
}
func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string) ([]descr.File, error) {
res := make([]descr.File, 0)
var err error
switch pathType {
case filecli.PathTypeRegexp:
files, err := oper.listFilesWithRegex(ctx, filepath)
if err != nil {
return res, err
}
res = files
case filecli.PathTypePrefix:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
files, err := oper.listFilesWithPrefix(ctx, filepath)
if err != nil {
return res, err
}
res = files
default:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
files, err := oper.listFilesInCollection(ctx, filepath)
if err != nil {
return res, err
}
res = files
}
return res, err
}
app/handler/file.go
+38
View File
@@ -254,6 +254,44 @@ func (hand *Handler) ListFiles(rctx *router.Context) {
rctx.SendJSON(code, res.Files)
}
func (hand *Handler) CheckFiles(rctx *router.Context) {
filepath, _ := rctx.GetSubpath("path")
if filepath == "" {
filepath = "/"
}
params := &fileoper.CheckFilesParams{
Path: filepath,
}
err := rctx.BindQuery(params)
if err != nil {
hand.logg.Errorf("CheckFiles binding error: %v", err)
rctx.SetStatus(http.StatusInternalServerError)
return
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteFiles, params.Path)
if err != nil {
rctx.SetStatus(http.StatusInternalServerError)
return
}
if !opEnable {
rctx.SetStatus(http.StatusMethodNotAllowed)
return
}
// Execution of the operation
ctx := rctx.GetContext()
code, res, err := hand.fiop.CheckFiles(ctx, operatorID, params)
if err != nil {
hand.logg.Errorf("CheckFiles error: %v", err)
rctx.SetStatus(code)
return
}
rctx.SendJSON(code, res.Files)
}
// GetProperty godoc
//
// @Summary List collections
app/service/service.go
+3
View File
@@ -60,6 +60,9 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections)
svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections)
app/storage/file.go
+38
View File
@@ -38,6 +38,24 @@ func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) FileExists(collection, filename string) (bool, int64, error) {
var err error
var exists bool
var size int64
filename = store.makeFilepath(collection, filename)
stat, err := os.Stat(filename)
if os.IsNotExist(err) {
err = nil
return exists, size, err
}
if err != nil {
return exists, size, err
}
size = stat.Size()
exists = true
return exists, size, err
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
@@ -51,6 +69,26 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser,
return res, err
}
func (store *Storage) GetFileCheksum(collection, filename string) (string, error) {
var err error
var res string
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
hasher := NewHasher()
_, err = io.Copy(hasher.Writer(), file)
if err != nil {
return res, err
}
res = hasher.Hex()
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64