From 1c894e190d989cb6f9da2a9783cac15e0913fcd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Mon, 30 Mar 2026 23:12:02 +0200 Subject: [PATCH] added minimal image checker --- app/fileoper/check.go | 60 +++++++++++++++++++++- app/fileoper/listfiles.go | 4 -- app/handler/manifest.go | 25 +++++++++ app/imageoper/checkimage.go | 66 ++++++++++++++++++++++++ app/maindb/blob.go | 17 +++++++ app/service/service.go | 7 ++- app/storage/file.go | 32 +++++++++--- cmd/mstorectl/imagecmd/checkimgs.go | 51 +++++++++++++++++++ cmd/mstorectl/imagecmd/imagecmd.go | 10 ++++ pkg/filecli/checkfiles.go | 2 +- pkg/repocli/checkimg.go | 78 +++++++++++++++++++++++++++++ pkg/repocli/referer.go | 5 ++ 12 files changed, 341 insertions(+), 16 deletions(-) create mode 100644 app/imageoper/checkimage.go create mode 100644 cmd/mstorectl/imagecmd/checkimgs.go create mode 100644 pkg/repocli/checkimg.go diff --git a/app/fileoper/check.go b/app/fileoper/check.go index 9524f81..d0f5b21 100644 --- a/app/fileoper/check.go +++ b/app/fileoper/check.go @@ -7,8 +7,11 @@ import ( "context" "net/http" "path/filepath" + "regexp" + "strings" "mstore/pkg/descr" + "mstore/pkg/filecli" ) // Check files @@ -50,7 +53,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params } } if size != file.Size { - oper.logg.Warningf("File has incorrect size: %s", fullpath) + oper.logg.Warningf("Delete file with incorrect size: %s", fullpath) res.Files = append(res.Files, file) err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) if err != nil { @@ -78,7 +81,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params } fullpath := filepath.Join(file.Collection, file.Name) if sum != file.Checksum { - oper.logg.Warningf("File has incorrect digest: %s", fullpath) + oper.logg.Warningf("Delete file with incorrect digest: %s", fullpath) res.Files = append(res.Files, file) err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name) if err != nil { @@ -92,5 +95,58 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params } } } + // Find orphans + filelist, err := oper.store.ListAllFiles() + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + for _, fullpath := range filelist { + switch params.PathType { + case filecli.PathTypeRegexp: + re, err := regexp.Compile(params.Path) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + + } + if !re.MatchString(fullpath) { + continue + } + case filecli.PathTypePrefix: + prefix, err := cleanFilepath(params.Path) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + if !strings.HasPrefix(fullpath, prefix) { + continue + } + default: + collection, err := cleanFilepath(params.Path) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + if filepath.Dir(fullpath) != collection { + continue + } + } + filename := filepath.Base(fullpath) + collection := filepath.Dir(fullpath) + exists, _, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + if !exists { + oper.logg.Warningf("Delete orphan file: %s", fullpath) + err = oper.store.DeleteFile(collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + } return code, res, err } diff --git a/app/fileoper/listfiles.go b/app/fileoper/listfiles.go index c58e1de..aa4be1a 100644 --- a/app/fileoper/listfiles.go +++ b/app/fileoper/listfiles.go @@ -110,10 +110,6 @@ func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string) } res = files default: - filepath, err = cleanFilepath(filepath) - if err != nil { - return res, err - } filepath, err = cleanFilepath(filepath) if err != nil { return res, err diff --git a/app/handler/manifest.go b/app/handler/manifest.go index 074bae3..87adcf8 100644 --- a/app/handler/manifest.go +++ b/app/handler/manifest.go @@ -236,3 +236,28 @@ func (hand *Handler) ListManifests(rctx *router.Context) { rctx.SendJSON(code, res.Repositories) } + +func (hand *Handler) CheckImages(rctx *router.Context) { + name, _ := rctx.GetSubpath("name") + params := &imageoper.CheckImagesParams{ + Name: name, + } + // Rigth checking + operatorID, _ := rctx.GetString(userTag) + opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteImages, name) + if err != nil { + rctx.SetStatus(http.StatusInternalServerError) + return + } + if !opEnable { + rctx.SetStatus(http.StatusMethodNotAllowed) + return + } + // Execution of the operation + ctx := rctx.GetContext() + res, code, err := hand.imop.CheckImages(ctx, params) + if err != nil { + hand.logg.Errorf("CheckImages error: %v", err) + } + rctx.SendJSON(code, res.Repositories) +} diff --git a/app/imageoper/checkimage.go b/app/imageoper/checkimage.go new file mode 100644 index 0000000..d89c510 --- /dev/null +++ b/app/imageoper/checkimage.go @@ -0,0 +1,66 @@ +/* + * Copyright 2026 Oleg Borodin + */ +package imageoper + +import ( + "context" + "encoding/json" + "net/http" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type CheckImagesParams struct { + Name string +} +type CheckImagesResult struct { + Repositories []string `json:"repositories"` +} + +func (oper *Operator) CheckImages(ctx context.Context, params *CheckImagesParams) (*CheckImagesResult, int, error) { + var err error + res := &CheckImagesResult{ + Repositories: make([]string, 0), + } + manDescrs, err := oper.mdb.ListAllManifests(ctx) + if err != nil { + return res, http.StatusInternalServerError, err + } + for _, manDescr := range manDescrs { + oper.logg.Debugf("Check image %s:%s", manDescr.Name, manDescr.Reference) + man := &ocispec.Manifest{} + err = json.Unmarshal([]byte(manDescr.Payload), man) + if err != nil { + return res, http.StatusInternalServerError, err + } + blobs := make([]ocispec.Descriptor, 0) + blobs = append(blobs, man.Config) + blobs = append(blobs, man.Layers...) + incorrectImage := false + for _, blob := range blobs { + oper.logg.Debugf("Check block %s", blob.Digest.String()) + blobExists, blobDescr, err := oper.mdb.GetBlobByNameRefDigest(ctx, manDescr.Name, manDescr.Reference, blob.Digest.String()) + if err != nil { + return res, http.StatusInternalServerError, err + } + blobExists, blobSize, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !blobExists || blobSize != blobDescr.Size { + incorrectImage = true + } + } + if incorrectImage { + repo := manDescr.Name + ":" + manDescr.Reference + oper.logg.Debugf("Delete incomplete image: %s", repo) + res.Repositories = append(res.Repositories, repo) + err = oper.deleteManifestObjects(ctx, manDescr.Name, manDescr.Reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + } + } + return res, http.StatusOK, err +} diff --git a/app/maindb/blob.go b/app/maindb/blob.go index cb69f9e..1513045 100644 --- a/app/maindb/blob.go +++ b/app/maindb/blob.go @@ -59,6 +59,23 @@ func (db *Database) GetBlobByNameDigest(ctx context.Context, name, digest string return exists, res, err } +func (db *Database) GetBlobByNameRefDigest(ctx context.Context, name, reference, digest string) (bool, descr.Blob, error) { + var err error + blobs := make([]descr.Blob, 0) + res := descr.Blob{} + exists := false + request := `SELECT * FROM blobs WHERE name = $1 AND reference = $2 AND digest = $3 LIMIT 1` + err = db.db.Select(&blobs, request, name, reference, digest) + if err != nil { + return exists, res, err + } + if len(blobs) > 0 { + res = blobs[0] + exists = true + } + return exists, res, err +} + func (db *Database) ListAllBlobs(ctx context.Context) ([]descr.Blob, error) { var err error blobs := make([]descr.Blob, 0) diff --git a/app/service/service.go b/app/service/service.go index 16d02e0..6e3df37 100644 --- a/app/service/service.go +++ b/app/service/service.go @@ -60,8 +60,8 @@ func (svc *Service) Build() error { svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles) svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles) - svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles) - svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles) + svc.rout.Post(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles) + svc.rout.Post(`/v3/api/checker/`, svc.hand.CheckFiles) svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections) svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections) @@ -88,6 +88,9 @@ func (svc *Service) Build() error { svc.rout.Get(`/v2/{name}/referrers/{digest}`, svc.hand.GetReferer) svc.rout.Get(`/v2/_catalog`, svc.hand.ListManifests) + svc.rout.Post(`/v2/checker/{name}`, svc.hand.CheckImages) + svc.rout.Post(`/v2/checker`, svc.hand.CheckImages) + svc.rout.Post(`/v3/api/account/create`, svc.hand.CreateAccount) svc.rout.Post(`/v3/api/account/get`, svc.hand.GetAccount) svc.rout.Post(`/v3/api/account/update`, svc.hand.UpdateAccount) diff --git a/app/storage/file.go b/app/storage/file.go index 9beb5cf..57a2978 100644 --- a/app/storage/file.go +++ b/app/storage/file.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strings" "mstore/pkg/auxuuid" ) @@ -59,7 +60,6 @@ func (store *Storage) FileExists(collection, filename string) (bool, int64, erro func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) { var err error var res io.ReadCloser - filename = store.makeFilepath(collection, filename) file, err := os.OpenFile(filename, os.O_RDONLY, 0) if err != nil { @@ -72,14 +72,12 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, func (store *Storage) GetFileCheksum(collection, filename string) (string, error) { var err error var res string - filename = store.makeFilepath(collection, filename) file, err := os.OpenFile(filename, os.O_RDONLY, 0) if err != nil { return res, err } defer file.Close() - hasher := NewHasher() _, err = io.Copy(hasher.Writer(), file) if err != nil { @@ -116,7 +114,6 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er return tmpName, size, csum, err } csum = hasher.Hex() - return tmpName, size, csum, err } @@ -158,8 +155,29 @@ func (store *Storage) DeleteFile(collection, filename string) error { if err != nil { return err } - // TODO: clean removing - dirname := store.makeCollecionpath(collection) - os.RemoveAll(dirname) + // TODO: clean dirs removing return err } + +func (store *Storage) ListAllFiles() ([]string, error) { + names := make([]string, 0) + var err error + rootdir := store.makeCollecionpath(string(filepath.Separator)) + walker := func(filename string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + if !fileInfo.Mode().IsRegular() { + return nil + } + name := strings.TrimPrefix(filename, filepath.Clean(rootdir)) + name = filepath.Join(string(filepath.Separator), name) + names = append(names, name) + return nil + } + err = filepath.Walk(rootdir, walker) + if err != nil { + return names, err + } + return names, err +} diff --git a/cmd/mstorectl/imagecmd/checkimgs.go b/cmd/mstorectl/imagecmd/checkimgs.go new file mode 100644 index 0000000..3976de3 --- /dev/null +++ b/cmd/mstorectl/imagecmd/checkimgs.go @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Oleg Borodin + */ +package imagecmd + +import ( + "context" + "time" + + "github.com/spf13/cobra" + + "mstore/pkg/repocli" +) + +// CheckImages +type CheckImagesParams struct { + Imagepath string +} + +type CheckImagesResult struct { + Repos []string `json:"repos"` +} + +func (util *ImageUtil) CheckImages(cmd *cobra.Command, args []string) { + util.checkImagesParams.Imagepath = args[0] + res, err := util.checkImages(&util.commonImageParams, &util.checkImagesParams) + printResponse(res, err) +} + +func (util *ImageUtil) checkImages(common *CommonImageParams, params *CheckImagesParams) (*CheckImagesResult, error) { + var err error + res := &CheckImagesResult{ + Repos: make([]string, 0), + } + timeout := time.Duration(common.Timeout) * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + + ref, err := repocli.NewReferer(params.Imagepath) + if err != nil { + return res, err + } + ref.SetUserinfo(common.Username, common.Password) + mw := repocli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := repocli.NewClient(nil, mw) + opres, err := cli.CheckImages(ctx, ref.Raw()) + if err != nil { + return res, err + } + res.Repos = opres + return res, err +} diff --git a/cmd/mstorectl/imagecmd/imagecmd.go b/cmd/mstorectl/imagecmd/imagecmd.go index 6045bd9..10ca262 100644 --- a/cmd/mstorectl/imagecmd/imagecmd.go +++ b/cmd/mstorectl/imagecmd/imagecmd.go @@ -20,6 +20,7 @@ type ImageUtil struct { pullImageParams PullImageParams pushImageParams PushImageParams deleteImageParams DeleteImageParams + checkImagesParams CheckImagesParams commonImageParams CommonImageParams } @@ -107,6 +108,15 @@ func (util *ImageUtil) CreateImageCmds() *cobra.Command { } subCmd.AddCommand(catalogImagesCmd) + // CheckFiles + var checkImagesCmd = &cobra.Command{ + Use: "check [user:pass@]hostname[:port][/path:tag]", + Short: "Check containet image", + Args: cobra.ExactArgs(1), + Run: util.CheckImages, + } + subCmd.AddCommand(checkImagesCmd) + return subCmd } diff --git a/pkg/filecli/checkfiles.go b/pkg/filecli/checkfiles.go index e6c8fef..9a03501 100644 --- a/pkg/filecli/checkfiles.go +++ b/pkg/filecli/checkfiles.go @@ -21,7 +21,7 @@ func (cli *Client) CheckFiles(ctx context.Context, rawpath string) ([]byte, erro } uri := ref.CheckEP() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, nil) if err != nil { return list, err } diff --git a/pkg/repocli/checkimg.go b/pkg/repocli/checkimg.go new file mode 100644 index 0000000..4ef362f --- /dev/null +++ b/pkg/repocli/checkimg.go @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Oleg Borodin + * + * +*/ + +package repocli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" +) + +func (cli *Client) CheckImages(ctx context.Context, rawrepo string) ([]string, error) { + var err error + list := make([]string, 0) + rawlist, err := cli.CheckImagesRaw(ctx, rawrepo) + err = json.Unmarshal(rawlist, &list) + if err != nil { + return list, err + } + if err != nil { + return list, err + } + return list, err +} + +func (cli *Client) CheckImagesRaw(ctx context.Context, rawrepo string) ([]byte, error) { + var err error + res := make([]byte, 0) + + ref, err := NewReferer(rawrepo) + if err != nil { + return res, err + } + uri := ref.CheckerEP() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, nil) + if err != nil { + return res, err + } + req.Header.Set("User-Agent", cli.userAgent) + req.Header.Set("Accept", "*/*") + resp, err := cli.httpClient.Do(req) + if err != nil { + return res, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return res, err + } + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("Unxected response code %s", resp.Status) + return res, err + } + + contentLength := resp.Header.Get("Content-Length") + if contentLength == "" { + err = errors.New("Empty Content-Length header") + return res, err + } + listSize, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return res, err + } + buffer := bytes.NewBuffer(nil) + recSize, err := Copy(ctx, buffer, resp.Body) + if listSize != recSize { + err := fmt.Errorf("Mismatch declared and actual size") + return res, err + } + res = buffer.Bytes() + return res, err +} diff --git a/pkg/repocli/referer.go b/pkg/repocli/referer.go index eb90452..1f748ba 100644 --- a/pkg/repocli/referer.go +++ b/pkg/repocli/referer.go @@ -137,6 +137,11 @@ func (ref *Referer) CatalogEP() string { return curl.String() } +func (ref *Referer) CheckerEP() string { + curl := ref.urlobj.JoinPath("/v2/checker", ref.base) + return curl.String() +} + func (ref *Referer) Userinfo() (string, string) { return ref.user, ref.pass }