ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

added minimal image checker

Browse Source
This commit is contained in:
Олег Бородин
2026-03-30 23:12:02 +02:00
parent 856ea529a7
commit 1c894e190d
12 changed files with 341 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fileoper/check.go
+58 -2
View File
@@ -7,8 +7,11 @@ import (
"context"
"net/http"
"path/filepath"
"regexp"
"strings"
"mstore/pkg/descr"
"mstore/pkg/filecli"
)
// Check files
@@ -50,7 +53,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
}
if size != file.Size {
oper.logg.Warningf("File has incorrect size: %s", fullpath)
oper.logg.Warningf("Delete file with incorrect size: %s", fullpath)
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
@@ -78,7 +81,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
fullpath := filepath.Join(file.Collection, file.Name)
if sum != file.Checksum {
oper.logg.Warningf("File has incorrect digest: %s", fullpath)
oper.logg.Warningf("Delete file with incorrect digest: %s", fullpath)
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
@@ -92,5 +95,58 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
}
}
// Find orphans
filelist, err := oper.store.ListAllFiles()
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, fullpath := range filelist {
switch params.PathType {
case filecli.PathTypeRegexp:
re, err := regexp.Compile(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !re.MatchString(fullpath) {
continue
}
case filecli.PathTypePrefix:
prefix, err := cleanFilepath(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !strings.HasPrefix(fullpath, prefix) {
continue
}
default:
collection, err := cleanFilepath(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if filepath.Dir(fullpath) != collection {
continue
}
}
filename := filepath.Base(fullpath)
collection := filepath.Dir(fullpath)
exists, _, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !exists {
oper.logg.Warningf("Delete orphan file: %s", fullpath)
err = oper.store.DeleteFile(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
return code, res, err
}
app/fileoper/listfiles.go
-4
View File
@@ -110,10 +110,6 @@ func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string)
}
res = files
default:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
app/handler/manifest.go
+25
View File
@@ -236,3 +236,28 @@ func (hand *Handler) ListManifests(rctx *router.Context) {
rctx.SendJSON(code, res.Repositories)
}
func (hand *Handler) CheckImages(rctx *router.Context) {
name, _ := rctx.GetSubpath("name")
params := &imageoper.CheckImagesParams{
Name: name,
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteImages, name)
if err != nil {
rctx.SetStatus(http.StatusInternalServerError)
return
}
if !opEnable {
rctx.SetStatus(http.StatusMethodNotAllowed)
return
}
// Execution of the operation
ctx := rctx.GetContext()
res, code, err := hand.imop.CheckImages(ctx, params)
if err != nil {
hand.logg.Errorf("CheckImages error: %v", err)
}
rctx.SendJSON(code, res.Repositories)
}
app/imageoper/checkimage.go
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package imageoper
import (
"context"
"encoding/json"
"net/http"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type CheckImagesParams struct {
Name string
}
type CheckImagesResult struct {
Repositories []string `json:"repositories"`
}
func (oper *Operator) CheckImages(ctx context.Context, params *CheckImagesParams) (*CheckImagesResult, int, error) {
var err error
res := &CheckImagesResult{
Repositories: make([]string, 0),
}
manDescrs, err := oper.mdb.ListAllManifests(ctx)
if err != nil {
return res, http.StatusInternalServerError, err
}
for _, manDescr := range manDescrs {
oper.logg.Debugf("Check image %s:%s", manDescr.Name, manDescr.Reference)
man := &ocispec.Manifest{}
err = json.Unmarshal([]byte(manDescr.Payload), man)
if err != nil {
return res, http.StatusInternalServerError, err
}
blobs := make([]ocispec.Descriptor, 0)
blobs = append(blobs, man.Config)
blobs = append(blobs, man.Layers...)
incorrectImage := false
for _, blob := range blobs {
oper.logg.Debugf("Check block %s", blob.Digest.String())
blobExists, blobDescr, err := oper.mdb.GetBlobByNameRefDigest(ctx, manDescr.Name, manDescr.Reference, blob.Digest.String())
if err != nil {
return res, http.StatusInternalServerError, err
}
blobExists, blobSize, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !blobExists || blobSize != blobDescr.Size {
incorrectImage = true
}
}
if incorrectImage {
repo := manDescr.Name + ":" + manDescr.Reference
oper.logg.Debugf("Delete incomplete image: %s", repo)
res.Repositories = append(res.Repositories, repo)
err = oper.deleteManifestObjects(ctx, manDescr.Name, manDescr.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
}
return res, http.StatusOK, err
}
app/maindb/blob.go
+17
View File
@@ -59,6 +59,23 @@ func (db *Database) GetBlobByNameDigest(ctx context.Context, name, digest string
return exists, res, err
}
func (db *Database) GetBlobByNameRefDigest(ctx context.Context, name, reference, digest string) (bool, descr.Blob, error) {
var err error
blobs := make([]descr.Blob, 0)
res := descr.Blob{}
exists := false
request := `SELECT * FROM blobs WHERE name = $1 AND reference = $2 AND digest = $3 LIMIT 1`
err = db.db.Select(&blobs, request, name, reference, digest)
if err != nil {
return exists, res, err
}
if len(blobs) > 0 {
res = blobs[0]
exists = true
}
return exists, res, err
}
func (db *Database) ListAllBlobs(ctx context.Context) ([]descr.Blob, error) {
var err error
blobs := make([]descr.Blob, 0)
app/service/service.go
+5 -2
View File
@@ -60,8 +60,8 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Post(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Post(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections)
svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections)
@@ -88,6 +88,9 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v2/{name}/referrers/{digest}`, svc.hand.GetReferer)
svc.rout.Get(`/v2/_catalog`, svc.hand.ListManifests)
svc.rout.Post(`/v2/checker/{name}`, svc.hand.CheckImages)
svc.rout.Post(`/v2/checker`, svc.hand.CheckImages)
svc.rout.Post(`/v3/api/account/create`, svc.hand.CreateAccount)
svc.rout.Post(`/v3/api/account/get`, svc.hand.GetAccount)
svc.rout.Post(`/v3/api/account/update`, svc.hand.UpdateAccount)
app/storage/file.go
+25 -7
View File
@@ -8,6 +8,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"mstore/pkg/auxuuid"
)
@@ -59,7 +60,6 @@ func (store *Storage) FileExists(collection, filename string) (bool, int64, erro
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
@@ -72,14 +72,12 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser,
func (store *Storage) GetFileCheksum(collection, filename string) (string, error) {
var err error
var res string
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
hasher := NewHasher()
_, err = io.Copy(hasher.Writer(), file)
if err != nil {
@@ -116,7 +114,6 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
@@ -158,8 +155,29 @@ func (store *Storage) DeleteFile(collection, filename string) error {
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
// TODO: clean dirs removing
return err
}
func (store *Storage) ListAllFiles() ([]string, error) {
names := make([]string, 0)
var err error
rootdir := store.makeCollecionpath(string(filepath.Separator))
walker := func(filename string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
}
if !fileInfo.Mode().IsRegular() {
return nil
}
name := strings.TrimPrefix(filename, filepath.Clean(rootdir))
name = filepath.Join(string(filepath.Separator), name)
names = append(names, name)
return nil
}
err = filepath.Walk(rootdir, walker)
if err != nil {
return names, err
}
return names, err
}
cmd/mstorectl/imagecmd/checkimgs.go
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package imagecmd
import (
"context"
"time"
"github.com/spf13/cobra"
"mstore/pkg/repocli"
)
// CheckImages
type CheckImagesParams struct {
Imagepath string
}
type CheckImagesResult struct {
Repos []string `json:"repos"`
}
func (util *ImageUtil) CheckImages(cmd *cobra.Command, args []string) {
util.checkImagesParams.Imagepath = args[0]
res, err := util.checkImages(&util.commonImageParams, &util.checkImagesParams)
printResponse(res, err)
}
func (util *ImageUtil) checkImages(common *CommonImageParams, params *CheckImagesParams) (*CheckImagesResult, error) {
var err error
res := &CheckImagesResult{
Repos: make([]string, 0),
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := repocli.NewReferer(params.Imagepath)
if err != nil {
return res, err
}
ref.SetUserinfo(common.Username, common.Password)
mw := repocli.NewBasicAuthMiddleware(ref.Userinfo())
cli := repocli.NewClient(nil, mw)
opres, err := cli.CheckImages(ctx, ref.Raw())
if err != nil {
return res, err
}
res.Repos = opres
return res, err
}
cmd/mstorectl/imagecmd/imagecmd.go
+10
View File
@@ -20,6 +20,7 @@ type ImageUtil struct {
pullImageParams PullImageParams
pushImageParams PushImageParams
deleteImageParams DeleteImageParams
checkImagesParams CheckImagesParams
commonImageParams CommonImageParams
}
@@ -107,6 +108,15 @@ func (util *ImageUtil) CreateImageCmds() *cobra.Command {
}
subCmd.AddCommand(catalogImagesCmd)
// CheckFiles
var checkImagesCmd = &cobra.Command{
Use: "check [user:pass@]hostname[:port][/path:tag]",
Short: "Check containet image",
Args: cobra.ExactArgs(1),
Run: util.CheckImages,
}
subCmd.AddCommand(checkImagesCmd)
return subCmd
}
pkg/filecli/checkfiles.go
+1 -1
View File
@@ -21,7 +21,7 @@ func (cli *Client) CheckFiles(ctx context.Context, rawpath string) ([]byte, erro
}
uri := ref.CheckEP()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, nil)
if err != nil {
return list, err
}
pkg/repocli/checkimg.go
+78
View File
@@ -0,0 +1,78 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
*
*/
package repocli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
)
func (cli *Client) CheckImages(ctx context.Context, rawrepo string) ([]string, error) {
var err error
list := make([]string, 0)
rawlist, err := cli.CheckImagesRaw(ctx, rawrepo)
err = json.Unmarshal(rawlist, &list)
if err != nil {
return list, err
}
if err != nil {
return list, err
}
return list, err
}
func (cli *Client) CheckImagesRaw(ctx context.Context, rawrepo string) ([]byte, error) {
var err error
res := make([]byte, 0)
ref, err := NewReferer(rawrepo)
if err != nil {
return res, err
}
uri := ref.CheckerEP()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, nil)
if err != nil {
return res, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return res, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return res, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unxected response code %s", resp.Status)
return res, err
}
contentLength := resp.Header.Get("Content-Length")
if contentLength == "" {
err = errors.New("Empty Content-Length header")
return res, err
}
listSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return res, err
}
buffer := bytes.NewBuffer(nil)
recSize, err := Copy(ctx, buffer, resp.Body)
if listSize != recSize {
err := fmt.Errorf("Mismatch declared and actual size")
return res, err
}
res = buffer.Bytes()
return res, err
}
pkg/repocli/referer.go
+5
View File
@@ -137,6 +137,11 @@ func (ref *Referer) CatalogEP() string {
return curl.String()
}
func (ref *Referer) CheckerEP() string {
curl := ref.urlobj.JoinPath("/v2/checker", ref.base)
return curl.String()
}
func (ref *Referer) Userinfo() (string, string) {
return ref.user, ref.pass
}