ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

added minimal image checker

Browse Source
This commit is contained in:
Олег Бородин
2026-03-30 23:12:02 +02:00
parent 856ea529a7
commit 1c894e190d
12 changed files with 341 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fileoper/check.go
+58 -2
View File
@@ -7,8 +7,11 @@ import (
"context"
"net/http"
"path/filepath"
"regexp"
"strings"
"mstore/pkg/descr"
"mstore/pkg/filecli"
)
// Check files
@@ -50,7 +53,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
}
if size != file.Size {
oper.logg.Warningf("File has incorrect size: %s", fullpath)
oper.logg.Warningf("Delete file with incorrect size: %s", fullpath)
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
@@ -78,7 +81,7 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
fullpath := filepath.Join(file.Collection, file.Name)
if sum != file.Checksum {
oper.logg.Warningf("File has incorrect digest: %s", fullpath)
oper.logg.Warningf("Delete file with incorrect digest: %s", fullpath)
res.Files = append(res.Files, file)
err = oper.mdb.DeleteFileByCollectionName(ctx, file.Collection, file.Name)
if err != nil {
@@ -92,5 +95,58 @@ func (oper *Operator) CheckFiles(ctx context.Context, operatorID string, params
}
}
}
// Find orphans
filelist, err := oper.store.ListAllFiles()
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
for _, fullpath := range filelist {
switch params.PathType {
case filecli.PathTypeRegexp:
re, err := regexp.Compile(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !re.MatchString(fullpath) {
continue
}
case filecli.PathTypePrefix:
prefix, err := cleanFilepath(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !strings.HasPrefix(fullpath, prefix) {
continue
}
default:
collection, err := cleanFilepath(params.Path)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if filepath.Dir(fullpath) != collection {
continue
}
}
filename := filepath.Base(fullpath)
collection := filepath.Dir(fullpath)
exists, _, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if !exists {
oper.logg.Warningf("Delete orphan file: %s", fullpath)
err = oper.store.DeleteFile(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
}
return code, res, err
}
app/fileoper/listfiles.go
-4
View File
@@ -110,10 +110,6 @@ func (oper *Operator) listFiles(ctx context.Context, pathType, filepath string)
}
res = files
default:
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
}
filepath, err = cleanFilepath(filepath)
if err != nil {
return res, err
app/handler/manifest.go
+25
View File
@@ -236,3 +236,28 @@ func (hand *Handler) ListManifests(rctx *router.Context) {
rctx.SendJSON(code, res.Repositories)
}
func (hand *Handler) CheckImages(rctx *router.Context) {
name, _ := rctx.GetSubpath("name")
params := &imageoper.CheckImagesParams{
Name: name,
}
// Rigth checking
operatorID, _ := rctx.GetString(userTag)
opEnable, err := hand.CheckRight(rctx.Ctx, operatorID, terms.RightWriteImages, name)
if err != nil {
rctx.SetStatus(http.StatusInternalServerError)
return
}
if !opEnable {
rctx.SetStatus(http.StatusMethodNotAllowed)
return
}
// Execution of the operation
ctx := rctx.GetContext()
res, code, err := hand.imop.CheckImages(ctx, params)
if err != nil {
hand.logg.Errorf("CheckImages error: %v", err)
}
rctx.SendJSON(code, res.Repositories)
}
app/imageoper/checkimage.go
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*/
package imageoper
import (
"context"
"encoding/json"
"net/http"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type CheckImagesParams struct {
Name string
}
type CheckImagesResult struct {
Repositories []string `json:"repositories"`
}
func (oper *Operator) CheckImages(ctx context.Context, params *CheckImagesParams) (*CheckImagesResult, int, error) {
var err error
res := &CheckImagesResult{
Repositories: make([]string, 0),
}
manDescrs, err := oper.mdb.ListAllManifests(ctx)
if err != nil {
return res, http.StatusInternalServerError, err
}
for _, manDescr := range manDescrs {
oper.logg.Debugf("Check image %s:%s", manDescr.Name, manDescr.Reference)
man := &ocispec.Manifest{}
err = json.Unmarshal([]byte(manDescr.Payload), man)
if err != nil {
return res, http.StatusInternalServerError, err
}
blobs := make([]ocispec.Descriptor, 0)
blobs = append(blobs, man.Config)
blobs = append(blobs, man.Layers...)
incorrectImage := false
for _, blob := range blobs {
oper.logg.Debugf("Check block %s", blob.Digest.String())
blobExists, blobDescr, err := oper.mdb.GetBlobByNameRefDigest(ctx, manDescr.Name, manDescr.Reference, blob.Digest.String())
if err != nil {
return res, http.StatusInternalServerError, err
}
blobExists, blobSize, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !blobExists || blobSize != blobDescr.Size {
incorrectImage = true
}
}
if incorrectImage {
repo := manDescr.Name + ":" + manDescr.Reference
oper.logg.Debugf("Delete incomplete image: %s", repo)
res.Repositories = append(res.Repositories, repo)
err = oper.deleteManifestObjects(ctx, manDescr.Name, manDescr.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
}
return res, http.StatusOK, err
}
app/maindb/blob.go
+17
View File
@@ -59,6 +59,23 @@ func (db *Database) GetBlobByNameDigest(ctx context.Context, name, digest string
return exists, res, err
}
func (db *Database) GetBlobByNameRefDigest(ctx context.Context, name, reference, digest string) (bool, descr.Blob, error) {
var err error
blobs := make([]descr.Blob, 0)
res := descr.Blob{}
exists := false
request := `SELECT * FROM blobs WHERE name = $1 AND reference = $2 AND digest = $3 LIMIT 1`
err = db.db.Select(&blobs, request, name, reference, digest)
if err != nil {
return exists, res, err
}
if len(blobs) > 0 {
res = blobs[0]
exists = true
}
return exists, res, err
}
func (db *Database) ListAllBlobs(ctx context.Context) ([]descr.Blob, error) {
var err error
blobs := make([]descr.Blob, 0)
app/service/service.go
+5 -2
View File
@@ -60,8 +60,8 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v3/api/files/{filepath}`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/files/`, svc.hand.ListFiles)
svc.rout.Get(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Post(`/v3/api/checker/{filepath}`, svc.hand.CheckFiles)
svc.rout.Post(`/v3/api/checker/`, svc.hand.CheckFiles)
svc.rout.Get(`/v3/api/collections/{path}`, svc.hand.ListCollections)
svc.rout.Get(`/v3/api/collections/`, svc.hand.ListCollections)
@@ -88,6 +88,9 @@ func (svc *Service) Build() error {
svc.rout.Get(`/v2/{name}/referrers/{digest}`, svc.hand.GetReferer)
svc.rout.Get(`/v2/_catalog`, svc.hand.ListManifests)
svc.rout.Post(`/v2/checker/{name}`, svc.hand.CheckImages)
svc.rout.Post(`/v2/checker`, svc.hand.CheckImages)
svc.rout.Post(`/v3/api/account/create`, svc.hand.CreateAccount)
svc.rout.Post(`/v3/api/account/get`, svc.hand.GetAccount)
svc.rout.Post(`/v3/api/account/update`, svc.hand.UpdateAccount)
app/storage/file.go
+25 -7
View File
@@ -8,6 +8,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"mstore/pkg/auxuuid"
)
@@ -59,7 +60,6 @@ func (store *Storage) FileExists(collection, filename string) (bool, int64, erro
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
@@ -72,14 +72,12 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser,
func (store *Storage) GetFileCheksum(collection, filename string) (string, error) {
var err error
var res string
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
hasher := NewHasher()
_, err = io.Copy(hasher.Writer(), file)
if err != nil {
@@ -116,7 +114,6 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
@@ -158,8 +155,29 @@ func (store *Storage) DeleteFile(collection, filename string) error {
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
// TODO: clean dirs removing
return err
}
func (store *Storage) ListAllFiles() ([]string, error) {
names := make([]string, 0)
var err error
rootdir := store.makeCollecionpath(string(filepath.Separator))
walker := func(filename string, fileInfo os.FileInfo, err error) error {
if err != nil {
return err
}
if !fileInfo.Mode().IsRegular() {
return nil
}
name := strings.TrimPrefix(filename, filepath.Clean(rootdir))
name = filepath.Join(string(filepath.Separator), name)
names = append(names, name)
return nil
}
err = filepath.Walk(rootdir, walker)
if err != nil {
return names, err
}
return names, err
}