ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

working commit

Browse Source
This commit is contained in:
Олег Бородин
2026-03-11 19:47:40 +02:00
parent a81334aedf
commit a064d942e7
56 changed files with 564 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/imageoper/delman.go
+11 -8
View File
@@ -15,6 +15,8 @@ import (
"net/http"
"mstore/pkg/descr"
ocidigest "github.com/opencontainers/go-digest"
)
type DeleteManifestParams struct {
@@ -43,26 +45,27 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
var exists bool
var reference string
manifestDescr := descr.Manifest{}
mandescr := descr.Manifest{}
// Check manifest by digest as name
if stringLikeSHADigest(params.Reference) {
digest := normalizeSHADigest(params.Reference)
exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest)
digobj, err := ocidigest.Parse(params.Reference)
if err == nil {
exists, mandescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digobj.String())
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
reference = manifestDescr.Reference
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
} else {
// Check manifest by name and reference
exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -70,8 +73,8 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
return res, http.StatusNotFound, err
}
reference = params.Reference
for _, manifestDescr := range manifestDescrs {
reference = manifestDescr.Reference
for _, mandescr := range mandescrs {
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
app/imageoper/getman.go
+22 -35
View File
@@ -15,8 +15,9 @@ import (
"net/http"
"strconv"
"mstore/pkg/auxoci"
"mstore/pkg/descr"
ocidigest "github.com/opencontainers/go-digest"
)
type GetManifestParams struct {
@@ -47,56 +48,42 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
oper.iLock.WaitAndLock(resName)
defer oper.iLock.Done(resName)
manifestDescr := descr.Manifest{}
manDescr := descr.Manifest{}
var exists bool
// TODO: checking layers?
if stringLikeSHADigest(params.Reference) {
digest := normalizeSHADigest(params.Reference)
exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest)
digobj, err := ocidigest.Parse(params.Reference)
if err == nil {
exists, manDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digobj.String())
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload)
res.DockerContentDigest = manifestDigest.String()
res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10)
res.ContentType = manifestDescr.ContentType
res.Payload = manifestDescr.Payload
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
} else {
// Create index of manifests
exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
/*
index, indexBytes, err := indexFromManigestDescrs(manifestDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
indexDigest := auxoci.SHA256DigestFromString(indexBytes)
res.DockerContentDigest = indexDigest.String()
res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10)
res.ContentType = index.MediaType
res.Payload = string(indexBytes)
*/
manifestDescr = manifestDescrs[0]
manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload)
res.DockerContentDigest = manifestDigest.String()
res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10)
res.ContentType = manifestDescr.ContentType
res.Payload = manifestDescr.Payload
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.SHA256.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Payload = string(indexdata)
}
return res, http.StatusOK, err
}
app/imageoper/imgaux.go
-59
View File
@@ -1,59 +0,0 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package imageoper
import (
"encoding/hex"
"strings"
)
const sha256prefix = "sha256:"
const sha512prefix = "sha512:"
func normalizeSHADigest(digest string) string {
if stringLikeSHA256Digest(digest) && !strings.HasPrefix(digest, sha256prefix) {
digest = sha256prefix + digest
} else if stringLikeSHA512Digest(digest) && !strings.HasPrefix(digest, sha512prefix) {
digest = sha512prefix + digest
}
return digest
}
func stringLikeSHADigest(some string) bool {
return stringLikeSHA256Digest(some) || stringLikeSHA512Digest(some)
}
func stringLikeSHA256Digest(some string) bool {
if strings.HasPrefix(some, sha256prefix) {
some = strings.TrimPrefix(some, sha256prefix)
}
_, err := hex.DecodeString(some)
if err != nil {
return false
}
if len(some) == 64 {
return true
}
return false
}
func stringLikeSHA512Digest(some string) bool {
if strings.HasPrefix(some, sha512prefix) {
some = strings.TrimPrefix(some, sha512prefix)
}
_, err := hex.DecodeString(some)
if err != nil {
return false
}
if len(some) == 128 {
return true
}
return false
}
app/imageoper/manexist.go
+27 -18
View File
@@ -15,8 +15,10 @@ import (
"net/http"
"strconv"
"mstore/pkg/auxoci"
//"mstore/pkg/auxoci"
"mstore/pkg/descr"
ocidigest "github.com/opencontainers/go-digest"
)
type ManifestExistsParams struct {
@@ -43,35 +45,42 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
return res, http.StatusBadRequest, err
}
var manifest descr.Manifest
manifests := make([]descr.Manifest, 0)
var exists bool
if stringLikeSHA256Digest(params.Reference) {
digest := fmt.Sprintf("%s:%s", sha256prefix, params.Reference)
exists, manifest, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest)
var man descr.Manifest
var exist bool
digobj, err := ocidigest.Parse(params.Reference)
if err == nil {
exist, man, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digobj.String())
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
if !exist {
return res, http.StatusNotFound, err
}
size := int64(len(man.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = man.ContentType
res.DockerContentDigest = man.Digest
res.Exists = exist
} else {
exists, manifests, err = oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
// Create index of manifests
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
manifest = manifests[0] // TODO: tmp
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.SHA256.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Exists = true
}
digest := auxoci.SHA256DigestFromString(manifest.Payload)
payloadSize := len(manifest.Payload)
res.ContentLength = strconv.FormatInt(int64(payloadSize), 10)
res.ContentType = manifest.ContentType
res.DockerContentDigest = digest.String()
res.Exists = exists
return res, http.StatusOK, err
}
app/imageoper/ociaux.go
+27 -22
View File
@@ -12,7 +12,7 @@ package imageoper
import (
"encoding/json"
"mstore/pkg/auxoci"
//"mstore/pkg/auxoci"
"mstore/pkg/auxtool"
"mstore/pkg/auxuuid"
"mstore/pkg/descr"
@@ -26,45 +26,50 @@ const (
oimMediaType = "application/vnd.oci.image.manifest.v1+json"
)
func indexFromManigestDescrs(manifestDescrs []descr.Manifest) (ocispec.Index, string, error) {
func indexFromManigestDescrs(mandescrs []descr.Manifest) (ocispec.Index, []byte, error) {
var err error
var payload string
var indexdata []byte
index := ocispec.Index{
MediaType: oiiMediaType,
Manifests: make([]ocispec.Descriptor, 0),
}
index.Versioned.SchemaVersion = 2
for _, manifestDescr := range manifestDescrs {
var ociManifest ocispec.Manifest
for _, mandescr := range mandescrs {
var man ocispec.Manifest
ociManifest.Subject = &ocispec.Descriptor{}
ociManifest.Subject.Platform = &ocispec.Platform{}
ociManifest.Config = ocispec.Descriptor{}
ociManifest.Config.Platform = &ocispec.Platform{}
man.Subject = &ocispec.Descriptor{}
man.Subject.Platform = &ocispec.Platform{}
man.Config = ocispec.Descriptor{}
man.Config.Platform = &ocispec.Platform{}
err = json.Unmarshal([]byte(manifestDescr.Payload), &ociManifest)
err = json.Unmarshal([]byte(mandescr.Payload), &man)
if err != nil {
return index, payload, err
return index, indexdata, err
}
digobj, err := ocidigest.Parse(mandescr.Digest)
if err != nil {
return index, indexdata, err
}
size := int64(len(mandescr.Payload))
descriptor := ocispec.Descriptor{
MediaType: oimMediaType,
Digest: auxoci.SHA256DigestFromString(manifestDescr.Payload),
Size: int64(len(manifestDescr.Payload)),
Platform: ociManifest.Subject.Platform,
MediaType: mandescr.ContentType,
Digest: digobj,
Size: size,
Platform: man.Subject.Platform,
}
index.Manifests = append(index.Manifests, descriptor)
}
indexBytes, err := json.Marshal(index)
indexdata, err = json.Marshal(index)
if err != nil {
return index, payload, err
return index, indexdata, err
}
payload = string(indexBytes)
return index, payload, err
indexdata = indexdata
return index, indexdata, err
}
func descrsFromManifest(name, reference string, manifest *ocispec.Manifest, rawManifest []byte) (descr.Manifest, []descr.Blob, error) {
var err error
manifestDescr := descr.Manifest{}
mandescr := descr.Manifest{}
//configDescr := descr.Blob{}
blobDescrs := make([]descr.Blob, 0)
@@ -72,7 +77,7 @@ func descrsFromManifest(name, reference string, manifest *ocispec.Manifest, rawM
// Make manifest descriptor
manifestDigest := ocidigest.SHA256.FromBytes(rawManifest).String()
manifestDescr = descr.Manifest{
mandescr = descr.Manifest{
ID: auxuuid.NewUUID(),
Name: name,
Reference: reference,
@@ -114,7 +119,7 @@ func descrsFromManifest(name, reference string, manifest *ocispec.Manifest, rawM
layerMap[string(layer.Digest)] = true
}
}
return manifestDescr, blobDescrs, err
return mandescr, blobDescrs, err
}
func layersDiff(name, reference string, existingManifest, incomingManifest *ocispec.Manifest, rawManifest []byte) ([]descr.Blob, []descr.Blob, error) {
app/imageoper/operator.go
-2
View File
@@ -26,7 +26,6 @@ type Operator struct {
store *storage.Storage
logg *logger.Logger
iLock *locker.Locker
fLock *locker.Locker
}
func NewOperator(params *OperatorParams) (*Operator, error) {
@@ -36,7 +35,6 @@ func NewOperator(params *OperatorParams) (*Operator, error) {
store: params.Store,
}
oper.iLock = locker.NewLocker()
oper.fLock = locker.NewLocker()
oper.logg = logger.NewLoggerWithSubject("imageoper")
return oper, err
}
app/imageoper/putman.go
+31 -34
View File
@@ -12,13 +12,14 @@ package imageoper
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"mstore/pkg/auxoci"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type PutManifestParams struct {
@@ -53,14 +54,8 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
err = fmt.Errorf("Empty name")
return res, http.StatusBadRequest, err
}
// Check Content-Type
var mimeIsAcceptably bool
mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oimMimeType
mimeIsAcceptably = mimeIsAcceptably || params.ContentType == ddmMimeType
//mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oicMimeType
//mimeIsAcceptably = mimeIsAcceptably || params.ContentType == dciMimeType
if !mimeIsAcceptably {
if params.ContentType != oimMimeType && params.ContentType != ddmMimeType {
err = fmt.Errorf("Unknown or empty Content-Type: %s", params.ContentType)
return res, http.StatusNotFound, err
}
@@ -86,47 +81,48 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
if err != nil {
return res, http.StatusInternalServerError, err
}
incomingManifestBytes := buffer.Bytes()
if int64(len(incomingManifestBytes)) != contentLength {
inManData := buffer.Bytes()
if int64(len(inManData)) != contentLength {
err = fmt.Errorf("Mismatch Content-Length and received manifest size: %d vs %d",
contentLength, len(incomingManifestBytes))
contentLength, len(inManData))
code := http.StatusInternalServerError
return res, code, err
}
if len(incomingManifestBytes) > (4 * 1024 * 1024) {
err = fmt.Errorf("Payload more 4M: %d bytes", len(incomingManifestBytes))
if len(inManData) > (4 * 1024 * 1024) {
err = fmt.Errorf("Payload more 4M: %d bytes", len(inManData))
code := http.StatusRequestEntityTooLarge
return res, code, err
}
incomingManifest, err := auxoci.ParseOCIManifest(incomingManifestBytes)
inMan := &ocispec.Manifest{}
err = json.Unmarshal(inManData, inMan)
if err != nil {
err = fmt.Errorf("Parsing OCI manifest error: %v", err)
err = fmt.Errorf("Manifest parsing error: %v", err)
return res, http.StatusInternalServerError, err
}
if incomingManifest.MediaType == "" {
incomingManifest.MediaType = params.ContentType
if inMan.MediaType == "" {
inMan.MediaType = params.ContentType
}
name := params.Name
reference := params.Reference
arch := incomingManifest.Subject.Platform.Architecture
os := incomingManifest.Subject.Platform.OS
variant := incomingManifest.Subject.Platform.Variant
if inMan.Subject == nil {
}
manifestExists, existingManifestDescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
arch := inMan.Subject.Platform.Architecture
os := inMan.Subject.Platform.OS
variant := inMan.Subject.Platform.Variant
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
if err != nil {
return res, http.StatusInternalServerError, err
}
incomingManifestDescr, incomingLayerDescrs, err := descrsFromManifest(name, reference, incomingManifest, incomingManifestBytes)
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
// Always check layer files for availability
var blobError error
for _, blobDescr := range incomingLayerDescrs {
for _, blobDescr := range inlayerdescrs {
blobExists, _, err := oper.store.BlobExists(blobDescr.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
@@ -141,32 +137,33 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
// TODO: more relevant code?
return res, http.StatusFailedDependency, blobError
}
if !manifestExists {
if !manexist {
// Store manifest and layesrs data
err = oper.mdb.InsertManifestWithLayers(ctx, &incomingManifestDescr, incomingLayerDescrs)
err = oper.mdb.InsertManifestWithLayers(ctx, &inManDescr, inlayerdescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
} else {
/* TODO: only update descr
if bytes.Equal(existingManifestBytes, incomingManifestBytes) {
if bytes.Equal(exManData, inManData) {
return res, http.StatusCreated, err
}
*/
existingManifestBytes := []byte(existingManifestDescr.Payload)
existingManifest, err := auxoci.ParseOCIManifest(existingManifestBytes)
exManData := []byte(exMandescr.Payload)
exMan := &ocispec.Manifest{}
err := json.Unmarshal(exManData, exMan)
if err != nil {
return res, http.StatusInternalServerError, err
}
addedBlobDescrs, uselessBlobDescrs, err := layersDiff(name, reference,
existingManifest, incomingManifest, incomingManifestBytes)
exMan, inMan, inManData)
if err != nil {
return res, http.StatusInternalServerError, err
}
// Starting manifest and blobs transaction
err = oper.mdb.UpdateManifestWithBlobs(ctx, &incomingManifestDescr, addedBlobDescrs, uselessBlobDescrs)
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -190,7 +187,7 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
}
}
for _, blobDescr := range incomingLayerDescrs {
for _, blobDescr := range inlayerdescrs {
// TODO: move the requests to db layer transaction
blobDescrExists, _, err := oper.mdb.GetBlobByNameDigest(ctx, blobDescr.Name, blobDescr.Digest)
if err != nil {