diff --git a/app/config/variant.go b/app/config/variant.go index 6edc328..6cf7189 100644 --- a/app/config/variant.go +++ b/app/config/variant.go @@ -6,5 +6,5 @@ const ( logdir = "/home/ziggi/Projects/mstore/tmp/log" datadir = "/home/ziggi/Projects/mstore/tmp/data" version = "0.2.0" - srvname = "mstored" + srvname = "mstored" ) diff --git a/app/imageoper/_blob.go b/app/imageoper/_blob.go deleted file mode 100644 index 3e59ba0..0000000 --- a/app/imageoper/_blob.go +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright 2026 Oleg Borodin - * - * This work is published and licensed under a Creative Commons - * Attribution-NonCommercial-NoDerivatives 4.0 International License. - * - * Distribution of this work is permitted, but commercial use and - * modifications are strictly prohibited. - */ -package imageoper - -import ( - "context" - "fmt" - "io" - "net/http" - "strconv" - - "mstore/pkg/auxuuid" -) - -type BlobExistsParams struct { - Name string - Digest string -} -type BlobExistsResult struct { - DockerContentDigest string - ContentLength string - ContentType string - //Exists bool -} - -func (oper *Operator) BlobExists(ctx context.Context, operatorID string, params *BlobExistsParams) (*BlobExistsResult, int, error) { - var err error - res := &BlobExistsResult{} - - if params.Digest == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - // Check blob descriptor - descrExists, blobDescr, err := oper.mdb.GetBlobByNameDigest(ctx, params.Digest, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !descrExists { - return res, http.StatusNotFound, err - } - // Check blob file - blobExists, _, err := oper.store.BlobExists(params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !blobExists { - return res, http.StatusNotFound, err - } - - res.ContentLength = strconv.FormatInt(blobDescr.Size, 10) - res.DockerContentDigest = blobDescr.Digest - res.ContentType = blobDescr.MediaType - - return res, http.StatusOK, err -} - -type PostUploadParams struct { - Name string - Digest string - Mount string - From string -} -type PostUploadResult struct { - DockerUploadUUID string - Location string - ContentLength string -} - -func (oper *Operator) PostUpload(ctx context.Context, operatorID string, params *PostUploadParams) (*PostUploadResult, int, error) { - var err error - res := &PostUploadResult{} - - if params.Digest == "" { - uuid := auxuuid.NewUUID() - location := fmt.Sprintf("/v2/%s/blobs/uploads/%s", params.Name, uuid) - res.DockerUploadUUID = uuid - res.Location = location - res.ContentLength = strconv.FormatInt(0, 10) - return res, http.StatusAccepted, err - } else { - err = fmt.Errorf("PostUpload: Not empty digest header") - return res, http.StatusInternalServerError, err - } - return res, http.StatusOK, err -} - -type PatchUploadParams struct { - ContentType string - ContentLength string - ContentRange string - Name string - Reference string - Reader io.Reader -} -type PatchUploadResult struct { - Location string - Range string -} - -// TODO: partial uploading by range? -func (oper *Operator) PatchUpload(ctx context.Context, operatorID string, params *PatchUploadParams) (*PatchUploadResult, int, error) { - var err error - res := &PatchUploadResult{} - - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - - exists, uploadSize, err := oper.store.UploadExists(params.Name, params.Reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - if exists { - res.Location = fmt.Sprintf("/v2/%s/uploads/%s", params.Name, params.Reference) - res.Range = fmt.Sprintf("0-%d", uploadSize-1) - return res, http.StatusNoContent, err - } - - if params.ContentType != "application/octet-stream" { - err = fmt.Errorf("Wrong Conten-Type header: %s", params.ContentType) - return res, http.StatusBadRequest, err - } - var contentLength int64 - - // Unfortunately, podman and used github.com/containers/image don't sent - // Content-length header for docker transport - - if params.ContentLength != "" { - contentLength, err = strconv.ParseInt(params.ContentLength, 10, 64) - if err != nil { - err = fmt.Errorf("Wrong Content-length header") - return res, http.StatusBadRequest, err - } - } - - recsize, _, err := oper.store.WriteUpload(params.Reference, params.Reader) - if err != nil { - return res, http.StatusInternalServerError, err - } - if contentLength != 0 && recsize != contentLength { - oper.store.RemoveUpload(params.Reference) - err = fmt.Errorf("Mismatch upload recorded size and content length") - return res, http.StatusInternalServerError, err - } - res.Location = fmt.Sprintf("/v2/%s/uploads/%s", params.Name, params.Reference) - res.Range = fmt.Sprintf("0-%d", recsize-1) - return res, http.StatusAccepted, err -} - -type PutUploadParams struct { - ContentType string - ContentLength string - ContentRange string - Name string - Reference string - Digest string - Reader io.Reader -} -type PutUploadResult struct { - Location string -} - -func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *PutUploadParams) (*PutUploadResult, int, error) { - var err error - res := &PutUploadResult{} - - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Digest == "" { - err = fmt.Errorf("Empty digest") - return res, http.StatusBadRequest, err - } - if params.ContentType != "application/octet-stream" { - err = fmt.Errorf("Wrong conten type: %s", params.ContentType) - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - var contentLength int64 - if params.ContentLength != "" { - contentLength, err = strconv.ParseInt(params.ContentLength, 10, 64) - if err != nil { - err = fmt.Errorf("Cannot convert Content-Length=%s to integer: %v", params.ContentLength, err) - return res, http.StatusBadRequest, err - } - - } - if contentLength != 0 { - recsize, _, err := oper.store.WriteUpload(params.Reference, params.Reader) - if err != nil { - return res, http.StatusInternalServerError, err - } - if contentLength != 0 && recsize != contentLength { - oper.store.RemoveUpload(params.Reference) - err = fmt.Errorf("Mismatch upload recorded size and content length") - return res, http.StatusInternalServerError, err - } - if err != nil { - return res, http.StatusInternalServerError, err - } - res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest) - } - - err = oper.store.LinkUpload(params.Reference, params.Digest) - if err != nil { - err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err) - return res, http.StatusInternalServerError, err - } - - res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest) - return res, http.StatusCreated, err -} - -type GetBlobParams struct { - Name string - Digest string -} -type GetBlobResult struct { - ContentLength string - ContentType string - DockerContentDigest string - ReadCloser io.ReadCloser -} - -func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *GetBlobParams) (*GetBlobResult, int, error) { - var err error - res := &GetBlobResult{} - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Digest == "" { - err = fmt.Errorf("Empty digest") - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - blobExists, blobSize, err := oper.store.BlobExists(params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - descrExist, blobDescr, err := oper.mdb.GetBlobByNameDigest(ctx, params.Name, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !blobExists || !descrExist { - return res, http.StatusNotFound, err - } - - _, readCloser, err := oper.store.BlobReader(params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - - res.ContentType = blobDescr.MediaType - res.ContentLength = strconv.FormatInt(blobSize, 10) - res.DockerContentDigest = params.Digest - res.ReadCloser = readCloser - return res, http.StatusOK, err -} - -type DeleteBlobParams struct { - Name string - Digest string -} -type DeleteBlobResult struct{} - -// Removing an individual layer is very probably to compromise data integrity. -// - If the data layers are to be reused, this is like shooting yourself in the foot. -// - It also prevents the manifest from being issued, as one -// of the layers is missing. - -func (oper *Operator) DeleteBlob(ctx context.Context, operatorID string, params *DeleteBlobParams) (*DeleteBlobResult, int, error) { - var err error - res := &DeleteBlobResult{} - - if params.Digest == "" { - err = fmt.Errorf("Empty digest") - return res, http.StatusBadRequest, err - } - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - // Check namespace record - descrExists, _, err := oper.mdb.GetBlobByNameDigest(ctx, params.Name, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - - if !descrExists { - return res, http.StatusNotFound, err - } - // Deleting blob record - oper.logg.Warningf("Deleting blob record %s:%s", params.Name, params.Digest) - err = oper.mdb.DeleteBlobByNameDigest(ctx, params.Name, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - // Removing the blob binary if usage == 0 - blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if blobUsage == 0 { - oper.logg.Warningf("Deleting useless blob binary %s", params.Digest) - oper.store.DeleteBlob(params.Digest) - } - return res, http.StatusOK, err -} diff --git a/app/imageoper/delman.go b/app/imageoper/delman.go new file mode 100644 index 0000000..7853c89 --- /dev/null +++ b/app/imageoper/delman.go @@ -0,0 +1,123 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "fmt" + "net/http" + + "mstore/pkg/descr" +) + +type DeleteManifestParams struct { + Name string + Reference string +} +type DeleteManifestResult struct{} + +func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifestParams) (*DeleteManifestResult, int, error) { + var err error + res := &DeleteManifestResult{} + + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + if params.Reference == "" { + err = fmt.Errorf("Empty reference") + return res, http.StatusBadRequest, err + } + + resName := params.Name + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + var exists bool + var reference string + + manifestDescr := descr.Manifest{} + + // Check manifest by digest as name + if stringLikeSHADigest(params.Reference) { + digest := normalizeSHADigest(params.Reference) + exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + reference = manifestDescr.Reference + err = oper.deleteManifestObjects(ctx, params.Name, reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + } else { + // Check manifest by name and reference + exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + reference = params.Reference + for _, manifestDescr := range manifestDescrs { + reference = manifestDescr.Reference + err = oper.deleteManifestObjects(ctx, params.Name, reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + } + } + // Get blobs associated with the name + return res, http.StatusAccepted, err +} + +func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference string) error { + var err error + // Get blobs associated with the name + layers, err := oper.mdb.GetBlobsByReferense(ctx, name, reference) + if err != nil { + return err + } + for _, blob := range layers { + // Delete descr record + err = oper.mdb.DeleteBlobByNameDigest(ctx, blob.Name, blob.Digest) + if err != nil { + return err + } + // Check blob file + exists, _, err := oper.store.BlobExists(blob.Digest) + if err != nil { + return err + } + if exists { + // Check blob usage + blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest) + if err != nil { + return err + } + // Delete if blob useless + if blobUsage == 0 { + err = oper.store.DeleteBlob(blob.Digest) + if err != nil { + return err + } + } + } + } + err = oper.mdb.DeleteManifest(ctx, name, reference) + if err != nil { + return err + } + return err +} diff --git a/app/imageoper/getman.go b/app/imageoper/getman.go new file mode 100644 index 0000000..ee229c7 --- /dev/null +++ b/app/imageoper/getman.go @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "mstore/pkg/auxoci" + "mstore/pkg/descr" +) + +type GetManifestParams struct { + Name string + Reference string +} +type GetManifestResult struct { + ContentLength string + ContentType string + DockerContentDigest string + Payload string +} + +func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams) (*GetManifestResult, int, error) { + var err error + res := &GetManifestResult{} + + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + if params.Reference == "" { + err = fmt.Errorf("Empty reference") + return res, http.StatusBadRequest, err + } + + resName := params.Name + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + manifestDescr := descr.Manifest{} + var exists bool + // TODO: checking layers? + if stringLikeSHADigest(params.Reference) { + digest := normalizeSHADigest(params.Reference) + exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + + manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload) + res.DockerContentDigest = manifestDigest.String() + + res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10) + res.ContentType = manifestDescr.ContentType + res.Payload = manifestDescr.Payload + + } else { + // Create index of manifests + exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + /* + index, indexBytes, err := indexFromManigestDescrs(manifestDescrs) + if err != nil { + return res, http.StatusInternalServerError, err + } + indexDigest := auxoci.SHA256DigestFromString(indexBytes) + res.DockerContentDigest = indexDigest.String() + + res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10) + res.ContentType = index.MediaType + res.Payload = string(indexBytes) + */ + manifestDescr = manifestDescrs[0] + + manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload) + res.DockerContentDigest = manifestDigest.String() + + res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10) + res.ContentType = manifestDescr.ContentType + res.Payload = manifestDescr.Payload + + } + return res, http.StatusOK, err +} diff --git a/app/imageoper/getrefer.go b/app/imageoper/getrefer.go new file mode 100644 index 0000000..d8bb198 --- /dev/null +++ b/app/imageoper/getrefer.go @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "fmt" + "net/http" +) + +type GetRefererParams struct { + Name string + Digest string +} +type GetRefererResult struct { + Reference string +} + +func (oper *Operator) GetReferer(ctx context.Context, params *GetRefererParams) (*GetRefererResult, int, error) { + var err error + res := &GetRefererResult{} + + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + if params.Digest == "" { + err = fmt.Errorf("Empty digest") + return res, http.StatusBadRequest, err + } + manifests, err := oper.mdb.GetReferer(ctx, params.Name, params.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if len(manifests) == 0 { + return res, http.StatusNotFound, err + } + res.Reference = manifests[0].Reference + return res, http.StatusOK, err +} diff --git a/app/imageoper/gettags.go b/app/imageoper/gettags.go new file mode 100644 index 0000000..cf0c655 --- /dev/null +++ b/app/imageoper/gettags.go @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "fmt" + "net/http" + + "mstore/pkg/descr" +) + +type GetTagsParams struct { + Name string +} +type GetTagsResult struct { + TagDescr descr.Tags +} + +func (oper *Operator) GetTags(ctx context.Context, params *GetTagsParams) (*GetTagsResult, int, error) { + var err error + res := &GetTagsResult{ + TagDescr: descr.Tags{ + Name: params.Name, + Tags: make([]string, 0), + }, + } + + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + manifestDescrs, err := oper.mdb.ListManifestsByName(ctx, params.Name) + if err != nil { + return res, http.StatusInternalServerError, err + } + for _, manifestDescr := range manifestDescrs { + res.TagDescr.Tags = append(res.TagDescr.Tags, manifestDescr.Reference) + } + return res, http.StatusOK, err +} diff --git a/app/imageoper/listman.go b/app/imageoper/listman.go new file mode 100644 index 0000000..8adf0e3 --- /dev/null +++ b/app/imageoper/listman.go @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "net/http" + "slices" +) + +type ListManifestsParams struct { + N int64 +} +type ListManifestsResult struct { + Repositories []string `json:"repositories"` +} + +func (oper *Operator) ListManifests(ctx context.Context, params *ListManifestsParams) (*ListManifestsResult, int, error) { + var err error + res := &ListManifestsResult{ + Repositories: make([]string, 0), + } + refMap := make(map[string]bool) + descrs, err := oper.mdb.ListAllManifests(ctx) + if err != nil { + return res, http.StatusInternalServerError, err + } + for _, descr := range descrs { + _, already := refMap[descr.Name] + if !already { + refMap[descr.Name] = true + } + } + for key, _ := range refMap { + res.Repositories = append(res.Repositories, key) + } + slices.Sort(res.Repositories) + return res, http.StatusOK, err +} diff --git a/app/imageoper/manexist.go b/app/imageoper/manexist.go new file mode 100644 index 0000000..22b4f39 --- /dev/null +++ b/app/imageoper/manexist.go @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "mstore/pkg/auxoci" + "mstore/pkg/descr" +) + +type ManifestExistsParams struct { + Name string + Reference string +} +type ManifestExistsResult struct { + ContentLength string + ContentType string + DockerContentDigest string + Exists bool +} + +func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExistsParams) (*ManifestExistsResult, int, error) { + var err error + res := &ManifestExistsResult{} + + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + if params.Reference == "" { + err = fmt.Errorf("Empty reference") + return res, http.StatusBadRequest, err + } + + var manifest descr.Manifest + manifests := make([]descr.Manifest, 0) + var exists bool + if stringLikeSHA256Digest(params.Reference) { + digest := fmt.Sprintf("%s:%s", sha256prefix, params.Reference) + exists, manifest, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + } else { + exists, manifests, err = oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !exists { + return res, http.StatusNotFound, err + } + manifest = manifests[0] // TODO: tmp + } + + digest := auxoci.SHA256DigestFromString(manifest.Payload) + payloadSize := len(manifest.Payload) + res.ContentLength = strconv.FormatInt(int64(payloadSize), 10) + res.ContentType = manifest.ContentType + res.DockerContentDigest = digest.String() + res.Exists = exists + + return res, http.StatusOK, err +} diff --git a/app/imageoper/manifest.go b/app/imageoper/manifest.go deleted file mode 100644 index 5a788a9..0000000 --- a/app/imageoper/manifest.go +++ /dev/null @@ -1,543 +0,0 @@ -/* - * Copyright 2026 Oleg Borodin - * - * This work is published and licensed under a Creative Commons - * Attribution-NonCommercial-NoDerivatives 4.0 International License. - * - * Distribution of this work is permitted, but commercial use and - * modifications are strictly prohibited. - */ -package imageoper - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "slices" - "strconv" - - "mstore/pkg/auxoci" - "mstore/pkg/descr" -) - -type ManifestExistsParams struct { - Name string - Reference string -} -type ManifestExistsResult struct { - ContentLength string - ContentType string - DockerContentDigest string - Exists bool -} - -func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExistsParams) (*ManifestExistsResult, int, error) { - var err error - res := &ManifestExistsResult{} - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - - var manifest descr.Manifest - manifests := make([]descr.Manifest, 0) - var exists bool - if stringLikeSHA256Digest(params.Reference) { - digest := fmt.Sprintf("%s:%s", sha256prefix, params.Reference) - exists, manifest, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - } else { - exists, manifests, err = oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - manifest = manifests[0] // TODO: tmp - } - - digest := auxoci.SHA256DigestFromString(manifest.Payload) - payloadSize := len(manifest.Payload) - res.ContentLength = strconv.FormatInt(int64(payloadSize), 10) - res.ContentType = manifest.ContentType - res.DockerContentDigest = digest.String() - res.Exists = exists - - return res, http.StatusOK, err -} - -type PutManifestParams struct { - ContentType string - ContentLength string - Name string - Reference string - Reader io.Reader -} -type PutManifestResult struct { - Location string -} - -const ( - ddmMimeType = "application/vnd.docker.distribution.manifest.v2+json" - oimMimeType = "application/vnd.oci.image.manifest.v1+json" - - XXXoicMimeType = "application/vnd.oci.image.config.v1+json" - XXXdciMimeType = "application/vnd.docker.container.image.v1+json" -) - -// TODO: lock for the name-reference or simular? -func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams) (*PutManifestResult, int, error) { - var err error - res := &PutManifestResult{} - - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - - // Check Content-Type - var mimeIsAcceptably bool - mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oimMimeType - mimeIsAcceptably = mimeIsAcceptably || params.ContentType == ddmMimeType - //mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oicMimeType - //mimeIsAcceptably = mimeIsAcceptably || params.ContentType == dciMimeType - if !mimeIsAcceptably { - err = fmt.Errorf("Unknown or empty Content-Type: %s", params.ContentType) - return res, http.StatusNotFound, err - } - if params.ContentLength == "" { - code := http.StatusLengthRequired - err = fmt.Errorf("Content-Length is empty") - return res, code, err - } - contentLength, err := strconv.ParseInt(params.ContentLength, 10, 64) - if err != nil { - err = fmt.Errorf("Cannot parse Content-Length value [%s]: %v", params.ContentLength, err) - code := http.StatusLengthRequired - return res, code, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - // Copy manifest data - buffer := bytes.NewBuffer(nil) - _, err = io.Copy(buffer, params.Reader) - if err != nil { - return res, http.StatusInternalServerError, err - } - - incomingManifestBytes := buffer.Bytes() - if int64(len(incomingManifestBytes)) != contentLength { - err = fmt.Errorf("Mismatch Content-Length and received manifest size: %d vs %d", - contentLength, len(incomingManifestBytes)) - code := http.StatusInternalServerError - return res, code, err - } - - if len(incomingManifestBytes) > (4 * 1024 * 1024) { - err = fmt.Errorf("Payload more 4M: %d bytes", len(incomingManifestBytes)) - code := http.StatusRequestEntityTooLarge - return res, code, err - } - - incomingManifest, err := auxoci.ParseOCIManifest(incomingManifestBytes) - if err != nil { - err = fmt.Errorf("Parsing OCI manifest error: %v", err) - return res, http.StatusInternalServerError, err - } - if incomingManifest.MediaType == "" { - incomingManifest.MediaType = params.ContentType - } - - name := params.Name - reference := params.Reference - - arch := incomingManifest.Subject.Platform.Architecture - os := incomingManifest.Subject.Platform.OS - variant := incomingManifest.Subject.Platform.Variant - - manifestExists, existingManifestDescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant) - if err != nil { - return res, http.StatusInternalServerError, err - } - - incomingManifestDescr, incomingLayerDescrs, err := descrsFromManifest(name, reference, incomingManifest, incomingManifestBytes) - // Always check layer files for availability - var blobError error - for _, blobDescr := range incomingLayerDescrs { - blobExists, _, err := oper.store.BlobExists(blobDescr.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !blobExists { - oper.logg.Warningf("Found incomleted blob binary for %s:%s", blobDescr.Name, blobDescr.Digest) - err := fmt.Errorf("Layer %s not found.", blobDescr.Digest) - blobError = errors.Join(blobError, err) - } - } - if blobError != nil { - // TODO: more relevant code? - return res, http.StatusFailedDependency, blobError - } - if !manifestExists { - // Store manifest and layesrs data - err = oper.mdb.InsertManifestWithLayers(ctx, &incomingManifestDescr, incomingLayerDescrs) - if err != nil { - return res, http.StatusInternalServerError, err - } - } else { - /* TODO: only update descr - if bytes.Equal(existingManifestBytes, incomingManifestBytes) { - return res, http.StatusCreated, err - } - */ - - existingManifestBytes := []byte(existingManifestDescr.Payload) - existingManifest, err := auxoci.ParseOCIManifest(existingManifestBytes) - if err != nil { - return res, http.StatusInternalServerError, err - } - addedBlobDescrs, uselessBlobDescrs, err := layersDiff(name, reference, - existingManifest, incomingManifest, incomingManifestBytes) - if err != nil { - return res, http.StatusInternalServerError, err - } - // Starting manifest and blobs transaction - err = oper.mdb.UpdateManifestWithBlobs(ctx, &incomingManifestDescr, addedBlobDescrs, uselessBlobDescrs) - if err != nil { - return res, http.StatusInternalServerError, err - } - for _, blob := range uselessBlobDescrs { - exists, _, err := oper.store.BlobExists(blob.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if exists && blobUsage == 0 { - err = oper.store.DeleteBlob(blob.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - } - } - } - - for _, blobDescr := range incomingLayerDescrs { - // TODO: move the requests to db layer transaction - blobDescrExists, _, err := oper.mdb.GetBlobByNameDigest(ctx, blobDescr.Name, blobDescr.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !blobDescrExists { - oper.logg.Warningf("Save incomleted blob descriptor for %s:%s", blobDescr.Name, blobDescr.Digest) - err = oper.mdb.InsertBlob(ctx, &blobDescr) - if err != nil { - return res, http.StatusInternalServerError, err - } - } - } - - res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference) - return res, http.StatusCreated, err -} - -type GetManifestParams struct { - Name string - Reference string -} -type GetManifestResult struct { - ContentLength string - ContentType string - DockerContentDigest string - Payload string -} - -func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams) (*GetManifestResult, int, error) { - var err error - res := &GetManifestResult{} - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - manifestDescr := descr.Manifest{} - var exists bool - // TODO: checking layers? - if stringLikeSHADigest(params.Reference) { - digest := normalizeSHADigest(params.Reference) - exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - - manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload) - res.DockerContentDigest = manifestDigest.String() - - res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10) - res.ContentType = manifestDescr.ContentType - res.Payload = manifestDescr.Payload - - } else { - // Create index of manifests - exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - /* - index, indexBytes, err := indexFromManigestDescrs(manifestDescrs) - if err != nil { - return res, http.StatusInternalServerError, err - } - indexDigest := auxoci.SHA256DigestFromString(indexBytes) - res.DockerContentDigest = indexDigest.String() - - res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10) - res.ContentType = index.MediaType - res.Payload = string(indexBytes) - */ - manifestDescr = manifestDescrs[0] - - manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload) - res.DockerContentDigest = manifestDigest.String() - - res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10) - res.ContentType = manifestDescr.ContentType - res.Payload = manifestDescr.Payload - - } - return res, http.StatusOK, err -} - -type DeleteManifestParams struct { - Name string - Reference string -} -type DeleteManifestResult struct{} - -func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifestParams) (*DeleteManifestResult, int, error) { - var err error - res := &DeleteManifestResult{} - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Reference == "" { - err = fmt.Errorf("Empty reference") - return res, http.StatusBadRequest, err - } - - resName := params.Name - oper.iLock.WaitAndLock(resName) - defer oper.iLock.Done(resName) - - var exists bool - var reference string - - manifestDescr := descr.Manifest{} - - // Check manifest by digest as name - if stringLikeSHADigest(params.Reference) { - digest := normalizeSHADigest(params.Reference) - exists, manifestDescr, err = oper.mdb.GetManifestByDigest(ctx, params.Name, digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - reference = manifestDescr.Reference - err = oper.deleteManifestObjects(ctx, params.Name, reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - } else { - // Check manifest by name and reference - exists, manifestDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - if !exists { - return res, http.StatusNotFound, err - } - reference = params.Reference - for _, manifestDescr := range manifestDescrs { - reference = manifestDescr.Reference - err = oper.deleteManifestObjects(ctx, params.Name, reference) - if err != nil { - return res, http.StatusInternalServerError, err - } - } - } - // Get blobs associated with the name - return res, http.StatusAccepted, err -} - -func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference string) error { - var err error - // Get blobs associated with the name - layers, err := oper.mdb.GetBlobsByReferense(ctx, name, reference) - if err != nil { - return err - } - for _, blob := range layers { - // Delete descr record - err = oper.mdb.DeleteBlobByNameDigest(ctx, blob.Name, blob.Digest) - if err != nil { - return err - } - // Check blob file - exists, _, err := oper.store.BlobExists(blob.Digest) - if err != nil { - return err - } - if exists { - // Check blob usage - blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest) - if err != nil { - return err - } - // Delete if blob useless - if blobUsage == 0 { - err = oper.store.DeleteBlob(blob.Digest) - if err != nil { - return err - } - } - } - } - err = oper.mdb.DeleteManifest(ctx, name, reference) - if err != nil { - return err - } - return err -} - -type GetTagsParams struct { - Name string -} -type GetTagsResult struct { - TagDescr descr.Tags -} - -func (oper *Operator) GetTags(ctx context.Context, params *GetTagsParams) (*GetTagsResult, int, error) { - var err error - res := &GetTagsResult{ - TagDescr: descr.Tags{ - Name: params.Name, - Tags: make([]string, 0), - }, - } - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - manifestDescrs, err := oper.mdb.ListManifestsByName(ctx, params.Name) - if err != nil { - return res, http.StatusInternalServerError, err - } - for _, manifestDescr := range manifestDescrs { - res.TagDescr.Tags = append(res.TagDescr.Tags, manifestDescr.Reference) - } - return res, http.StatusOK, err -} - -type GetRefererParams struct { - Name string - Digest string -} -type GetRefererResult struct { - Reference string -} - -func (oper *Operator) GetReferer(ctx context.Context, params *GetRefererParams) (*GetRefererResult, int, error) { - var err error - res := &GetRefererResult{} - - if params.Name == "" { - err = fmt.Errorf("Empty name") - return res, http.StatusBadRequest, err - } - if params.Digest == "" { - err = fmt.Errorf("Empty digest") - return res, http.StatusBadRequest, err - } - manifests, err := oper.mdb.GetReferer(ctx, params.Name, params.Digest) - if err != nil { - return res, http.StatusInternalServerError, err - } - if len(manifests) == 0 { - return res, http.StatusNotFound, err - } - res.Reference = manifests[0].Reference - return res, http.StatusOK, err -} - -type ListManifestsParams struct { - N int64 -} -type ListManifestsResult struct { - Repositories []string `json:"repositories"` -} - -func (oper *Operator) ListManifests(ctx context.Context, params *ListManifestsParams) (*ListManifestsResult, int, error) { - var err error - res := &ListManifestsResult{ - Repositories: make([]string, 0), - } - refMap := make(map[string]bool) - descrs, err := oper.mdb.ListAllManifests(ctx) - if err != nil { - return res, http.StatusInternalServerError, err - } - for _, descr := range descrs { - _, already := refMap[descr.Name] - if !already { - refMap[descr.Name] = true - } - } - for key, _ := range refMap { - res.Repositories = append(res.Repositories, key) - } - slices.Sort(res.Repositories) - return res, http.StatusOK, err -} diff --git a/app/imageoper/putman.go b/app/imageoper/putman.go new file mode 100644 index 0000000..28b4c81 --- /dev/null +++ b/app/imageoper/putman.go @@ -0,0 +1,206 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package imageoper + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strconv" + + "mstore/pkg/auxoci" +) + +type PutManifestParams struct { + ContentType string + ContentLength string + Name string + Reference string + Reader io.Reader +} +type PutManifestResult struct { + Location string +} + +const ( + ddmMimeType = "application/vnd.docker.distribution.manifest.v2+json" + oimMimeType = "application/vnd.oci.image.manifest.v1+json" + + XXXoicMimeType = "application/vnd.oci.image.config.v1+json" + XXXdciMimeType = "application/vnd.docker.container.image.v1+json" +) + +// TODO: lock for the name-reference or simular? +func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams) (*PutManifestResult, int, error) { + var err error + res := &PutManifestResult{} + + if params.Reference == "" { + err = fmt.Errorf("Empty reference") + return res, http.StatusBadRequest, err + } + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + + // Check Content-Type + var mimeIsAcceptably bool + mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oimMimeType + mimeIsAcceptably = mimeIsAcceptably || params.ContentType == ddmMimeType + //mimeIsAcceptably = mimeIsAcceptably || params.ContentType == oicMimeType + //mimeIsAcceptably = mimeIsAcceptably || params.ContentType == dciMimeType + if !mimeIsAcceptably { + err = fmt.Errorf("Unknown or empty Content-Type: %s", params.ContentType) + return res, http.StatusNotFound, err + } + if params.ContentLength == "" { + code := http.StatusLengthRequired + err = fmt.Errorf("Content-Length is empty") + return res, code, err + } + contentLength, err := strconv.ParseInt(params.ContentLength, 10, 64) + if err != nil { + err = fmt.Errorf("Cannot parse Content-Length value [%s]: %v", params.ContentLength, err) + code := http.StatusLengthRequired + return res, code, err + } + + resName := params.Name + oper.iLock.WaitAndLock(resName) + defer oper.iLock.Done(resName) + + // Copy manifest data + buffer := bytes.NewBuffer(nil) + _, err = io.Copy(buffer, params.Reader) + if err != nil { + return res, http.StatusInternalServerError, err + } + + incomingManifestBytes := buffer.Bytes() + if int64(len(incomingManifestBytes)) != contentLength { + err = fmt.Errorf("Mismatch Content-Length and received manifest size: %d vs %d", + contentLength, len(incomingManifestBytes)) + code := http.StatusInternalServerError + return res, code, err + } + + if len(incomingManifestBytes) > (4 * 1024 * 1024) { + err = fmt.Errorf("Payload more 4M: %d bytes", len(incomingManifestBytes)) + code := http.StatusRequestEntityTooLarge + return res, code, err + } + + incomingManifest, err := auxoci.ParseOCIManifest(incomingManifestBytes) + if err != nil { + err = fmt.Errorf("Parsing OCI manifest error: %v", err) + return res, http.StatusInternalServerError, err + } + if incomingManifest.MediaType == "" { + incomingManifest.MediaType = params.ContentType + } + + name := params.Name + reference := params.Reference + + arch := incomingManifest.Subject.Platform.Architecture + os := incomingManifest.Subject.Platform.OS + variant := incomingManifest.Subject.Platform.Variant + + manifestExists, existingManifestDescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant) + if err != nil { + return res, http.StatusInternalServerError, err + } + + incomingManifestDescr, incomingLayerDescrs, err := descrsFromManifest(name, reference, incomingManifest, incomingManifestBytes) + // Always check layer files for availability + var blobError error + for _, blobDescr := range incomingLayerDescrs { + blobExists, _, err := oper.store.BlobExists(blobDescr.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !blobExists { + oper.logg.Warningf("Found incomleted blob binary for %s:%s", blobDescr.Name, blobDescr.Digest) + err := fmt.Errorf("Layer %s not found.", blobDescr.Digest) + blobError = errors.Join(blobError, err) + } + } + if blobError != nil { + // TODO: more relevant code? + return res, http.StatusFailedDependency, blobError + } + if !manifestExists { + // Store manifest and layesrs data + err = oper.mdb.InsertManifestWithLayers(ctx, &incomingManifestDescr, incomingLayerDescrs) + if err != nil { + return res, http.StatusInternalServerError, err + } + } else { + /* TODO: only update descr + if bytes.Equal(existingManifestBytes, incomingManifestBytes) { + return res, http.StatusCreated, err + } + */ + + existingManifestBytes := []byte(existingManifestDescr.Payload) + existingManifest, err := auxoci.ParseOCIManifest(existingManifestBytes) + if err != nil { + return res, http.StatusInternalServerError, err + } + addedBlobDescrs, uselessBlobDescrs, err := layersDiff(name, reference, + existingManifest, incomingManifest, incomingManifestBytes) + if err != nil { + return res, http.StatusInternalServerError, err + } + // Starting manifest and blobs transaction + err = oper.mdb.UpdateManifestWithBlobs(ctx, &incomingManifestDescr, addedBlobDescrs, uselessBlobDescrs) + if err != nil { + return res, http.StatusInternalServerError, err + } + for _, blob := range uselessBlobDescrs { + exists, _, err := oper.store.BlobExists(blob.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if exists && blobUsage == 0 { + err = oper.store.DeleteBlob(blob.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + } + } + } + + for _, blobDescr := range incomingLayerDescrs { + // TODO: move the requests to db layer transaction + blobDescrExists, _, err := oper.mdb.GetBlobByNameDigest(ctx, blobDescr.Name, blobDescr.Digest) + if err != nil { + return res, http.StatusInternalServerError, err + } + if !blobDescrExists { + oper.logg.Warningf("Save incomleted blob descriptor for %s:%s", blobDescr.Name, blobDescr.Digest) + err = oper.mdb.InsertBlob(ctx, &blobDescr) + if err != nil { + return res, http.StatusInternalServerError, err + } + } + } + + res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference) + return res, http.StatusCreated, err +} diff --git a/cmd/mstorectl/util.go b/cmd/mstorectl/util.go index 78238ac..f44e48e 100644 --- a/cmd/mstorectl/util.go +++ b/cmd/mstorectl/util.go @@ -22,10 +22,12 @@ import ( ) type Util struct { + // TODO: delete mixed object and use simple object? accountcmd.AccountUtil filecmd.FileUtil imagecmd.ImageUtil accountcmd.GrantUtil + rootCmd *cobra.Command }