app/imageoper, app/maindb: one image - one arch; app/storage: blob to name subdir
This commit is contained in:
@@ -21,7 +21,7 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
|
||||
name, _ := rctx.GetSubpath("name")
|
||||
reference, _ := rctx.GetSubpath("reference")
|
||||
|
||||
//hand.DumpHeaders("ManigestExists:\n", rctx)
|
||||
hand.DumpHeaders("ManigestExists:\n", rctx)
|
||||
|
||||
params := &imageoper.ManifestExistsParams{
|
||||
Name: name,
|
||||
@@ -48,6 +48,8 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
|
||||
rctx.SetHeader("Content-Type", res.ContentType)
|
||||
rctx.SetHeader("Docker-Content-Digest", res.DockerContentDigest)
|
||||
}
|
||||
hand.logg.Debugf(res.ContentType)
|
||||
hand.logg.Debugf(res.DockerContentDigest)
|
||||
rctx.SetStatus(code)
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
//"net/http"
|
||||
|
||||
"mstore/app/imageoper"
|
||||
"mstore/app/router"
|
||||
@@ -21,12 +21,14 @@ func (hand *Handler) GetVersion(rctx *router.Context) {
|
||||
params := &imageoper.GetVersionParams{}
|
||||
|
||||
//hand.DumpHeaders("GetVersion", rctx)
|
||||
authorization := rctx.GetHeader("Authorization")
|
||||
if authorization == "" {
|
||||
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
|
||||
rctx.SetStatus(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
/*
|
||||
authorization := rctx.GetHeader("Authorization")
|
||||
if authorization == "" {
|
||||
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
|
||||
rctx.SetStatus(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
*/
|
||||
ctx := rctx.GetContext()
|
||||
_, code, err := hand.imop.GetVersion(ctx, params)
|
||||
if err != nil {
|
||||
|
||||
@@ -52,7 +52,7 @@ func (oper *Operator) BlobExists(ctx context.Context, operatorID string, params
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
// Check blob file
|
||||
blobExists, _, err := oper.store.BlobExists(params.Digest)
|
||||
blobExists, _, err := oper.store.BlobExists(params.Name, params.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
@@ -59,13 +59,13 @@ func (oper *Operator) DeleteBlob(ctx context.Context, operatorID string, params
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
// Removing the blob binary if usage == 0
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Digest)
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Name, params.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if blobUsage == 0 {
|
||||
oper.logg.Warningf("Deleting useless blob binary %s", params.Digest)
|
||||
oper.store.DeleteBlob(params.Digest)
|
||||
oper.store.DeleteBlob(params.Name, params.Digest)
|
||||
}
|
||||
return res, http.StatusOK, err
|
||||
}
|
||||
|
||||
+25
-11
@@ -64,21 +64,35 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
// Check manifest by name and reference
|
||||
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
reference = params.Reference
|
||||
for _, mandescr := range mandescrs {
|
||||
reference = mandescr.Reference
|
||||
err = oper.deleteManifestObjects(ctx, params.Name, reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
}
|
||||
*/
|
||||
// Check manifest by name and reference
|
||||
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
exists, mandescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
reference = params.Reference
|
||||
for _, mandescr := range mandescrs {
|
||||
reference = mandescr.Reference
|
||||
err = oper.deleteManifestObjects(ctx, params.Name, reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
err = oper.deleteManifestObjects(ctx, mandescr.Name, mandescr.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
}
|
||||
// Get blobs associated with the name
|
||||
@@ -99,19 +113,19 @@ func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference
|
||||
return err
|
||||
}
|
||||
// Check blob file
|
||||
exists, _, err := oper.store.BlobExists(blob.Digest)
|
||||
exists, _, err := oper.store.BlobExists(name, blob.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
// Check blob usage
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Delete if blob useless
|
||||
if blobUsage == 0 {
|
||||
err = oper.store.DeleteBlob(blob.Digest)
|
||||
err = oper.store.DeleteBlob(name, blob.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
|
||||
oper.iLock.WaitAndLock(resName)
|
||||
defer oper.iLock.Done(resName)
|
||||
|
||||
blobExists, blobSize, err := oper.store.BlobExists(params.Digest)
|
||||
blobExists, blobSize, err := oper.store.BlobExists(params.Name, params.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
|
||||
_, readCloser, err := oper.store.BlobReader(params.Digest)
|
||||
_, readCloser, err := oper.store.BlobReader(params.Name, params.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
+36
-21
@@ -66,34 +66,49 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
|
||||
res.Payload = manDescr.Payload
|
||||
|
||||
} else {
|
||||
// Create index of manifests. Or not.
|
||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
/*
|
||||
// Create index of manifests. Or not.
|
||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
if len(manDescrs) == 1 {
|
||||
manDescr = manDescrs[0]
|
||||
res.DockerContentDigest = manDescr.Digest
|
||||
size := int64(len(manDescr.Payload))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = manDescr.ContentType
|
||||
res.Payload = manDescr.Payload
|
||||
} else {
|
||||
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
digobj := ocidigest.FromBytes(indexdata)
|
||||
res.DockerContentDigest = digobj.String()
|
||||
|
||||
size := int64(len(indexdata))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = oiiMediaType
|
||||
res.Payload = string(indexdata)
|
||||
}
|
||||
*/
|
||||
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
if len(manDescrs) == 1 {
|
||||
manDescr = manDescrs[0]
|
||||
res.DockerContentDigest = manDescr.Digest
|
||||
size := int64(len(manDescr.Payload))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = manDescr.ContentType
|
||||
res.Payload = manDescr.Payload
|
||||
} else {
|
||||
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
digobj := ocidigest.FromBytes(indexdata)
|
||||
res.DockerContentDigest = digobj.String()
|
||||
res.DockerContentDigest = manDescr.Digest
|
||||
size := int64(len(manDescr.Payload))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = manDescr.ContentType
|
||||
res.Payload = manDescr.Payload
|
||||
|
||||
size := int64(len(indexdata))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = oiiMediaType
|
||||
res.Payload = string(indexdata)
|
||||
}
|
||||
}
|
||||
return res, http.StatusOK, err
|
||||
}
|
||||
|
||||
+32
-11
@@ -62,24 +62,45 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
|
||||
res.DockerContentDigest = man.Digest
|
||||
res.Exists = exist
|
||||
} else {
|
||||
// Create index of manifests
|
||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
/*
|
||||
// Create index of manifests
|
||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
if len(manDescrs) == 1 {
|
||||
manDescr := manDescrs[0]
|
||||
res.DockerContentDigest = manDescr.Digest
|
||||
size := int64(len(manDescr.Payload))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = manDescr.ContentType
|
||||
} else {
|
||||
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
digobj := ocidigest.FromBytes(indexdata)
|
||||
res.DockerContentDigest = digobj.String()
|
||||
size := int64(len(indexdata))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = oiiMediaType
|
||||
res.Exists = true
|
||||
}
|
||||
*/
|
||||
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if !exists {
|
||||
return res, http.StatusNotFound, err
|
||||
}
|
||||
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
digobj := ocidigest.FromBytes(indexdata)
|
||||
res.DockerContentDigest = digobj.String()
|
||||
size := int64(len(indexdata))
|
||||
res.DockerContentDigest = manDescr.Digest
|
||||
size := int64(len(manDescr.Payload))
|
||||
res.ContentLength = strconv.FormatInt(size, 10)
|
||||
res.ContentType = oiiMediaType
|
||||
res.Exists = true
|
||||
res.ContentType = manDescr.ContentType
|
||||
}
|
||||
return res, http.StatusOK, err
|
||||
}
|
||||
|
||||
+14
-16
@@ -111,21 +111,26 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
||||
name := params.Name
|
||||
reference := params.Reference
|
||||
|
||||
arch := inMan.Subject.Platform.Architecture
|
||||
os := inMan.Subject.Platform.OS
|
||||
variant := inMan.Subject.Platform.Variant
|
||||
/*
|
||||
arch := inMan.Subject.Platform.Architecture
|
||||
os := inMan.Subject.Platform.OS
|
||||
variant := inMan.Subject.Platform.Variant
|
||||
|
||||
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
|
||||
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
*/
|
||||
manexist, exMandescr, err := oper.mdb.GetManifestByReference(ctx, name, reference)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
|
||||
inManDescr.Digest = digstr
|
||||
// Always check layer files for availability
|
||||
var blobError error
|
||||
for _, blobDescr := range inlayerdescrs {
|
||||
blobExists, _, err := oper.store.BlobExists(blobDescr.Digest)
|
||||
blobExists, _, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
@@ -151,7 +156,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
||||
return res, http.StatusCreated, err
|
||||
}
|
||||
*/
|
||||
|
||||
exManData := []byte(exMandescr.Payload)
|
||||
exMan := &ocispec.Manifest{}
|
||||
err := json.Unmarshal(exManData, exMan)
|
||||
@@ -163,25 +167,22 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
// Starting manifest and blobs transaction
|
||||
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
//goto end
|
||||
for _, blob := range uselessBlobDescrs {
|
||||
exists, _, err := oper.store.BlobExists(blob.Digest)
|
||||
exists, _, err := oper.store.BlobExists(blob.Name, blob.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
|
||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
if exists && blobUsage == 0 {
|
||||
err = oper.store.DeleteBlob(blob.Digest)
|
||||
err = oper.store.DeleteBlob(blob.Name, blob.Digest)
|
||||
if err != nil {
|
||||
return res, http.StatusInternalServerError, err
|
||||
}
|
||||
@@ -202,9 +203,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//end:
|
||||
|
||||
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
|
||||
return res, http.StatusCreated, err
|
||||
}
|
||||
|
||||
@@ -80,7 +80,7 @@ func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *
|
||||
res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest)
|
||||
}
|
||||
|
||||
err = oper.store.LinkUpload(params.Reference, params.Digest)
|
||||
err = oper.store.LinkUpload(params.Reference, params.Name, params.Digest)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err)
|
||||
return res, http.StatusInternalServerError, err
|
||||
|
||||
+3
-3
@@ -103,12 +103,12 @@ func (db *Database) GetBlobsByReferense(ctx context.Context, name, reference str
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
func (db *Database) GetBlobUsage(ctx context.Context, digest string) (int64, error) {
|
||||
func (db *Database) GetBlobUsage(ctx context.Context, name, digest string) (int64, error) {
|
||||
var err error
|
||||
var usage int64
|
||||
count := make([]int64, 0)
|
||||
request := `SELECT count(id) AS count FROM blobs WHERE digest = $1`
|
||||
err = db.db.Select(&count, request, digest)
|
||||
request := `SELECT count(id) AS count FROM blobs WHERE name = $1 AND digest = $2`
|
||||
err = db.db.Select(&count, request, name, digest)
|
||||
if err != nil {
|
||||
return usage, err
|
||||
}
|
||||
|
||||
@@ -209,23 +209,24 @@ func (db *Database) GetManifestByDigest(ctx context.Context, name, digest string
|
||||
return exists, manifest, err
|
||||
}
|
||||
|
||||
func (db *Database) GetManifestsByReference(ctx context.Context, name, reference string) (bool, []descr.Manifest, error) {
|
||||
func (db *Database) GetManifestByReference(ctx context.Context, name, reference string) (bool, descr.Manifest, error) {
|
||||
var err error
|
||||
exists := false
|
||||
manifest := descr.Manifest{}
|
||||
manifests := make([]descr.Manifest, 0)
|
||||
request := `SELECT * FROM manifests WHERE name = $1 AND reference = $2 LIMIT 1`
|
||||
err = db.db.Select(&manifests, request, name, reference)
|
||||
if err != nil {
|
||||
return exists, manifests, err
|
||||
return exists, manifest, err
|
||||
}
|
||||
|
||||
if len(manifests) > 0 {
|
||||
manifest = manifests[0]
|
||||
exists = true
|
||||
}
|
||||
return exists, manifests, err
|
||||
return exists, manifest, err
|
||||
}
|
||||
|
||||
func (db *Database) GetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
|
||||
func (db *Database) xxxGetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
|
||||
var err error
|
||||
exists := false
|
||||
manifest := descr.Manifest{}
|
||||
|
||||
@@ -43,8 +43,11 @@ const schema = `
|
||||
os VARCHAR(255) NOT NULL,
|
||||
variant VARCHAR(255) NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index01
|
||||
ON manifests(name, reference, architecture, os, variant);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index02
|
||||
ON manifests(name, reference);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
id VARCHAR(255) NOT NULL,
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||
*
|
||||
* This work is published and licensed under a Creative Commons
|
||||
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||
*
|
||||
* Distribution of this work is permitted, but commercial use and
|
||||
* modifications are strictly prohibited.
|
||||
*/
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
upsubdir = "uploads"
|
||||
blobsubdir = "blobs"
|
||||
)
|
||||
|
||||
func (store *Storage) makeUppath(upname string) string {
|
||||
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
|
||||
}
|
||||
|
||||
func (store *Storage) makeUpsubdir() string {
|
||||
return filepath.Join(store.basepath, upsubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) makeBlobpath(name, digstr string) string {
|
||||
return filepath.Join(store.basepath, blobsubdir, name, digstr) + ".bin"
|
||||
}
|
||||
|
||||
func (store *Storage) makeBlobsubdir(name string) string {
|
||||
return filepath.Join(store.basepath, blobsubdir, name)
|
||||
}
|
||||
|
||||
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
|
||||
var err error
|
||||
var recsize int64
|
||||
|
||||
uploadDir := store.makeUpsubdir()
|
||||
_, err = os.Stat(uploadDir)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(uploadDir, 0750)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
upPath := store.makeUppath(upID)
|
||||
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
defer upFile.Close()
|
||||
recsize, err = io.Copy(upFile, source)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
return recsize, err
|
||||
}
|
||||
|
||||
func (store *Storage) LinkUpload(reference, name, digest string) error {
|
||||
var err error
|
||||
upPath := store.makeUppath(reference)
|
||||
|
||||
blobdir := store.makeBlobsubdir(name)
|
||||
_, err = os.Stat(blobdir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.MkdirAll(blobdir, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobPath := store.makeBlobpath(name, digest)
|
||||
_, err = os.Stat(blobPath)
|
||||
if err == nil {
|
||||
err = os.Remove(blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Link(upPath, blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(upPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Storage) RemoveUpload(digest string) error {
|
||||
var err error
|
||||
upPath := store.makeUppath(digest)
|
||||
err = os.Remove(upPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
upPath := st.makeUppath(reference)
|
||||
fileStat, err := os.Stat(upPath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
size = fileStat.Size()
|
||||
return true, size, err
|
||||
}
|
||||
|
||||
func (store *Storage) WriteBlob(name, digstr string, source io.Reader) (int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
blobDir := store.makeBlobsubdir(name)
|
||||
_, err = os.Stat(blobDir)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(blobDir, 0750)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
blobPath := store.makeBlobpath(name, digstr)
|
||||
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
defer blobFile.Close()
|
||||
size, err = io.Copy(blobFile, source)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (st *Storage) BlobExists(name, digest string) (bool, int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
|
||||
blobPath := st.makeBlobpath(name, digest)
|
||||
fileStat, err := os.Stat(blobPath)
|
||||
if os.IsNotExist(err) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
size = fileStat.Size()
|
||||
return true, size, err
|
||||
}
|
||||
|
||||
func (store *Storage) BlobReader(name, digest string) (int64, io.ReadCloser, error) {
|
||||
var err error
|
||||
var size int64
|
||||
blobPath := store.makeBlobpath(name, digest)
|
||||
nop := io.NopCloser(bytes.NewReader(nil))
|
||||
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return size, nop, err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
file.Close()
|
||||
}
|
||||
}()
|
||||
filestat, err := file.Stat()
|
||||
if err != nil {
|
||||
return size, nop, err
|
||||
}
|
||||
size = filestat.Size()
|
||||
return size, file, err
|
||||
}
|
||||
|
||||
func (store *Storage) DeleteBlob(name, digest string) error {
|
||||
var err error
|
||||
blobPath := store.makeBlobpath(name, digest)
|
||||
err = os.Remove(blobPath)
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: remove dirs
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||
*
|
||||
* This work is published and licensed under a Creative Commons
|
||||
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||
*
|
||||
* Distribution of this work is permitted, but commercial use and
|
||||
* modifications are strictly prohibited.
|
||||
*/
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"mstore/pkg/auxuuid"
|
||||
)
|
||||
|
||||
const (
|
||||
sha256prefix = "sha256:"
|
||||
fileSubdir = "files"
|
||||
tmpSubdir = "tmps"
|
||||
)
|
||||
|
||||
func (store *Storage) makeCollecionpath(collection string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir, collection)
|
||||
}
|
||||
|
||||
func (store *Storage) makeFilepath(collection, filename string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir, collection, filename)
|
||||
}
|
||||
|
||||
func (store *Storage) makeTmppath(tmpName string) string {
|
||||
return filepath.Join(store.basepath, tmpSubdir, tmpName)
|
||||
}
|
||||
|
||||
func (store *Storage) makeTmpsubdir() string {
|
||||
return filepath.Join(store.basepath, tmpSubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) makeFilesubdir(collection, filename string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
||||
var err error
|
||||
var res io.ReadCloser
|
||||
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res = file
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
|
||||
var err error
|
||||
var size int64
|
||||
var csum string
|
||||
|
||||
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
|
||||
tmpPath := store.makeTmppath(tmpName)
|
||||
|
||||
tmpdirpath := store.makeTmpsubdir()
|
||||
err = os.MkdirAll(tmpdirpath, 0750)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := NewHasher()
|
||||
mWriter := io.MultiWriter(file, hasher.Writer())
|
||||
size, err = io.Copy(mWriter, source)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
csum = hasher.Hex()
|
||||
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
|
||||
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
|
||||
var err error
|
||||
|
||||
dirname := store.makeCollecionpath(collection)
|
||||
_, err = os.Stat(dirname)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(dirname, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
_, err = os.Stat(dirname)
|
||||
if err == nil {
|
||||
os.Remove(filename) // TODO: safe removing
|
||||
}
|
||||
tmpName = store.makeTmppath(tmpName)
|
||||
err = os.Link(tmpName, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(tmpName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Storage) DeleteFile(collection, filename string) error {
|
||||
var err error
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
err = os.Remove(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: clean removing
|
||||
dirname := store.makeCollecionpath(collection)
|
||||
os.RemoveAll(dirname)
|
||||
return err
|
||||
}
|
||||
@@ -1,341 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||
*
|
||||
* This work is published and licensed under a Creative Commons
|
||||
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||
*
|
||||
* Distribution of this work is permitted, but commercial use and
|
||||
* modifications are strictly prohibited.
|
||||
*/
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"mstore/app/logger"
|
||||
"mstore/pkg/auxuuid"
|
||||
)
|
||||
|
||||
const (
|
||||
sha256prefix = "sha256:"
|
||||
fileSubdir = "files"
|
||||
tmpSubdir = "tmps"
|
||||
)
|
||||
|
||||
type Storage struct {
|
||||
basepath string
|
||||
logg *logger.Logger
|
||||
}
|
||||
|
||||
func NewStorage(basepath string) *Storage {
|
||||
res := &Storage{
|
||||
basepath: basepath,
|
||||
}
|
||||
res.logg = logger.NewLoggerWithSubject("storage")
|
||||
return res
|
||||
}
|
||||
|
||||
func (store *Storage) makeCollecionpath(collection string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir, collection)
|
||||
}
|
||||
|
||||
func (store *Storage) makeFilepath(collection, filename string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir, collection, filename)
|
||||
}
|
||||
|
||||
func (store *Storage) makeTmppath(tmpName string) string {
|
||||
return filepath.Join(store.basepath, tmpSubdir, tmpName)
|
||||
}
|
||||
|
||||
func (store *Storage) makeTmpsubdir() string {
|
||||
return filepath.Join(store.basepath, tmpSubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) makeFilesubdir(collection, filename string) string {
|
||||
return filepath.Join(store.basepath, fileSubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
||||
var err error
|
||||
var res io.ReadCloser
|
||||
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res = file
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
|
||||
var err error
|
||||
var size int64
|
||||
var csum string
|
||||
|
||||
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
|
||||
tmpPath := store.makeTmppath(tmpName)
|
||||
|
||||
tmpdirpath := store.makeTmpsubdir()
|
||||
err = os.MkdirAll(tmpdirpath, 0750)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := NewHasher()
|
||||
mWriter := io.MultiWriter(file, hasher.Writer())
|
||||
size, err = io.Copy(mWriter, source)
|
||||
if err != nil {
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
csum = hasher.Hex()
|
||||
|
||||
return tmpName, size, csum, err
|
||||
}
|
||||
|
||||
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
|
||||
var err error
|
||||
|
||||
dirname := store.makeCollecionpath(collection)
|
||||
_, err = os.Stat(dirname)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(dirname, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
_, err = os.Stat(dirname)
|
||||
if err == nil {
|
||||
os.Remove(filename) // TODO: safe removing
|
||||
}
|
||||
tmpName = store.makeTmppath(tmpName)
|
||||
err = os.Link(tmpName, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(tmpName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Storage) DeleteFile(collection, filename string) error {
|
||||
var err error
|
||||
filename = store.makeFilepath(collection, filename)
|
||||
err = os.Remove(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: clean removing
|
||||
dirname := store.makeCollecionpath(collection)
|
||||
os.RemoveAll(dirname)
|
||||
return err
|
||||
}
|
||||
|
||||
const (
|
||||
upsubdir = "uploads"
|
||||
blobsubdir = "blobs"
|
||||
)
|
||||
|
||||
func (store *Storage) makeUppath(upname string) string {
|
||||
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
|
||||
}
|
||||
|
||||
func (store *Storage) makeUpsubdir() string {
|
||||
return filepath.Join(store.basepath, upsubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) makeBlobpath(upname string) string {
|
||||
return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
|
||||
}
|
||||
|
||||
func (store *Storage) makeBlobsubdir() string {
|
||||
return filepath.Join(store.basepath, blobsubdir)
|
||||
}
|
||||
|
||||
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
|
||||
var err error
|
||||
var recsize int64
|
||||
|
||||
uploadDir := store.makeUpsubdir()
|
||||
_, err = os.Stat(uploadDir)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(uploadDir, 0750)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
upPath := store.makeUppath(upID)
|
||||
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
defer upFile.Close()
|
||||
recsize, err = io.Copy(upFile, source)
|
||||
if err != nil {
|
||||
return recsize, err
|
||||
}
|
||||
return recsize, err
|
||||
}
|
||||
|
||||
func (store *Storage) LinkUpload(reference, digest string) error {
|
||||
var err error
|
||||
upPath := store.makeUppath(reference)
|
||||
|
||||
blobdir := store.makeBlobsubdir()
|
||||
_, err = os.Stat(blobdir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.MkdirAll(blobdir, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobPath := store.makeBlobpath(digest)
|
||||
_, err = os.Stat(blobPath)
|
||||
if err == nil {
|
||||
err = os.Remove(blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Link(upPath, blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(upPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *Storage) RemoveUpload(digest string) error {
|
||||
var err error
|
||||
upPath := store.makeUppath(digest)
|
||||
err = os.Remove(upPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
upPath := st.makeUppath(reference)
|
||||
fileStat, err := os.Stat(upPath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
size = fileStat.Size()
|
||||
return true, size, err
|
||||
}
|
||||
|
||||
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
blobDir := store.makeBlobsubdir()
|
||||
_, err = os.Stat(blobDir)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(blobDir, 0750)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
blobPath := store.makeBlobpath(digstr)
|
||||
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
defer blobFile.Close()
|
||||
size, err = io.Copy(blobFile, source)
|
||||
if err != nil {
|
||||
return size, err
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
|
||||
var err error
|
||||
var size int64
|
||||
|
||||
blobPath := st.makeBlobpath(digest)
|
||||
fileStat, err := os.Stat(blobPath)
|
||||
if os.IsNotExist(err) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
size = fileStat.Size()
|
||||
return true, size, err
|
||||
}
|
||||
|
||||
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
|
||||
var err error
|
||||
var size int64
|
||||
blobPath := store.makeBlobpath(digest)
|
||||
nop := io.NopCloser(bytes.NewReader(nil))
|
||||
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return size, nop, err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
file.Close()
|
||||
}
|
||||
}()
|
||||
filestat, err := file.Stat()
|
||||
if err != nil {
|
||||
return size, nop, err
|
||||
}
|
||||
size = filestat.Size()
|
||||
return size, file, err
|
||||
}
|
||||
|
||||
func (store *Storage) DeleteBlob(digest string) error {
|
||||
var err error
|
||||
blobPath := store.makeBlobpath(digest)
|
||||
err = os.Remove(blobPath)
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||
*
|
||||
* This work is published and licensed under a Creative Commons
|
||||
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||
*
|
||||
* Distribution of this work is permitted, but commercial use and
|
||||
* modifications are strictly prohibited.
|
||||
*/
|
||||
package storage
|
||||
|
||||
import (
|
||||
"mstore/app/logger"
|
||||
)
|
||||
|
||||
type Storage struct {
|
||||
basepath string
|
||||
logg *logger.Logger
|
||||
}
|
||||
|
||||
func NewStorage(basepath string) *Storage {
|
||||
res := &Storage{
|
||||
basepath: basepath,
|
||||
}
|
||||
res.logg = logger.NewLoggerWithSubject("storage")
|
||||
return res
|
||||
}
|
||||
Reference in New Issue
Block a user