app/imageoper, app/maindb: one image - one arch; app/storage: blob to name subdir

This commit is contained in:
2026-03-17 09:40:53 +02:00
parent 13b1905e05
commit 791e04cdf1
28 changed files with 680 additions and 569 deletions
+3 -1
View File
@@ -21,7 +21,7 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
name, _ := rctx.GetSubpath("name")
reference, _ := rctx.GetSubpath("reference")
//hand.DumpHeaders("ManigestExists:\n", rctx)
hand.DumpHeaders("ManigestExists:\n", rctx)
params := &imageoper.ManifestExistsParams{
Name: name,
@@ -48,6 +48,8 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
rctx.SetHeader("Content-Type", res.ContentType)
rctx.SetHeader("Docker-Content-Digest", res.DockerContentDigest)
}
hand.logg.Debugf(res.ContentType)
hand.logg.Debugf(res.DockerContentDigest)
rctx.SetStatus(code)
}
+9 -7
View File
@@ -10,7 +10,7 @@
package handler
import (
"net/http"
//"net/http"
"mstore/app/imageoper"
"mstore/app/router"
@@ -21,12 +21,14 @@ func (hand *Handler) GetVersion(rctx *router.Context) {
params := &imageoper.GetVersionParams{}
//hand.DumpHeaders("GetVersion", rctx)
authorization := rctx.GetHeader("Authorization")
if authorization == "" {
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
rctx.SetStatus(http.StatusUnauthorized)
return
}
/*
authorization := rctx.GetHeader("Authorization")
if authorization == "" {
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
rctx.SetStatus(http.StatusUnauthorized)
return
}
*/
ctx := rctx.GetContext()
_, code, err := hand.imop.GetVersion(ctx, params)
if err != nil {
+1 -1
View File
@@ -52,7 +52,7 @@ func (oper *Operator) BlobExists(ctx context.Context, operatorID string, params
return res, http.StatusNotFound, err
}
// Check blob file
blobExists, _, err := oper.store.BlobExists(params.Digest)
blobExists, _, err := oper.store.BlobExists(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
+2 -2
View File
@@ -59,13 +59,13 @@ func (oper *Operator) DeleteBlob(ctx context.Context, operatorID string, params
return res, http.StatusInternalServerError, err
}
// Removing the blob binary if usage == 0
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if blobUsage == 0 {
oper.logg.Warningf("Deleting useless blob binary %s", params.Digest)
oper.store.DeleteBlob(params.Digest)
oper.store.DeleteBlob(params.Name, params.Digest)
}
return res, http.StatusOK, err
}
+25 -11
View File
@@ -64,21 +64,35 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
return res, http.StatusInternalServerError, err
}
} else {
/*
// Check manifest by name and reference
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
reference = params.Reference
for _, mandescr := range mandescrs {
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
*/
// Check manifest by name and reference
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
exists, mandescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
reference = params.Reference
for _, mandescr := range mandescrs {
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
err = oper.deleteManifestObjects(ctx, mandescr.Name, mandescr.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
// Get blobs associated with the name
@@ -99,19 +113,19 @@ func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference
return err
}
// Check blob file
exists, _, err := oper.store.BlobExists(blob.Digest)
exists, _, err := oper.store.BlobExists(name, blob.Digest)
if err != nil {
return err
}
if exists {
// Check blob usage
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
if err != nil {
return err
}
// Delete if blob useless
if blobUsage == 0 {
err = oper.store.DeleteBlob(blob.Digest)
err = oper.store.DeleteBlob(name, blob.Digest)
if err != nil {
return err
}
+2 -2
View File
@@ -45,7 +45,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
oper.iLock.WaitAndLock(resName)
defer oper.iLock.Done(resName)
blobExists, blobSize, err := oper.store.BlobExists(params.Digest)
blobExists, blobSize, err := oper.store.BlobExists(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -57,7 +57,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
return res, http.StatusNotFound, err
}
_, readCloser, err := oper.store.BlobReader(params.Digest)
_, readCloser, err := oper.store.BlobReader(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
+36 -21
View File
@@ -66,34 +66,49 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
res.Payload = manDescr.Payload
} else {
// Create index of manifests. Or not.
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
/*
// Create index of manifests. Or not.
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr = manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Payload = string(indexdata)
}
*/
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr = manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Payload = string(indexdata)
}
}
return res, http.StatusOK, err
}
+32 -11
View File
@@ -62,24 +62,45 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
res.DockerContentDigest = man.Digest
res.Exists = exist
} else {
// Create index of manifests
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
/*
// Create index of manifests
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr := manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Exists = true
}
*/
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Exists = true
res.ContentType = manDescr.ContentType
}
return res, http.StatusOK, err
}
+14 -16
View File
@@ -111,21 +111,26 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
name := params.Name
reference := params.Reference
arch := inMan.Subject.Platform.Architecture
os := inMan.Subject.Platform.OS
variant := inMan.Subject.Platform.Variant
/*
arch := inMan.Subject.Platform.Architecture
os := inMan.Subject.Platform.OS
variant := inMan.Subject.Platform.Variant
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
if err != nil {
return res, http.StatusInternalServerError, err
}
*/
manexist, exMandescr, err := oper.mdb.GetManifestByReference(ctx, name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
inManDescr.Digest = digstr
// Always check layer files for availability
var blobError error
for _, blobDescr := range inlayerdescrs {
blobExists, _, err := oper.store.BlobExists(blobDescr.Digest)
blobExists, _, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -151,7 +156,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
return res, http.StatusCreated, err
}
*/
exManData := []byte(exMandescr.Payload)
exMan := &ocispec.Manifest{}
err := json.Unmarshal(exManData, exMan)
@@ -163,25 +167,22 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
if err != nil {
return res, http.StatusInternalServerError, err
}
// Starting manifest and blobs transaction
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
//goto end
for _, blob := range uselessBlobDescrs {
exists, _, err := oper.store.BlobExists(blob.Digest)
exists, _, err := oper.store.BlobExists(blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if exists && blobUsage == 0 {
err = oper.store.DeleteBlob(blob.Digest)
err = oper.store.DeleteBlob(blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -202,9 +203,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
}
}
}
//end:
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
return res, http.StatusCreated, err
}
+1 -1
View File
@@ -80,7 +80,7 @@ func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *
res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest)
}
err = oper.store.LinkUpload(params.Reference, params.Digest)
err = oper.store.LinkUpload(params.Reference, params.Name, params.Digest)
if err != nil {
err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err)
return res, http.StatusInternalServerError, err
+3 -3
View File
@@ -103,12 +103,12 @@ func (db *Database) GetBlobsByReferense(ctx context.Context, name, reference str
return blobs, err
}
func (db *Database) GetBlobUsage(ctx context.Context, digest string) (int64, error) {
func (db *Database) GetBlobUsage(ctx context.Context, name, digest string) (int64, error) {
var err error
var usage int64
count := make([]int64, 0)
request := `SELECT count(id) AS count FROM blobs WHERE digest = $1`
err = db.db.Select(&count, request, digest)
request := `SELECT count(id) AS count FROM blobs WHERE name = $1 AND digest = $2`
err = db.db.Select(&count, request, name, digest)
if err != nil {
return usage, err
}
+6 -5
View File
@@ -209,23 +209,24 @@ func (db *Database) GetManifestByDigest(ctx context.Context, name, digest string
return exists, manifest, err
}
func (db *Database) GetManifestsByReference(ctx context.Context, name, reference string) (bool, []descr.Manifest, error) {
func (db *Database) GetManifestByReference(ctx context.Context, name, reference string) (bool, descr.Manifest, error) {
var err error
exists := false
manifest := descr.Manifest{}
manifests := make([]descr.Manifest, 0)
request := `SELECT * FROM manifests WHERE name = $1 AND reference = $2 LIMIT 1`
err = db.db.Select(&manifests, request, name, reference)
if err != nil {
return exists, manifests, err
return exists, manifest, err
}
if len(manifests) > 0 {
manifest = manifests[0]
exists = true
}
return exists, manifests, err
return exists, manifest, err
}
func (db *Database) GetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
func (db *Database) xxxGetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
var err error
exists := false
manifest := descr.Manifest{}
+4 -1
View File
@@ -43,8 +43,11 @@ const schema = `
os VARCHAR(255) NOT NULL,
variant VARCHAR(255) NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index01
ON manifests(name, reference, architecture, os, variant);
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index02
ON manifests(name, reference);
CREATE TABLE IF NOT EXISTS blobs (
id VARCHAR(255) NOT NULL,
+211
View File
@@ -0,0 +1,211 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"io"
"os"
"path/filepath"
)
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(name, digstr string) string {
return filepath.Join(store.basepath, blobsubdir, name, digstr) + ".bin"
}
func (store *Storage) makeBlobsubdir(name string) string {
return filepath.Join(store.basepath, blobsubdir, name)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, name, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir(name)
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(name, digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(name, digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir(name)
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(name, digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(name, digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(name, digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(name, digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(name, digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(name, digest string) error {
var err error
blobPath := store.makeBlobpath(name, digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
// TODO: remove dirs
return err
}
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"fmt"
"io"
"os"
"path/filepath"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
-341
View File
@@ -1,341 +0,0 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"mstore/app/logger"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(upname string) string {
return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
}
func (store *Storage) makeBlobsubdir() string {
return filepath.Join(store.basepath, blobsubdir)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir()
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir()
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(digest string) error {
var err error
blobPath := store.makeBlobpath(digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
return err
}
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"mstore/app/logger"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}