app/imageoper, app/maindb: one image - one arch; app/storage: blob to name subdir

This commit is contained in:
2026-03-17 09:40:53 +02:00
parent 13b1905e05
commit 791e04cdf1
28 changed files with 680 additions and 569 deletions
+211
View File
@@ -0,0 +1,211 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"io"
"os"
"path/filepath"
)
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(name, digstr string) string {
return filepath.Join(store.basepath, blobsubdir, name, digstr) + ".bin"
}
func (store *Storage) makeBlobsubdir(name string) string {
return filepath.Join(store.basepath, blobsubdir, name)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, name, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir(name)
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(name, digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(name, digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir(name)
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(name, digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(name, digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(name, digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(name, digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(name, digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(name, digest string) error {
var err error
blobPath := store.makeBlobpath(name, digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
// TODO: remove dirs
return err
}
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"fmt"
"io"
"os"
"path/filepath"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
-341
View File
@@ -1,341 +0,0 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"mstore/app/logger"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(upname string) string {
return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
}
func (store *Storage) makeBlobsubdir() string {
return filepath.Join(store.basepath, blobsubdir)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir()
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir()
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(digest string) error {
var err error
blobPath := store.makeBlobpath(digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
return err
}
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"mstore/app/logger"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}