mstore/app/storage/storage.go

package storage

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"mstore/app/logger"
	"mstore/pkg/auxuuid"
)

const (
	sha256prefix = "sha256:"
	filesubdir   = "files"
	tmpsubdir    = "tmps"
)

type Storage struct {
	basepath string
	logg     *logger.Logger
}

func NewStorage(basepath string) *Storage {
	res := &Storage{
		basepath: basepath,
	}
	res.logg = logger.NewLoggerWithSubject("storage")
	return res
}

func (store *Storage) makeCollecionpath(collection string) string {
	return filepath.Join(store.basepath, filesubdir, collection)
}

func (store *Storage) makeFilepath(collection, filename string) string {
	return filepath.Join(store.basepath, filesubdir, collection, filename)
}

func (store *Storage) makeTmppath(tmpname string) string {
	return filepath.Join(store.basepath, tmpsubdir, tmpname)
}

func (store *Storage) makeTmpsubdir() string {
	return filepath.Join(store.basepath, tmpsubdir)
}

func (store *Storage) makeFilesubdir(collection, filename string) string {
	return filepath.Join(store.basepath, filesubdir)
}

func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
	var err error
	var res io.ReadCloser

	filename = store.makeFilepath(collection, filename)
	file, err := os.OpenFile(filename, os.O_RDONLY, 0)
	if err != nil {
		return res, err
	}
	res = file
	return res, err
}

func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
	var err error
	var size int64
	var csum string

	tmpname := auxuuid.NewUUID()
	tmpname = fmt.Sprintf("file-%s.tmp", tmpname)
	tmppath := store.makeTmppath(tmpname)

	tmpdirpath := store.makeTmpsubdir()
	err = os.MkdirAll(tmpdirpath, 0750)
	if err != nil {
		return tmpname, size, csum, err
	}

	file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		return tmpname, size, csum, err
	}
	defer file.Close()

	hasher := sha256.New()
	writer := io.MultiWriter(file, hasher)

	size, err = io.Copy(writer, source)
	if err != nil {
		return tmpname, size, csum, err
	}
	csum = hex.EncodeToString(hasher.Sum(nil))
	csum = sha256prefix + csum

	return tmpname, size, csum, err
}

func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
	var err error

	dirname := store.makeCollecionpath(collection)
	_, err = os.Stat(dirname)
	if errors.Is(err, os.ErrNotExist) {
		err = os.MkdirAll(dirname, 0750)
		if err != nil {
			return err
		}
	}
	if err != nil {
		return err
	}

	filename = store.makeFilepath(collection, filename)
	os.Remove(filename) // TODO

	tmpname = store.makeTmppath(tmpname)
	err = os.Link(tmpname, filename)
	if err != nil {
		return err
	}
	err = os.Remove(tmpname)
	if err != nil {
		return err
	}
	return err
}

func (store *Storage) DeleteFile(collection, filename string) error {
	var err error
	filename = store.makeFilepath(collection, filename)
	err = os.Remove(filename)
	if err != nil {
		return err
	}
	// TODO: safe dir removing
	dirname := store.makeCollecionpath(collection)
	err = os.RemoveAll(dirname)
	if err != nil {
		return err
	}
	return err
}

const (
	upsubdir   = "uploads"
	blobsubdir = "blobs"
)

func (store *Storage) makeUppath(upname string) string {
	return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}

func (store *Storage) makeUpsubdir() string {
	return filepath.Join(store.basepath, upsubdir)
}

func (store *Storage) makeBlobpath(upname string) string {
	return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
}

func (store *Storage) makeBlobsubdir() string {
	return filepath.Join(store.basepath, blobsubdir)
}

func (store *Storage) WriteUpload(uploadID string, source io.Reader) (int64, string, error) {
	var err error
	var recsize int64
	var recsum string

	uploadDir := store.makeUpsubdir()
	_, err = os.Stat(uploadDir)
	if errors.Is(err, os.ErrNotExist) {
		err = os.MkdirAll(uploadDir, 0750)
		if err != nil {
			return recsize, recsum, err
		}
	}
	if err != nil {
		return recsize, recsum, err
	}

	uploadPath := store.makeUppath(uploadID)
	uploadFile, err := os.OpenFile(uploadPath, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return recsize, recsum, err
	}
	defer uploadFile.Close()

	hasher := sha256.New() // TODO: upload cheking
	streamWriter := io.MultiWriter(uploadFile, hasher)
	recsize, err = io.Copy(streamWriter, source)
	if err != nil {
		return recsize, recsum, err
	}
	recsum = hex.EncodeToString(hasher.Sum(nil))
	recsum = sha256prefix + recsum

	return recsize, recsum, err
}

func (store *Storage) LinkUpload(reference, digest string) error {
	var err error
	uploadPath := store.makeUppath(reference)

	blobdir := store.makeBlobsubdir()
	_, err = os.Stat(blobdir)
	if errors.Is(err, os.ErrNotExist) {
		err = os.MkdirAll(blobdir, 0750)
		if err != nil {
			return err
		}
	}
	if err != nil {
		return err
	}
	blobPath := store.makeBlobpath(digest)
	_, err = os.Stat(blobPath)
	if err == nil {
		err = os.Remove(blobPath)
		if err != nil {
			return err
		}
	}
	if errors.Is(err, os.ErrNotExist) {
		err = nil
	}
	if err != nil {
		return err
	}

	err = os.Link(uploadPath, blobPath)
	if err != nil {
		return err
	}
	err = os.Remove(uploadPath)
	if err != nil {
		return err
	}
	return err
}

func (store *Storage) RemoveUpload(digest string) error {
	var err error
	uploadPath := store.makeUppath(digest)
	err = os.Remove(uploadPath)
	if err != nil {
		return err
	}
	return err
}

func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
	var err error
	var fileSize int64

	uploadPath := st.makeUppath(reference)

	fileStat, err := os.Stat(uploadPath)
	if errors.Is(err, os.ErrNotExist) {
		return false, 0, nil
	}
	if err != nil {
		return false, 0, err
	}

	fileSize = fileStat.Size()
	return true, fileSize, err
}

func (store *Storage) WriteBlob(digest string, source io.Reader) (int64, string, error) {
	var err error
	var recsize int64
	var recsum string

	//defer source.Close()

	blobdir := store.makeBlobsubdir()
	_, err = os.Stat(blobdir)
	if errors.Is(err, os.ErrNotExist) {
		err = os.MkdirAll(blobdir, 0750)
		if err != nil {
			return recsize, digest, err
		}
	}
	if err != nil {
		return recsize, digest, err
	}

	blobpath := store.makeBlobpath(digest)
	blobfile, err := os.OpenFile(blobpath, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return recsize, digest, err
	}
	defer blobfile.Close()

	hasher := sha256.New() // TODO
	multiWriter := io.MultiWriter(blobfile, hasher)
	recsize, err = io.Copy(multiWriter, source)
	if err != nil {
		return recsize, digest, err
	}
	recsum = hex.EncodeToString(hasher.Sum(nil))
	recsum = sha256prefix + recsum

	return recsize, recsum, err
}

func (st *Storage) BlobExists(digest string) (bool, int64, error) {
	var err error
	var fileSize int64

	blobPath := st.makeBlobpath(digest)

	fileStat, err := os.Stat(blobPath)
	if errors.Is(err, os.ErrNotExist) {
		return false, 0, nil
	}
	if err != nil {
		return false, 0, err
	}

	fileSize = fileStat.Size()
	return true, fileSize, err
}

func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
	var err error
	var filesize int64
	blobpath := store.makeBlobpath(digest)

	emptyReadCloser := io.NopCloser(bytes.NewReader(nil))

	file, err := os.OpenFile(blobpath, os.O_RDONLY, 0)
	if err != nil {
		return filesize, emptyReadCloser, err
	}
	defer func() {
		if err != nil {
			file.Close()
		}
	}()
	filestat, err := file.Stat()
	if err != nil {
		return filesize, emptyReadCloser, err
	}
	filesize = filestat.Size()
	return filesize, file, err
}

func (store *Storage) DeleteBlob(digest string) error {
	var err error
	blobpath := store.makeBlobpath(digest)
	err = os.Remove(blobpath)
	if errors.Is(err, os.ErrNotExist) {
		err = nil
	}
	if err != nil {
		return err
	}
	return err
}