ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

app: added helm tgz handle

Browse Source
This commit is contained in:
Олег Бородин
2026-03-12 16:59:29 +02:00
parent 0d67944966
commit 95ed9ddb97
3182 changed files with 957097 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fileoper/putfile.go
+13 -1
View File
@@ -31,7 +31,10 @@ type PutFileParams struct {
}
type PutFileResult struct{}
const defaultContentType = "application/octet-stream"
const (
defaultContentType = "application/octet-stream"
hcMimeType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
)
// TODO: checking catalog and file names conflict
func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *PutFileParams) (int, *PutFileResult, error) {
@@ -71,6 +74,11 @@ func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *Pu
code := http.StatusInternalServerError
return code, res, err
}
var helmHash string
var helmMeta string
if contentType == hcMimeType {
helmHash, helmMeta, err = oper.store.HelmMeta(tmpname)
}
descrExists, fileDescr, err := oper.mdb.GetFileByCollectionName(ctx, collection, filename)
if err != nil {
@@ -85,6 +93,8 @@ func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *Pu
fileDescr.UpdatedAt = now
fileDescr.Type = contentType
fileDescr.UpdatedBy = operatorID
fileDescr.HelmMeta = helmMeta
fileDescr.HelmHash = helmHash
err = oper.mdb.UpdateFileByID(ctx, fileDescr.ID, fileDescr)
if err != nil {
code := http.StatusInternalServerError
@@ -102,6 +112,8 @@ func (oper *Operator) PutFile(ctx context.Context, operatorID string, params *Pu
UpdatedAt: now,
CreatedBy: operatorID,
UpdatedBy: operatorID,
HelmMeta: helmMeta,
HelmHash: helmHash,
}
err = oper.mdb.InsertFile(ctx, fileDescr)
if err != nil {
app/imageoper/patchupload.go
+1 -1
View File
@@ -71,7 +71,7 @@ func (oper *Operator) PatchUpload(ctx context.Context, operatorID string, params
}
}
recsize, _, err := oper.store.WriteUpload(params.Reference, params.Reader)
recsize, err := oper.store.WriteUpload(params.Reference, params.Reader)
if err != nil {
return res, http.StatusInternalServerError, err
}
app/imageoper/putupload.go
+1 -1
View File
@@ -65,7 +65,7 @@ func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *
}
if contentLength != 0 {
recsize, _, err := oper.store.WriteUpload(params.Reference, params.Reader)
recsize, err := oper.store.WriteUpload(params.Reference, params.Reader)
if err != nil {
return res, http.StatusInternalServerError, err
}
app/maindb/file.go
+10 -6
View File
@@ -17,10 +17,12 @@ import (
func (db *Database) InsertFile(ctx context.Context, file *descr.File) error {
var err error
request := `INSERT INTO files(id, collection, name, type, checksum, size, created_at, updated_at, created_by, updated_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`
request := `INSERT INTO files(id, collection, name, type, checksum, size,
created_at, updated_at, created_by, updated_by, helm_hash, helm_meta)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum, file.Size,
file.CreatedAt, file.UpdatedAt, file.CreatedBy, file.UpdatedBy)
file.CreatedAt, file.UpdatedAt, file.CreatedBy, file.UpdatedBy,
file.HelmHash, file.HelmMeta)
if err != nil {
return err
}
@@ -30,10 +32,12 @@ func (db *Database) InsertFile(ctx context.Context, file *descr.File) error {
func (db *Database) UpdateFileByID(ctx context.Context, fileID string, file *descr.File) error {
var err error
request := `UPDATE files SET id = $1, collection = $2, name = $3, type = $4, checksum = $5,
size = $6, updated_at = $7, created_by = $8, updated_by = $9
WHERE id = $10`
size = $6, updated_at = $7, created_by = $8, updated_by = $9,
helm_hash = $10, helm_meta = $11
WHERE id = $12`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum,
file.Size, file.UpdatedAt, file.CreatedBy, file.UpdatedBy, fileID)
file.Size, file.UpdatedAt, file.CreatedBy, file.UpdatedBy,
file.HelmHash, file.HelmMeta, fileID)
if err != nil {
return err
}
app/maindb/scheme.go
+3 -1
View File
@@ -20,7 +20,9 @@ const schema = `
created_at VARCHAR(255) NOT NULL,
updated_at VARCHAR(255) NOT NULL,
created_by VARCHAR(255) NOT NULL,
updated_by VARCHAR(255) NOT NULL
updated_by VARCHAR(255) NOT NULL,
helm_hash VARCHAR(255) NOT NULL,
helm_meta VARCHAR(4096) NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS files_index
ON files(collection, name);
app/storage/hasher.go
+32
View File
@@ -0,0 +1,32 @@
package storage
import (
"crypto/sha256"
"encoding/hex"
"hash"
"io"
)
type Hasher struct {
hasher hash.Hash
}
func NewHasher() *Hasher {
return &Hasher{
hasher: sha256.New(),
}
}
func (ha *Hasher) Writer() io.Writer {
return ha.hasher
}
func (ha *Hasher) Hex() string {
data := ha.hasher.Sum(nil)
res := hex.EncodeToString(data)
return "sha256:" + res
}
func (ha *Hasher) Verify(hash string) bool {
return hash == ha.Hex()
}
app/storage/helmmeta.go
+28
View File
@@ -0,0 +1,28 @@
package storage
import (
"encoding/json"
"helm.sh/helm/v4/pkg/chart/v2/loader"
"helm.sh/helm/v4/pkg/provenance"
)
func (store *Storage) HelmMeta(filename string) (string, string, error) {
var err error
var meta string
var hash string
chart, err := loader.Load(filename)
if err != nil {
return meta, hash, err
}
hash, err = provenance.DigestFile(filename)
if err != nil {
return meta, hash, err
}
metadata, err := json.Marshal(chart.Metadata)
if err != nil {
return meta, hash, err
}
meta = string(metadata)
return meta, hash, err
}
app/storage/storage.go
+80 -110
View File
@@ -11,8 +11,6 @@ package storage
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
@@ -25,8 +23,8 @@ import (
const (
sha256prefix = "sha256:"
filesubdir = "files"
tmpsubdir = "tmps"
fileSubdir = "files"
tmpSubdir = "tmps"
)
type Storage struct {
@@ -43,23 +41,23 @@ func NewStorage(basepath string) *Storage {
}
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, filesubdir, collection)
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, filesubdir, collection, filename)
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpname string) string {
return filepath.Join(store.basepath, tmpsubdir, tmpname)
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpsubdir)
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, filesubdir)
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
@@ -80,40 +78,38 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
var size int64
var csum string
tmpname := fmt.Sprintf("file-%d-%d.tmp", auxuuid.NewUUID(), auxuuid.NewUUID())
tmppath := store.makeTmppath(tmpname)
tmpName := fmt.Sprintf("%d.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpname, size, csum, err
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640)
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpname, size, csum, err
return tmpName, size, csum, err
}
defer file.Close()
hasher := sha256.New()
writer := io.MultiWriter(file, hasher)
size, err = io.Copy(writer, source)
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpname, size, csum, err
return tmpName, size, csum, err
}
csum = hex.EncodeToString(hasher.Sum(nil))
csum = sha256prefix + csum
csum = hasher.Hex()
return tmpname, size, csum, err
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if errors.Is(err, os.ErrNotExist) {
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
@@ -122,16 +118,17 @@ func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
os.Remove(filename) // TODO: safe removing
tmpname = store.makeTmppath(tmpname)
err = os.Link(tmpname, filename)
_, err = os.Stat(dirname)
if os.IsExist(err) {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpname)
err = os.Remove(tmpName)
if err != nil {
return err
}
@@ -172,45 +169,37 @@ func (store *Storage) makeBlobsubdir() string {
return filepath.Join(store.basepath, blobsubdir)
}
func (store *Storage) WriteUpload(uploadID string, source io.Reader) (int64, string, error) {
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
var recsum string
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if errors.Is(err, os.ErrNotExist) {
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, recsum, err
return recsize, err
}
}
if err != nil {
return recsize, recsum, err
return recsize, err
}
uploadPath := store.makeUppath(uploadID)
uploadFile, err := os.OpenFile(uploadPath, os.O_WRONLY|os.O_CREATE, 0644)
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, recsum, err
return recsize, err
}
defer uploadFile.Close()
hasher := sha256.New() // TODO: upload cheking
streamWriter := io.MultiWriter(uploadFile, hasher)
recsize, err = io.Copy(streamWriter, source)
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, recsum, err
return recsize, err
}
recsum = hex.EncodeToString(hasher.Sum(nil))
recsum = sha256prefix + recsum
return recsize, recsum, err
return recsize, err
}
func (store *Storage) LinkUpload(reference, digest string) error {
var err error
uploadPath := store.makeUppath(reference)
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir()
_, err = os.Stat(blobdir)
@@ -231,18 +220,17 @@ func (store *Storage) LinkUpload(reference, digest string) error {
return err
}
}
if errors.Is(err, os.ErrNotExist) {
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(uploadPath, blobPath)
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(uploadPath)
err = os.Remove(upPath)
if err != nil {
return err
}
@@ -251,8 +239,8 @@ func (store *Storage) LinkUpload(reference, digest string) error {
func (store *Storage) RemoveUpload(digest string) error {
var err error
uploadPath := store.makeUppath(digest)
err = os.Remove(uploadPath)
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
@@ -261,88 +249,70 @@ func (store *Storage) RemoveUpload(digest string) error {
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var fileSize int64
uploadPath := st.makeUppath(reference)
fileStat, err := os.Stat(uploadPath)
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
fileSize = fileStat.Size()
return true, fileSize, err
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(digest string, source io.Reader) (int64, string, error) {
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
var err error
var recsize int64
var recsum string
//defer source.Close()
blobdir := store.makeBlobsubdir()
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
var size int64
blobDir := store.makeBlobsubdir()
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return recsize, digest, err
return size, err
}
}
if err != nil {
return recsize, digest, err
return size, err
}
blobpath := store.makeBlobpath(digest)
blobfile, err := os.OpenFile(blobpath, os.O_WRONLY|os.O_CREATE, 0644)
blobPath := store.makeBlobpath(digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, digest, err
return size, err
}
defer blobfile.Close()
hasher := sha256.New() // TODO
multiWriter := io.MultiWriter(blobfile, hasher)
recsize, err = io.Copy(multiWriter, source)
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return recsize, digest, err
return size, err
}
recsum = hex.EncodeToString(hasher.Sum(nil))
recsum = sha256prefix + recsum
return recsize, recsum, err
return size, err
}
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
var err error
var fileSize int64
var size int64
blobPath := st.makeBlobpath(digest)
fileStat, err := os.Stat(blobPath)
if errors.Is(err, os.ErrNotExist) {
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
fileSize = fileStat.Size()
return true, fileSize, err
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
var err error
var filesize int64
blobpath := store.makeBlobpath(digest)
emptyReadCloser := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobpath, os.O_RDONLY, 0)
var size int64
blobPath := store.makeBlobpath(digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return filesize, emptyReadCloser, err
return size, nop, err
}
defer func() {
if err != nil {
@@ -351,17 +321,17 @@ func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
}()
filestat, err := file.Stat()
if err != nil {
return filesize, emptyReadCloser, err
return size, nop, err
}
filesize = filestat.Size()
return filesize, file, err
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(digest string) error {
var err error
blobpath := store.makeBlobpath(digest)
err = os.Remove(blobpath)
if errors.Is(err, os.ErrNotExist) {
blobPath := store.makeBlobpath(digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {