ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

working commit

Browse Source
This commit is contained in:
Олег Бородин
2026-01-28 13:06:04 +02:00
parent 6db319088e
commit d647854e98
13 changed files with 293 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/config.go
+8
View File
@@ -3,6 +3,7 @@ package config
type Config struct {
Service Service `json:"service" yaml:"service"`
Database Database `json:"database" yaml:"database"`
Storage Storage `json:"storage" yaml:"storage"`
}
type Service struct {
@@ -14,6 +15,10 @@ type Database struct {
Basepath string `json:"basepath" yaml:"basepath"`
}
type Storage struct {
Basepath string `json:"basepath" yaml:"basepath"`
}
func NewConfig() (*Config, error) {
var err error
return &Config{
@@ -24,5 +29,8 @@ func NewConfig() (*Config, error) {
Database: Database{
Basepath: datadir,
},
Storage: Storage{
Basepath: datadir,
},
}, err
}
app/descr/file.go
+1
View File
@@ -4,6 +4,7 @@ type File struct {
ID string `db:"id" json:"id,omitempty" yaml:"id,omitempty"`
Collection string `db:"collection" json:"collection,omitempty" yaml:"collection,omitempty"`
Name string `db:"name" json:"name,omitempty" yaml:"name,omitempty"`
Type string `db:"type" json:"type,omitempty" yaml:"type,omitempty"`
Checksum string `db:"checksum" json:"checksum,omitempty" yaml:"checksum,omitempty"`
Size int64 `db:"size" json:"size,omitempty" yaml:"size,omitempty"`
CreatedAt string `db:"created_at" json:"createdAt,omitempty" yaml:"createdAt,omitempty"`
app/handler/file.go
+17 -10
View File
@@ -6,7 +6,7 @@ import (
)
func (hand *Handler) FileExists(rctx *router.Context) {
hand.logg.Debugf("handle FileExists")
hand.logg.Debugf("Handle FileExists")
filepath := rctx.PathMap["filepath"]
params := &operator.FileExistsParams{
@@ -19,21 +19,28 @@ func (hand *Handler) FileExists(rctx *router.Context) {
}
func (hand *Handler) PutFile(rctx *router.Context) {
hand.logg.Debugf("handle PutFile")
hand.logg.Debugf("Handle PutFile")
contentLength := rctx.GetHeader("Content-Length")
contentType := rctx.GetHeader("Content-Type")
filepath := rctx.PathMap["filepath"]
params := &operator.PutFileParams{
Filepath: filepath,
Source: rctx.Request.Body,
}
hand.logg.Debugf("filepath: %s", filepath)
code, _, _ := hand.oper.PutFile(params)
params := &operator.PutFileParams{
Filepath: filepath,
ContentLength: contentLength,
ContentType: contentType,
Source: rctx.Request.Body,
}
code, res, err := hand.oper.PutFile(params)
if err != nil {
hand.logg.Errorf("Error: %v", err)
}
rctx.SetStatus(code)
hand.SendResult(rctx, res)
}
func (hand *Handler) GetFile(rctx *router.Context) {
hand.logg.Debugf("handle GetFile")
hand.logg.Debugf("Handle GetFile")
filepath := rctx.PathMap["filepath"]
params := &operator.GetFileParams{
@@ -48,7 +55,7 @@ func (hand *Handler) GetFile(rctx *router.Context) {
}
func (hand *Handler) DeleteFile(rctx *router.Context) {
hand.logg.Debugf("handle DeleteFile")
hand.logg.Debugf("Handle DeleteFile")
filepath := rctx.PathMap["filepath"]
params := &operator.DeleteFileParams{
app/handler/service.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
)
func (hand *Handler) SendHello(rctx *router.Context) {
hand.logg.Debugf("handle sendHello")
hand.logg.Debugf("Handle SendHello")
params := &operator.SendHelloParams{}
res, _ := hand.oper.SendHello(params)
app/maindb/file.go
+7 -7
View File
@@ -6,9 +6,9 @@ import (
func (db *Database) InsertFile(file *descr.File) error {
var err error
request := `INSERT INTO file(id, collection, name, checksum, size, created_at, updated_at, created_by, updated_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Checksum, file.Size,
request := `INSERT INTO file(id, collection, name, type, checksum, size, created_at, updated_at, created_by, updated_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum, file.Size,
file.CreatedAt, file.UpdatedAt, file.CreatedBy, file.UpdatedBy)
if err != nil {
return err
@@ -18,10 +18,10 @@ func (db *Database) InsertFile(file *descr.File) error {
func (db *Database) UpdateFileByID(fileID string, file *descr.File) error {
var err error
request := `UPDATE file SET id = $1, collection = $2, name = $3, checksum = $4,
size = $5, updated_at = $6, created_by = $7, updated_by = $8
WHERE id = $9`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Checksum,
request := `UPDATE file SET id = $1, collection = $2, name = $3, type = $4, checksum = $5,
size = $6, updated_at = $7, created_by = $8, updated_by = $9
WHERE id = $10`
_, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum,
file.Size, file.UpdatedAt, file.CreatedBy, file.UpdatedBy, fileID)
if err != nil {
return err
app/maindb/file_test.go
+4
View File
@@ -30,6 +30,7 @@ func TestFile(t *testing.T) {
ID: id,
Collection: "foo",
Name: "bare",
Type: "application/octet-stream",
CreatedAt: timenow,
UpdatedAt: timenow,
CreatedBy: creator,
@@ -43,4 +44,7 @@ func TestFile(t *testing.T) {
require.NoError(t, err)
require.Equal(t, len(files), 1)
require.Equal(t, files[0].ID, id)
require.Equal(t, files[0].Type, "application/octet-stream")
require.Equal(t, files[0].Name, "bare")
require.Equal(t, files[0].Collection, "foo")
}
app/maindb/schema.go
+1
View File
@@ -5,6 +5,7 @@ const schema = `
id VARCHAR(255) NOT NULL,
collection VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
type VARCHAR(255) NOT NULL,
checksum VARCHAR(255) NOT NULL,
size INTEGER,
created_at VARCHAR(255) NOT NULL,
app/operator/file.go
+87 -8
View File
@@ -3,7 +3,12 @@ package operator
import (
"io"
"net/http"
//"path"
"path"
"strconv"
"mstore/app/descr"
"mstore/pkg/auxtool"
"mstore/pkg/auxuuid"
)
// File exists
@@ -12,16 +17,23 @@ type FileExistsParams struct {
Source string
Dest string
}
type FileExistsResult struct{}
type FileExistsResult struct {
Descr *descr.File
}
func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) {
var err error
//filename := path.Base(param.Filepath)
//dirname := path.Dir(filepath)
code := http.StatusNotFound
res := &FileExistsResult{}
code := http.StatusOK
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
exist, file, err := oper.mdb.GetFileByCollection(collection, filename)
if exist {
code = http.StatusOK
res.Descr = file
}
return code, res, err
}
@@ -32,11 +44,78 @@ type PutFileParams struct {
Filepath string
Source io.ReadCloser
}
type PutFileResult struct{}
type PutFileResult struct {
Descr *descr.File
}
const defaultContentType = "application/octet-stream"
func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error) {
var err error
res := &PutFileResult{}
size, err := strconv.ParseInt(param.ContentLength, 10, 64)
if err != nil {
code := http.StatusLengthRequired
return code, res, err
}
contentType := param.ContentType
if contentType == "" {
contentType = defaultContentType
}
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
oper.logg.Debugf("Put file %s %s", collection, filename)
tmpname, size, checksum, err := oper.store.WriteTempFile(param.Source)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
exists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
now := auxtool.TimeNow()
if exists {
fileDescr.Size = size
fileDescr.Checksum = checksum
fileDescr.UpdatedAt = now
fileDescr.Type = contentType
err = oper.mdb.UpdateFileByID(fileDescr.ID, fileDescr)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
} else {
fileDescr = &descr.File{
ID: auxuuid.NewUUID(),
Name: filename,
Collection: collection,
Size: size,
Type: contentType,
Checksum: checksum,
CreatedAt: now,
UpdatedAt: now,
}
err = oper.mdb.InsertFile(fileDescr)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
}
err = oper.store.LinkFile(tmpname, collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Descr = fileDescr
code := http.StatusOK
return code, res, err
}
app/operator/operator.go
+9 -2
View File
@@ -1,21 +1,28 @@
package operator
import (
"mstore/app/logger"
"mstore/app/maindb"
"mstore/app/storage"
)
type OperatorParams struct {
MainDB *maindb.Database
Store *storage.Storage
}
type Operator struct {
maindb *maindb.Database
mdb *maindb.Database
store *storage.Storage
logg *logger.Logger
}
func NewOperator(params *OperatorParams) (*Operator, error) {
var err error
oper := &Operator{
maindb: params.MainDB,
mdb: params.MainDB,
store: params.Store,
}
oper.logg = logger.NewLogger("operator")
return oper, err
}
app/router/context.go
+1 -1
View File
@@ -43,5 +43,5 @@ func (rctx *Context) SendText(payload string) {
}
func (rctx *Context) GetHeader(key string) string {
return rctx.Request.URL.Query().Get(key)
return rctx.Request.Header.Get(key)
}
app/server/server.go
+33 -11
View File
@@ -18,15 +18,17 @@ import (
"mstore/app/maindb"
"mstore/app/operator"
"mstore/app/service"
"mstore/app/storage"
)
type Server struct {
conf *config.Config
oper *operator.Operator
svc *service.Service
maindb *maindb.Database
hand *handler.Handler
logg *logger.Logger
conf *config.Config
oper *operator.Operator
svc *service.Service
mdb *maindb.Database
hand *handler.Handler
logg *logger.Logger
stor *storage.Storage
}
func NewServer() (*Server, error) {
@@ -56,31 +58,49 @@ func (srv *Server) Build() error {
srv.logg.Infof("Server build")
// Database create
srv.logg.Infof("Create database directory")
dbdir := srv.conf.Database.Basepath
err = os.MkdirAll(srv.conf.Database.Basepath, 0750)
err = os.MkdirAll(dbdir, 0750)
if err != nil {
return err
}
maindb := maindb.NewDatabase(dbdir)
err = maindb.OpenDatabase()
mdb := maindb.NewDatabase(dbdir)
srv.logg.Infof("Open database")
err = mdb.OpenDatabase()
if err != nil {
return err
}
err = maindb.InitDatabase()
srv.logg.Infof("Initialize database")
err = mdb.InitDatabase()
if err != nil {
return err
}
srv.mdb = mdb
// Storage create
srv.logg.Infof("Create storage directory")
datadir := srv.conf.Database.Basepath
err = os.MkdirAll(datadir, 0750)
if err != nil {
return err
}
srv.logg.Infof("Create storage")
store := storage.NewStorage(datadir)
srv.stor = store
// Operator create
srv.logg.Infof("Create operator")
operatorParams := &operator.OperatorParams{
MainDB: srv.maindb,
MainDB: srv.mdb,
Store: srv.stor,
}
srv.oper, err = operator.NewOperator(operatorParams)
if err != nil {
return err
}
// Handler create
srv.logg.Infof("Create handler")
handlerParams := &handler.HandlerParams{
Operator: srv.oper,
}
@@ -92,6 +112,7 @@ func (srv *Server) Build() error {
serviceParams := &service.ServiceParams{
Handler: srv.hand,
}
srv.logg.Infof("Create service")
srv.svc, err = service.NewService(serviceParams)
if err != nil {
return err
@@ -118,6 +139,7 @@ func (srv *Server) Run() error {
svcDone := make(chan error, 1)
// Service run
srv.logg.Infof("Start service")
startService := func(svc *service.Service, done chan error) {
err = svc.Run()
if err != nil {
app/storage/storage.go
+75
View File
@@ -1 +1,76 @@
package storage
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"mstore/app/logger"
"mstore/pkg/auxuuid"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLogger("storage")
return res
}
func (store *Storage) LinkFile(tmpname, collection, filename string) error {
var err error
dirname := filepath.Join(store.basepath, collection)
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
filename = filepath.Join(store.basepath, collection, filename)
os.Remove(filename)
err = os.Link(tmpname, filename)
if err != nil {
return err
}
err = os.Remove(tmpname)
if err != nil {
return err
}
return err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var digest string
tmpname := auxuuid.NewUUID()
tmpname = fmt.Sprintf("file-%s.tmp", tmpname)
tmppath := filepath.Join(store.basepath, tmpname)
file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmppath, size, digest, err
}
defer file.Close()
hasher := sha256.New()
writer := io.MultiWriter(file, hasher)
size, err = io.Copy(writer, source)
if err != nil {
return tmppath, size, digest, err
}
digest = hex.EncodeToString(hasher.Sum(nil))
digest = fmt.Sprintf("sha256:%s", digest)
return tmppath, size, digest, err
}
test/svc_test.go
+49 -5
View File
@@ -1,20 +1,21 @@
package test
import (
"github.com/stretchr/testify/require"
"bytes"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"testing"
"mstore/app/router"
"mstore/app/server"
"github.com/stretchr/testify/require"
)
func MakeServer(t *testing.T) *server.Server {
var err error
srv, err := server.NewServer()
require.NoError(t, err)
@@ -28,7 +29,10 @@ func MakeServer(t *testing.T) *server.Server {
return srv
}
func TestFileExists(t *testing.T) {
func TestFileLife(t *testing.T) {
}
func xxxTestFileExists(t *testing.T) {
var err error
srv := MakeServer(t)
@@ -46,6 +50,46 @@ func TestFileExists(t *testing.T) {
request, err := http.NewRequest("HEAD", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusNotFound, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func TestPutFile(t *testing.T) {
var err error
srv := MakeServer(t)
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Put(routePath, hand.PutFile)
datasize := 16
filedata := make([]byte, datasize)
_, err = rand.Read(filedata)
require.NoError(t, err)
source := bytes.NewReader(filedata)
request, err := http.NewRequest("PUT", reqPath, source)
require.NoError(t, err)
request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
request.Header.Set("Content-Type", "application/octet-stream")
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
@@ -58,7 +102,7 @@ func TestFileExists(t *testing.T) {
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func TestServiceHello(t *testing.T) {
func xxxTestServiceHello(t *testing.T) {
var err error
srv := MakeServer(t)