From d647854e98b2c457dd8a1541061f7ab55419f71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Wed, 28 Jan 2026 13:06:04 +0200 Subject: [PATCH] working commit --- app/config/config.go | 8 ++++ app/descr/file.go | 1 + app/handler/file.go | 27 +++++++----- app/handler/service.go | 2 +- app/maindb/file.go | 14 +++--- app/maindb/file_test.go | 4 ++ app/maindb/schema.go | 1 + app/operator/file.go | 95 ++++++++++++++++++++++++++++++++++++---- app/operator/operator.go | 11 ++++- app/router/context.go | 2 +- app/server/server.go | 44 ++++++++++++++----- app/storage/storage.go | 75 +++++++++++++++++++++++++++++++ test/svc_test.go | 54 ++++++++++++++++++++--- 13 files changed, 293 insertions(+), 45 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index 789b8e9..b437f19 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -3,6 +3,7 @@ package config type Config struct { Service Service `json:"service" yaml:"service"` Database Database `json:"database" yaml:"database"` + Storage Storage `json:"storage" yaml:"storage"` } type Service struct { @@ -14,6 +15,10 @@ type Database struct { Basepath string `json:"basepath" yaml:"basepath"` } +type Storage struct { + Basepath string `json:"basepath" yaml:"basepath"` +} + func NewConfig() (*Config, error) { var err error return &Config{ @@ -24,5 +29,8 @@ func NewConfig() (*Config, error) { Database: Database{ Basepath: datadir, }, + Storage: Storage{ + Basepath: datadir, + }, }, err } diff --git a/app/descr/file.go b/app/descr/file.go index 6baa720..f48a4cb 100644 --- a/app/descr/file.go +++ b/app/descr/file.go @@ -4,6 +4,7 @@ type File struct { ID string `db:"id" json:"id,omitempty" yaml:"id,omitempty"` Collection string `db:"collection" json:"collection,omitempty" yaml:"collection,omitempty"` Name string `db:"name" json:"name,omitempty" yaml:"name,omitempty"` + Type string `db:"type" json:"type,omitempty" yaml:"type,omitempty"` Checksum string `db:"checksum" json:"checksum,omitempty" yaml:"checksum,omitempty"` Size int64 `db:"size" json:"size,omitempty" yaml:"size,omitempty"` CreatedAt string `db:"created_at" json:"createdAt,omitempty" yaml:"createdAt,omitempty"` diff --git a/app/handler/file.go b/app/handler/file.go index 20d7d2e..42a12ae 100644 --- a/app/handler/file.go +++ b/app/handler/file.go @@ -6,7 +6,7 @@ import ( ) func (hand *Handler) FileExists(rctx *router.Context) { - hand.logg.Debugf("handle FileExists") + hand.logg.Debugf("Handle FileExists") filepath := rctx.PathMap["filepath"] params := &operator.FileExistsParams{ @@ -19,21 +19,28 @@ func (hand *Handler) FileExists(rctx *router.Context) { } func (hand *Handler) PutFile(rctx *router.Context) { - hand.logg.Debugf("handle PutFile") + hand.logg.Debugf("Handle PutFile") + contentLength := rctx.GetHeader("Content-Length") + contentType := rctx.GetHeader("Content-Type") filepath := rctx.PathMap["filepath"] - params := &operator.PutFileParams{ - Filepath: filepath, - Source: rctx.Request.Body, - } - hand.logg.Debugf("filepath: %s", filepath) - code, _, _ := hand.oper.PutFile(params) + params := &operator.PutFileParams{ + Filepath: filepath, + ContentLength: contentLength, + ContentType: contentType, + Source: rctx.Request.Body, + } + code, res, err := hand.oper.PutFile(params) + if err != nil { + hand.logg.Errorf("Error: %v", err) + } rctx.SetStatus(code) + hand.SendResult(rctx, res) } func (hand *Handler) GetFile(rctx *router.Context) { - hand.logg.Debugf("handle GetFile") + hand.logg.Debugf("Handle GetFile") filepath := rctx.PathMap["filepath"] params := &operator.GetFileParams{ @@ -48,7 +55,7 @@ func (hand *Handler) GetFile(rctx *router.Context) { } func (hand *Handler) DeleteFile(rctx *router.Context) { - hand.logg.Debugf("handle DeleteFile") + hand.logg.Debugf("Handle DeleteFile") filepath := rctx.PathMap["filepath"] params := &operator.DeleteFileParams{ diff --git a/app/handler/service.go b/app/handler/service.go index 94778ab..59354aa 100644 --- a/app/handler/service.go +++ b/app/handler/service.go @@ -6,7 +6,7 @@ import ( ) func (hand *Handler) SendHello(rctx *router.Context) { - hand.logg.Debugf("handle sendHello") + hand.logg.Debugf("Handle SendHello") params := &operator.SendHelloParams{} res, _ := hand.oper.SendHello(params) diff --git a/app/maindb/file.go b/app/maindb/file.go index cbffab3..88deb92 100644 --- a/app/maindb/file.go +++ b/app/maindb/file.go @@ -6,9 +6,9 @@ import ( func (db *Database) InsertFile(file *descr.File) error { var err error - request := `INSERT INTO file(id, collection, name, checksum, size, created_at, updated_at, created_by, updated_by) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` - _, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Checksum, file.Size, + request := `INSERT INTO file(id, collection, name, type, checksum, size, created_at, updated_at, created_by, updated_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)` + _, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum, file.Size, file.CreatedAt, file.UpdatedAt, file.CreatedBy, file.UpdatedBy) if err != nil { return err @@ -18,10 +18,10 @@ func (db *Database) InsertFile(file *descr.File) error { func (db *Database) UpdateFileByID(fileID string, file *descr.File) error { var err error - request := `UPDATE file SET id = $1, collection = $2, name = $3, checksum = $4, - size = $5, updated_at = $6, created_by = $7, updated_by = $8 - WHERE id = $9` - _, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Checksum, + request := `UPDATE file SET id = $1, collection = $2, name = $3, type = $4, checksum = $5, + size = $6, updated_at = $7, created_by = $8, updated_by = $9 + WHERE id = $10` + _, err = db.db.Exec(request, file.ID, file.Collection, file.Name, file.Type, file.Checksum, file.Size, file.UpdatedAt, file.CreatedBy, file.UpdatedBy, fileID) if err != nil { return err diff --git a/app/maindb/file_test.go b/app/maindb/file_test.go index a972717..7d4c541 100644 --- a/app/maindb/file_test.go +++ b/app/maindb/file_test.go @@ -30,6 +30,7 @@ func TestFile(t *testing.T) { ID: id, Collection: "foo", Name: "bare", + Type: "application/octet-stream", CreatedAt: timenow, UpdatedAt: timenow, CreatedBy: creator, @@ -43,4 +44,7 @@ func TestFile(t *testing.T) { require.NoError(t, err) require.Equal(t, len(files), 1) require.Equal(t, files[0].ID, id) + require.Equal(t, files[0].Type, "application/octet-stream") + require.Equal(t, files[0].Name, "bare") + require.Equal(t, files[0].Collection, "foo") } diff --git a/app/maindb/schema.go b/app/maindb/schema.go index 8940297..bd49b23 100644 --- a/app/maindb/schema.go +++ b/app/maindb/schema.go @@ -5,6 +5,7 @@ const schema = ` id VARCHAR(255) NOT NULL, collection VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, checksum VARCHAR(255) NOT NULL, size INTEGER, created_at VARCHAR(255) NOT NULL, diff --git a/app/operator/file.go b/app/operator/file.go index 0a0df50..bd93f99 100644 --- a/app/operator/file.go +++ b/app/operator/file.go @@ -3,7 +3,12 @@ package operator import ( "io" "net/http" - //"path" + "path" + "strconv" + + "mstore/app/descr" + "mstore/pkg/auxtool" + "mstore/pkg/auxuuid" ) // File exists @@ -12,16 +17,23 @@ type FileExistsParams struct { Source string Dest string } -type FileExistsResult struct{} +type FileExistsResult struct { + Descr *descr.File +} func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) { var err error - - //filename := path.Base(param.Filepath) - //dirname := path.Dir(filepath) - + code := http.StatusNotFound res := &FileExistsResult{} - code := http.StatusOK + + filename := path.Base(param.Filepath) + collection := path.Dir(param.Filepath) + + exist, file, err := oper.mdb.GetFileByCollection(collection, filename) + if exist { + code = http.StatusOK + res.Descr = file + } return code, res, err } @@ -32,11 +44,78 @@ type PutFileParams struct { Filepath string Source io.ReadCloser } -type PutFileResult struct{} +type PutFileResult struct { + Descr *descr.File +} + +const defaultContentType = "application/octet-stream" func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error) { var err error res := &PutFileResult{} + + size, err := strconv.ParseInt(param.ContentLength, 10, 64) + if err != nil { + code := http.StatusLengthRequired + return code, res, err + } + contentType := param.ContentType + if contentType == "" { + contentType = defaultContentType + } + + filename := path.Base(param.Filepath) + collection := path.Dir(param.Filepath) + oper.logg.Debugf("Put file %s %s", collection, filename) + + tmpname, size, checksum, err := oper.store.WriteTempFile(param.Source) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + exists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + now := auxtool.TimeNow() + if exists { + fileDescr.Size = size + fileDescr.Checksum = checksum + fileDescr.UpdatedAt = now + fileDescr.Type = contentType + err = oper.mdb.UpdateFileByID(fileDescr.ID, fileDescr) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } else { + fileDescr = &descr.File{ + ID: auxuuid.NewUUID(), + Name: filename, + Collection: collection, + Size: size, + Type: contentType, + Checksum: checksum, + CreatedAt: now, + UpdatedAt: now, + } + err = oper.mdb.InsertFile(fileDescr) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + } + + err = oper.store.LinkFile(tmpname, collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + res.Descr = fileDescr code := http.StatusOK return code, res, err } diff --git a/app/operator/operator.go b/app/operator/operator.go index 535d190..21432a4 100644 --- a/app/operator/operator.go +++ b/app/operator/operator.go @@ -1,21 +1,28 @@ package operator import ( + "mstore/app/logger" "mstore/app/maindb" + "mstore/app/storage" ) type OperatorParams struct { MainDB *maindb.Database + Store *storage.Storage } type Operator struct { - maindb *maindb.Database + mdb *maindb.Database + store *storage.Storage + logg *logger.Logger } func NewOperator(params *OperatorParams) (*Operator, error) { var err error oper := &Operator{ - maindb: params.MainDB, + mdb: params.MainDB, + store: params.Store, } + oper.logg = logger.NewLogger("operator") return oper, err } diff --git a/app/router/context.go b/app/router/context.go index c64842b..51b7e10 100644 --- a/app/router/context.go +++ b/app/router/context.go @@ -43,5 +43,5 @@ func (rctx *Context) SendText(payload string) { } func (rctx *Context) GetHeader(key string) string { - return rctx.Request.URL.Query().Get(key) + return rctx.Request.Header.Get(key) } diff --git a/app/server/server.go b/app/server/server.go index ae7e1d3..e704f95 100644 --- a/app/server/server.go +++ b/app/server/server.go @@ -18,15 +18,17 @@ import ( "mstore/app/maindb" "mstore/app/operator" "mstore/app/service" + "mstore/app/storage" ) type Server struct { - conf *config.Config - oper *operator.Operator - svc *service.Service - maindb *maindb.Database - hand *handler.Handler - logg *logger.Logger + conf *config.Config + oper *operator.Operator + svc *service.Service + mdb *maindb.Database + hand *handler.Handler + logg *logger.Logger + stor *storage.Storage } func NewServer() (*Server, error) { @@ -56,31 +58,49 @@ func (srv *Server) Build() error { srv.logg.Infof("Server build") // Database create + srv.logg.Infof("Create database directory") dbdir := srv.conf.Database.Basepath - err = os.MkdirAll(srv.conf.Database.Basepath, 0750) + err = os.MkdirAll(dbdir, 0750) if err != nil { return err } - maindb := maindb.NewDatabase(dbdir) - err = maindb.OpenDatabase() + mdb := maindb.NewDatabase(dbdir) + srv.logg.Infof("Open database") + err = mdb.OpenDatabase() if err != nil { return err } - err = maindb.InitDatabase() + srv.logg.Infof("Initialize database") + err = mdb.InitDatabase() if err != nil { return err } + srv.mdb = mdb + + // Storage create + srv.logg.Infof("Create storage directory") + datadir := srv.conf.Database.Basepath + err = os.MkdirAll(datadir, 0750) + if err != nil { + return err + } + srv.logg.Infof("Create storage") + store := storage.NewStorage(datadir) + srv.stor = store // Operator create + srv.logg.Infof("Create operator") operatorParams := &operator.OperatorParams{ - MainDB: srv.maindb, + MainDB: srv.mdb, + Store: srv.stor, } srv.oper, err = operator.NewOperator(operatorParams) if err != nil { return err } // Handler create + srv.logg.Infof("Create handler") handlerParams := &handler.HandlerParams{ Operator: srv.oper, } @@ -92,6 +112,7 @@ func (srv *Server) Build() error { serviceParams := &service.ServiceParams{ Handler: srv.hand, } + srv.logg.Infof("Create service") srv.svc, err = service.NewService(serviceParams) if err != nil { return err @@ -118,6 +139,7 @@ func (srv *Server) Run() error { svcDone := make(chan error, 1) // Service run + srv.logg.Infof("Start service") startService := func(svc *service.Service, done chan error) { err = svc.Run() if err != nil { diff --git a/app/storage/storage.go b/app/storage/storage.go index 82be054..1b20d5d 100644 --- a/app/storage/storage.go +++ b/app/storage/storage.go @@ -1 +1,76 @@ package storage + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + + "mstore/app/logger" + "mstore/pkg/auxuuid" +) + +type Storage struct { + basepath string + logg *logger.Logger +} + +func NewStorage(basepath string) *Storage { + res := &Storage{ + basepath: basepath, + } + res.logg = logger.NewLogger("storage") + return res +} + +func (store *Storage) LinkFile(tmpname, collection, filename string) error { + var err error + dirname := filepath.Join(store.basepath, collection) + err = os.MkdirAll(dirname, 0750) + if err != nil { + return err + } + + filename = filepath.Join(store.basepath, collection, filename) + os.Remove(filename) + + err = os.Link(tmpname, filename) + if err != nil { + return err + } + err = os.Remove(tmpname) + if err != nil { + return err + } + return err +} + +func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) { + var err error + var size int64 + var digest string + + tmpname := auxuuid.NewUUID() + tmpname = fmt.Sprintf("file-%s.tmp", tmpname) + tmppath := filepath.Join(store.basepath, tmpname) + + file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + return tmppath, size, digest, err + } + defer file.Close() + + hasher := sha256.New() + writer := io.MultiWriter(file, hasher) + + size, err = io.Copy(writer, source) + if err != nil { + return tmppath, size, digest, err + } + digest = hex.EncodeToString(hasher.Sum(nil)) + digest = fmt.Sprintf("sha256:%s", digest) + + return tmppath, size, digest, err +} diff --git a/test/svc_test.go b/test/svc_test.go index ef13b83..3ee5580 100644 --- a/test/svc_test.go +++ b/test/svc_test.go @@ -1,20 +1,21 @@ package test import ( + "github.com/stretchr/testify/require" + + "bytes" "fmt" "io" + "math/rand" "net/http" "net/http/httptest" "testing" "mstore/app/router" "mstore/app/server" - - "github.com/stretchr/testify/require" ) func MakeServer(t *testing.T) *server.Server { - var err error srv, err := server.NewServer() require.NoError(t, err) @@ -28,7 +29,10 @@ func MakeServer(t *testing.T) *server.Server { return srv } -func TestFileExists(t *testing.T) { +func TestFileLife(t *testing.T) { +} + +func xxxTestFileExists(t *testing.T) { var err error srv := MakeServer(t) @@ -46,6 +50,46 @@ func TestFileExists(t *testing.T) { request, err := http.NewRequest("HEAD", reqPath, nil) require.NoError(t, err) + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusNotFound, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func TestPutFile(t *testing.T) { + var err error + + srv := MakeServer(t) + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Put(routePath, hand.PutFile) + + datasize := 16 + filedata := make([]byte, datasize) + _, err = rand.Read(filedata) + require.NoError(t, err) + + source := bytes.NewReader(filedata) + + request, err := http.NewRequest("PUT", reqPath, source) + require.NoError(t, err) + + request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize)) + request.Header.Set("Content-Type", "application/octet-stream") + recorder := httptest.NewRecorder() rout.ServeHTTP(recorder, request) require.Equal(t, http.StatusOK, recorder.Code) @@ -58,7 +102,7 @@ func TestFileExists(t *testing.T) { fmt.Printf("Response body: %s\n", string(bodyBytes)) } -func TestServiceHello(t *testing.T) { +func xxxTestServiceHello(t *testing.T) { var err error srv := MakeServer(t)