From d6496a427ba7cf05c0320b5bc428a255b41a1eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Wed, 28 Jan 2026 15:33:19 +0200 Subject: [PATCH] first draft of file storage --- app/handler/file.go | 47 ++++++-- app/operator/file.go | 85 +++++++++++--- app/router/router.go | 4 + app/storage/storage.go | 33 +++++- test/file_test.go | 251 +++++++++++++++++++++++++++++++++++++++++ test/main_test.go | 1 - test/svc_test.go | 131 --------------------- 7 files changed, 394 insertions(+), 158 deletions(-) create mode 100644 test/file_test.go delete mode 100644 test/main_test.go delete mode 100644 test/svc_test.go diff --git a/app/handler/file.go b/app/handler/file.go index 42a12ae..67256f1 100644 --- a/app/handler/file.go +++ b/app/handler/file.go @@ -1,6 +1,8 @@ package handler import ( + "io" + "mstore/app/operator" "mstore/app/router" ) @@ -12,10 +14,18 @@ func (hand *Handler) FileExists(rctx *router.Context) { params := &operator.FileExistsParams{ Filepath: filepath, } - hand.logg.Debugf("filepath: %s", filepath) - code, _, _ := hand.oper.FileExists(params) + code, res, err := hand.oper.FileExists(params) + if err != nil { + hand.logg.Errorf("FileExists error: %v", err) + rctx.SetStatus(code) + return + } + rctx.SetHeader("Content-Type", res.ContentType) + rctx.SetHeader("Content-Length", res.ContentLength) + rctx.SetHeader("Content-Digest", res.ContentDigest) rctx.SetStatus(code) + } func (hand *Handler) PutFile(rctx *router.Context) { @@ -31,12 +41,13 @@ func (hand *Handler) PutFile(rctx *router.Context) { ContentType: contentType, Source: rctx.Request.Body, } - code, res, err := hand.oper.PutFile(params) + code, _, err := hand.oper.PutFile(params) if err != nil { - hand.logg.Errorf("Error: %v", err) + hand.logg.Errorf("PutFile error: %v", err) + rctx.SetStatus(code) + return } rctx.SetStatus(code) - hand.SendResult(rctx, res) } func (hand *Handler) GetFile(rctx *router.Context) { @@ -48,10 +59,25 @@ func (hand *Handler) GetFile(rctx *router.Context) { } hand.logg.Debugf("filepath: %s", filepath) - code, res, _ := hand.oper.GetFile(params) + code, res, err := hand.oper.GetFile(params) + if err != nil { + hand.logg.Errorf("PutFile error: %v", err) + rctx.SetStatus(code) + return + } + rctx.SetStatus(code) rctx.SetHeader("Content-Type", res.ContentType) rctx.SetHeader("Content-Length", res.ContentLength) - rctx.SetStatus(code) + rctx.SetHeader("Content-Digest", res.ContentDigest) + + if res.Source != nil { + defer res.Source.Close() + _, err = io.Copy(rctx.Writer, res.Source) + if err != nil { + hand.logg.Errorf("GetFile error: %v", err) + return + } + } } func (hand *Handler) DeleteFile(rctx *router.Context) { @@ -61,8 +87,9 @@ func (hand *Handler) DeleteFile(rctx *router.Context) { params := &operator.DeleteFileParams{ Filepath: filepath, } - hand.logg.Debugf("filepath: %s", filepath) - - code, _, _ := hand.oper.DeleteFile(params) + code, _, err := hand.oper.DeleteFile(params) + if err != nil { + hand.logg.Errorf("GetFile error: %v", err) + } rctx.SetStatus(code) } diff --git a/app/operator/file.go b/app/operator/file.go index bd93f99..9f40eba 100644 --- a/app/operator/file.go +++ b/app/operator/file.go @@ -18,21 +18,28 @@ type FileExistsParams struct { Dest string } type FileExistsResult struct { - Descr *descr.File + ContentType string + ContentLength string + ContentDigest string } func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) { var err error - code := http.StatusNotFound + code := http.StatusOK res := &FileExistsResult{} filename := path.Base(param.Filepath) collection := path.Dir(param.Filepath) - exist, file, err := oper.mdb.GetFileByCollection(collection, filename) - if exist { - code = http.StatusOK - res.Descr = file + exist, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename) + if !exist { + code = http.StatusNotFound + return code, res, err + } + res = &FileExistsResult{ + ContentLength: strconv.FormatInt(fileDescr.Size, 10), + ContentType: fileDescr.Type, + ContentDigest: fileDescr.Checksum, } return code, res, err } @@ -44,9 +51,7 @@ type PutFileParams struct { Filepath string Source io.ReadCloser } -type PutFileResult struct { - Descr *descr.File -} +type PutFileResult struct{} const defaultContentType = "application/octet-stream" @@ -64,6 +69,8 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error) contentType = defaultContentType } + // TODO: convert file path to a unified and secure state + filename := path.Base(param.Filepath) collection := path.Dir(param.Filepath) oper.logg.Debugf("Put file %s %s", collection, filename) @@ -74,14 +81,14 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error) return code, res, err } - exists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename) + descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename) if err != nil { code := http.StatusInternalServerError return code, res, err } now := auxtool.TimeNow() - if exists { + if descrExists { fileDescr.Size = size fileDescr.Checksum = checksum fileDescr.UpdatedAt = now @@ -109,13 +116,11 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error) } } - err = oper.store.LinkFile(tmpname, collection, filename) + err = oper.store.HardlinkFile(tmpname, collection, filename) if err != nil { code := http.StatusInternalServerError return code, res, err } - - res.Descr = fileDescr code := http.StatusOK return code, res, err } @@ -127,12 +132,41 @@ type GetFileParams struct { type GetFileResult struct { ContentType string ContentLength string + ContentDigest string Source io.ReadCloser } func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error) { var err error res := &GetFileResult{} + + // TODO: convert file path to a unified and secure state + + filename := path.Base(param.Filepath) + collection := path.Dir(param.Filepath) + + oper.logg.Debugf("Gut file %s %s", collection, filename) + + descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + if !descrExists { + code := http.StatusNotFound + return code, res, err + } + reader, err := oper.store.GetFileReader(collection, filename) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res = &GetFileResult{ + ContentLength: strconv.FormatInt(fileDescr.Size, 10), + ContentType: fileDescr.Type, + ContentDigest: fileDescr.Checksum, + Source: reader, + } code := http.StatusOK return code, res, err } @@ -147,5 +181,28 @@ func (oper *Operator) DeleteFile(param *DeleteFileParams) (int, *DeleteFileResul var err error res := &DeleteFileResult{} code := http.StatusOK + + filename := path.Base(param.Filepath) + collection := path.Dir(param.Filepath) + + exist, _, err := oper.mdb.GetFileByCollection(collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + if exist { + err = oper.mdb.DeleteFileByCollection(collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + } + err = oper.store.DeleteFile(collection, filename) + if err != nil { + code = http.StatusInternalServerError + return code, res, err + } + return code, res, err + } diff --git a/app/router/router.go b/app/router/router.go index d700643..ca942c5 100644 --- a/app/router/router.go +++ b/app/router/router.go @@ -46,6 +46,10 @@ func (rout *Router) Put(path string, handlerFunc HandlerFunc) { rout.routeHandler.AddRoute("PUT", path, handlerFunc) } +func (rout *Router) Delete(path string, handlerFunc HandlerFunc) { + rout.routeHandler.AddRoute("DELETE", path, handlerFunc) +} + func (rout *Router) ServeHTTP(writer http.ResponseWriter, req *http.Request) { rctx := NewContext(writer, req) rout.routeHandler.ServeHTTP(rctx) diff --git a/app/storage/storage.go b/app/storage/storage.go index 1b20d5d..b4e92e3 100644 --- a/app/storage/storage.go +++ b/app/storage/storage.go @@ -25,7 +25,20 @@ func NewStorage(basepath string) *Storage { return res } -func (store *Storage) LinkFile(tmpname, collection, filename string) error { +func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) { + var err error + var res io.ReadCloser + + filename = filepath.Join(store.basepath, collection, filename) + file, err := os.OpenFile(filename, os.O_RDONLY, 0) + if err != nil { + return res, err + } + res = file + return res, err +} + +func (store *Storage) HardlinkFile(tmpname, collection, filename string) error { var err error dirname := filepath.Join(store.basepath, collection) err = os.MkdirAll(dirname, 0750) @@ -34,7 +47,7 @@ func (store *Storage) LinkFile(tmpname, collection, filename string) error { } filename = filepath.Join(store.basepath, collection, filename) - os.Remove(filename) + os.Remove(filename) // TODO err = os.Link(tmpname, filename) if err != nil { @@ -74,3 +87,19 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er return tmppath, size, digest, err } + +func (store *Storage) DeleteFile(collection, filename string) error { + var err error + filename = filepath.Join(store.basepath, collection, filename) + err = os.Remove(filename) + if err != nil { + return err + } + + dirname := filepath.Join(store.basepath, collection) + err = os.RemoveAll(dirname) + if err != nil { + return err + } + return err +} diff --git a/test/file_test.go b/test/file_test.go new file mode 100644 index 0000000..c6f12ab --- /dev/null +++ b/test/file_test.go @@ -0,0 +1,251 @@ +package test + +import ( + "github.com/stretchr/testify/require" + + "bytes" + "encoding/hex" + "fmt" + "io" + "math/rand" + "net/http" + "net/http/httptest" + "testing" + + "mstore/app/router" + "mstore/app/server" +) + +func TestFileLife(t *testing.T) { + var tester FileTester + tester.MakeServer(t) + tester.TestServiceHello(t) + tester.TestPutFile(t) + tester.TestFileExists(t) + tester.TestGetFile(t) + tester.TestDeleteFile(t) + tester.TestFileNotExists(t) +} + +type FileTester struct { + srv *server.Server +} + +func (tester *FileTester) MakeServer(t *testing.T) { + var err error + + fmt.Printf("=== MakeServer ===\n") + + srv, err := server.NewServer() + require.NoError(t, err) + + err = srv.Configure() + require.NoError(t, err) + + err = srv.Build() + require.NoError(t, err) + + tester.srv = srv +} + +func (tester *FileTester) TestPutFile(t *testing.T) { + var err error + + fmt.Printf("=== PutFile ===\n") + + srv := tester.srv + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Put(routePath, hand.PutFile) + + datasize := 16 + filedata := make([]byte, datasize) + _, err = rand.Read(filedata) + require.NoError(t, err) + + filedata = []byte(hex.EncodeToString(filedata)) + + source := bytes.NewReader(filedata) + + request, err := http.NewRequest("PUT", reqPath, source) + require.NoError(t, err) + + request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize)) + request.Header.Set("Content-Type", "application/octet-stream") + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusOK, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func (tester *FileTester) TestFileExists(t *testing.T) { + var err error + fmt.Printf("=== FileExists ===\n") + + srv := tester.srv + + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Head(routePath, hand.FileExists) + + request, err := http.NewRequest("HEAD", reqPath, nil) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusOK, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func (tester *FileTester) TestGetFile(t *testing.T) { + var err error + + fmt.Printf("=== GetFile ===\n") + + srv := tester.srv + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Get(routePath, hand.GetFile) + + request, err := http.NewRequest("GET", reqPath, nil) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusOK, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func (tester *FileTester) TestDeleteFile(t *testing.T) { + var err error + + fmt.Printf("=== DeleteFile ===\n") + + srv := tester.srv + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Delete(routePath, hand.DeleteFile) + + request, err := http.NewRequest("DELETE", reqPath, nil) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusOK, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func (tester *FileTester) TestFileNotExists(t *testing.T) { + var err error + fmt.Printf("=== FileNotExists ===\n") + + srv := tester.srv + + require.NotNil(t, srv) + + reqPath := `/v3/api/file/foo/bare` + routePath := `/v3/api/file/{filepath}` + + rout := router.NewRouter() + hand := srv.Handler() + require.NotNil(t, hand) + + rout.Head(routePath, hand.FileExists) + + request, err := http.NewRequest("HEAD", reqPath, nil) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusNotFound, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} + +func (tester *FileTester) TestServiceHello(t *testing.T) { + var err error + + fmt.Printf("=== ServiceHello ===\n") + + srv := tester.srv + require.NotNil(t, srv) + + reqPath := "/service/hello" + routePath := "/service/hello" + + rout := router.NewRouter() + hand := srv.Handler() + rout.Get(routePath, hand.SendHello) + + request, err := http.NewRequest("GET", reqPath, nil) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + rout.ServeHTTP(recorder, request) + require.Equal(t, http.StatusOK, recorder.Code) + + fmt.Printf("Response code: %d\n", recorder.Code) + + bodyReader := recorder.Body + bodyBytes, err := io.ReadAll(bodyReader) + + fmt.Printf("Response body: %s\n", string(bodyBytes)) +} diff --git a/test/main_test.go b/test/main_test.go deleted file mode 100644 index 56e5404..0000000 --- a/test/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package test diff --git a/test/svc_test.go b/test/svc_test.go deleted file mode 100644 index 3ee5580..0000000 --- a/test/svc_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package test - -import ( - "github.com/stretchr/testify/require" - - "bytes" - "fmt" - "io" - "math/rand" - "net/http" - "net/http/httptest" - "testing" - - "mstore/app/router" - "mstore/app/server" -) - -func MakeServer(t *testing.T) *server.Server { - var err error - srv, err := server.NewServer() - require.NoError(t, err) - - err = srv.Configure() - require.NoError(t, err) - - err = srv.Build() - require.NoError(t, err) - - return srv -} - -func TestFileLife(t *testing.T) { -} - -func xxxTestFileExists(t *testing.T) { - var err error - - srv := MakeServer(t) - require.NotNil(t, srv) - - reqPath := `/v3/api/file/foo/bare` - routePath := `/v3/api/file/{filepath}` - - rout := router.NewRouter() - hand := srv.Handler() - require.NotNil(t, hand) - - rout.Head(routePath, hand.FileExists) - - request, err := http.NewRequest("HEAD", reqPath, nil) - require.NoError(t, err) - - recorder := httptest.NewRecorder() - rout.ServeHTTP(recorder, request) - require.Equal(t, http.StatusNotFound, recorder.Code) - - fmt.Printf("Response code: %d\n", recorder.Code) - - bodyReader := recorder.Body - bodyBytes, err := io.ReadAll(bodyReader) - - fmt.Printf("Response body: %s\n", string(bodyBytes)) -} - -func TestPutFile(t *testing.T) { - var err error - - srv := MakeServer(t) - require.NotNil(t, srv) - - reqPath := `/v3/api/file/foo/bare` - routePath := `/v3/api/file/{filepath}` - - rout := router.NewRouter() - hand := srv.Handler() - require.NotNil(t, hand) - - rout.Put(routePath, hand.PutFile) - - datasize := 16 - filedata := make([]byte, datasize) - _, err = rand.Read(filedata) - require.NoError(t, err) - - source := bytes.NewReader(filedata) - - request, err := http.NewRequest("PUT", reqPath, source) - require.NoError(t, err) - - request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize)) - request.Header.Set("Content-Type", "application/octet-stream") - - recorder := httptest.NewRecorder() - rout.ServeHTTP(recorder, request) - require.Equal(t, http.StatusOK, recorder.Code) - - fmt.Printf("Response code: %d\n", recorder.Code) - - bodyReader := recorder.Body - bodyBytes, err := io.ReadAll(bodyReader) - - fmt.Printf("Response body: %s\n", string(bodyBytes)) -} - -func xxxTestServiceHello(t *testing.T) { - var err error - - srv := MakeServer(t) - require.NotNil(t, srv) - - reqPath := "/service/hello" - routePath := "/service/hello" - - rout := router.NewRouter() - hand := srv.Handler() - rout.Get(routePath, hand.SendHello) - - request, err := http.NewRequest("GET", reqPath, nil) - require.NoError(t, err) - - recorder := httptest.NewRecorder() - rout.ServeHTTP(recorder, request) - require.Equal(t, http.StatusOK, recorder.Code) - - fmt.Printf("Response code: %d\n", recorder.Code) - - bodyReader := recorder.Body - bodyBytes, err := io.ReadAll(bodyReader) - - fmt.Printf("Response body: %s\n", string(bodyBytes)) -}