ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

first draft of file storage

Browse Source
This commit is contained in:
Олег Бородин
2026-01-28 15:33:19 +02:00
parent d647854e98
commit d6496a427b
7 changed files with 394 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/handler/file.go
+37 -10
View File
@@ -1,6 +1,8 @@
package handler
import (
"io"
"mstore/app/operator"
"mstore/app/router"
)
@@ -12,10 +14,18 @@ func (hand *Handler) FileExists(rctx *router.Context) {
params := &operator.FileExistsParams{
Filepath: filepath,
}
hand.logg.Debugf("filepath: %s", filepath)
code, _, _ := hand.oper.FileExists(params)
code, res, err := hand.oper.FileExists(params)
if err != nil {
hand.logg.Errorf("FileExists error: %v", err)
rctx.SetStatus(code)
return
}
rctx.SetHeader("Content-Type", res.ContentType)
rctx.SetHeader("Content-Length", res.ContentLength)
rctx.SetHeader("Content-Digest", res.ContentDigest)
rctx.SetStatus(code)
}
func (hand *Handler) PutFile(rctx *router.Context) {
@@ -31,12 +41,13 @@ func (hand *Handler) PutFile(rctx *router.Context) {
ContentType: contentType,
Source: rctx.Request.Body,
}
code, res, err := hand.oper.PutFile(params)
code, _, err := hand.oper.PutFile(params)
if err != nil {
hand.logg.Errorf("Error: %v", err)
hand.logg.Errorf("PutFile error: %v", err)
rctx.SetStatus(code)
return
}
rctx.SetStatus(code)
hand.SendResult(rctx, res)
}
func (hand *Handler) GetFile(rctx *router.Context) {
@@ -48,10 +59,25 @@ func (hand *Handler) GetFile(rctx *router.Context) {
}
hand.logg.Debugf("filepath: %s", filepath)
code, res, _ := hand.oper.GetFile(params)
code, res, err := hand.oper.GetFile(params)
if err != nil {
hand.logg.Errorf("PutFile error: %v", err)
rctx.SetStatus(code)
return
}
rctx.SetStatus(code)
rctx.SetHeader("Content-Type", res.ContentType)
rctx.SetHeader("Content-Length", res.ContentLength)
rctx.SetStatus(code)
rctx.SetHeader("Content-Digest", res.ContentDigest)
if res.Source != nil {
defer res.Source.Close()
_, err = io.Copy(rctx.Writer, res.Source)
if err != nil {
hand.logg.Errorf("GetFile error: %v", err)
return
}
}
}
func (hand *Handler) DeleteFile(rctx *router.Context) {
@@ -61,8 +87,9 @@ func (hand *Handler) DeleteFile(rctx *router.Context) {
params := &operator.DeleteFileParams{
Filepath: filepath,
}
hand.logg.Debugf("filepath: %s", filepath)
code, _, _ := hand.oper.DeleteFile(params)
code, _, err := hand.oper.DeleteFile(params)
if err != nil {
hand.logg.Errorf("GetFile error: %v", err)
}
rctx.SetStatus(code)
}
app/operator/file.go
+71 -14
View File
@@ -18,21 +18,28 @@ type FileExistsParams struct {
Dest string
}
type FileExistsResult struct {
Descr *descr.File
ContentType string
ContentLength string
ContentDigest string
}
func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) {
var err error
code := http.StatusNotFound
code := http.StatusOK
res := &FileExistsResult{}
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
exist, file, err := oper.mdb.GetFileByCollection(collection, filename)
if exist {
code = http.StatusOK
res.Descr = file
exist, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
if !exist {
code = http.StatusNotFound
return code, res, err
}
res = &FileExistsResult{
ContentLength: strconv.FormatInt(fileDescr.Size, 10),
ContentType: fileDescr.Type,
ContentDigest: fileDescr.Checksum,
}
return code, res, err
}
@@ -44,9 +51,7 @@ type PutFileParams struct {
Filepath string
Source io.ReadCloser
}
type PutFileResult struct {
Descr *descr.File
}
type PutFileResult struct{}
const defaultContentType = "application/octet-stream"
@@ -64,6 +69,8 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
contentType = defaultContentType
}
// TODO: convert file path to a unified and secure state
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
oper.logg.Debugf("Put file %s %s", collection, filename)
@@ -74,14 +81,14 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
return code, res, err
}
exists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
now := auxtool.TimeNow()
if exists {
if descrExists {
fileDescr.Size = size
fileDescr.Checksum = checksum
fileDescr.UpdatedAt = now
@@ -109,13 +116,11 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
}
}
err = oper.store.LinkFile(tmpname, collection, filename)
err = oper.store.HardlinkFile(tmpname, collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Descr = fileDescr
code := http.StatusOK
return code, res, err
}
@@ -127,12 +132,41 @@ type GetFileParams struct {
type GetFileResult struct {
ContentType string
ContentLength string
ContentDigest string
Source io.ReadCloser
}
func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error) {
var err error
res := &GetFileResult{}
// TODO: convert file path to a unified and secure state
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
oper.logg.Debugf("Gut file %s %s", collection, filename)
descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
if !descrExists {
code := http.StatusNotFound
return code, res, err
}
reader, err := oper.store.GetFileReader(collection, filename)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res = &GetFileResult{
ContentLength: strconv.FormatInt(fileDescr.Size, 10),
ContentType: fileDescr.Type,
ContentDigest: fileDescr.Checksum,
Source: reader,
}
code := http.StatusOK
return code, res, err
}
@@ -147,5 +181,28 @@ func (oper *Operator) DeleteFile(param *DeleteFileParams) (int, *DeleteFileResul
var err error
res := &DeleteFileResult{}
code := http.StatusOK
filename := path.Base(param.Filepath)
collection := path.Dir(param.Filepath)
exist, _, err := oper.mdb.GetFileByCollection(collection, filename)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
if exist {
err = oper.mdb.DeleteFileByCollection(collection, filename)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
}
err = oper.store.DeleteFile(collection, filename)
if err != nil {
code = http.StatusInternalServerError
return code, res, err
}
return code, res, err
}
app/router/router.go
+4
View File
@@ -46,6 +46,10 @@ func (rout *Router) Put(path string, handlerFunc HandlerFunc) {
rout.routeHandler.AddRoute("PUT", path, handlerFunc)
}
func (rout *Router) Delete(path string, handlerFunc HandlerFunc) {
rout.routeHandler.AddRoute("DELETE", path, handlerFunc)
}
func (rout *Router) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
rctx := NewContext(writer, req)
rout.routeHandler.ServeHTTP(rctx)
app/storage/storage.go
+31 -2
View File
@@ -25,7 +25,20 @@ func NewStorage(basepath string) *Storage {
return res
}
func (store *Storage) LinkFile(tmpname, collection, filename string) error {
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = filepath.Join(store.basepath, collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
var err error
dirname := filepath.Join(store.basepath, collection)
err = os.MkdirAll(dirname, 0750)
@@ -34,7 +47,7 @@ func (store *Storage) LinkFile(tmpname, collection, filename string) error {
}
filename = filepath.Join(store.basepath, collection, filename)
os.Remove(filename)
os.Remove(filename) // TODO
err = os.Link(tmpname, filename)
if err != nil {
@@ -74,3 +87,19 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
return tmppath, size, digest, err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = filepath.Join(store.basepath, collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
dirname := filepath.Join(store.basepath, collection)
err = os.RemoveAll(dirname)
if err != nil {
return err
}
return err
}
test/file_test.go
+251
View File
@@ -0,0 +1,251 @@
package test
import (
"github.com/stretchr/testify/require"
"bytes"
"encoding/hex"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"testing"
"mstore/app/router"
"mstore/app/server"
)
func TestFileLife(t *testing.T) {
var tester FileTester
tester.MakeServer(t)
tester.TestServiceHello(t)
tester.TestPutFile(t)
tester.TestFileExists(t)
tester.TestGetFile(t)
tester.TestDeleteFile(t)
tester.TestFileNotExists(t)
}
type FileTester struct {
srv *server.Server
}
func (tester *FileTester) MakeServer(t *testing.T) {
var err error
fmt.Printf("=== MakeServer ===\n")
srv, err := server.NewServer()
require.NoError(t, err)
err = srv.Configure()
require.NoError(t, err)
err = srv.Build()
require.NoError(t, err)
tester.srv = srv
}
func (tester *FileTester) TestPutFile(t *testing.T) {
var err error
fmt.Printf("=== PutFile ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Put(routePath, hand.PutFile)
datasize := 16
filedata := make([]byte, datasize)
_, err = rand.Read(filedata)
require.NoError(t, err)
filedata = []byte(hex.EncodeToString(filedata))
source := bytes.NewReader(filedata)
request, err := http.NewRequest("PUT", reqPath, source)
require.NoError(t, err)
request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
request.Header.Set("Content-Type", "application/octet-stream")
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func (tester *FileTester) TestFileExists(t *testing.T) {
var err error
fmt.Printf("=== FileExists ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Head(routePath, hand.FileExists)
request, err := http.NewRequest("HEAD", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func (tester *FileTester) TestGetFile(t *testing.T) {
var err error
fmt.Printf("=== GetFile ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Get(routePath, hand.GetFile)
request, err := http.NewRequest("GET", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func (tester *FileTester) TestDeleteFile(t *testing.T) {
var err error
fmt.Printf("=== DeleteFile ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Delete(routePath, hand.DeleteFile)
request, err := http.NewRequest("DELETE", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func (tester *FileTester) TestFileNotExists(t *testing.T) {
var err error
fmt.Printf("=== FileNotExists ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Head(routePath, hand.FileExists)
request, err := http.NewRequest("HEAD", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusNotFound, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func (tester *FileTester) TestServiceHello(t *testing.T) {
var err error
fmt.Printf("=== ServiceHello ===\n")
srv := tester.srv
require.NotNil(t, srv)
reqPath := "/service/hello"
routePath := "/service/hello"
rout := router.NewRouter()
hand := srv.Handler()
rout.Get(routePath, hand.SendHello)
request, err := http.NewRequest("GET", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
test/main_test.go
-1
View File
@@ -1 +0,0 @@
package test
test/svc_test.go
-131
View File
@@ -1,131 +0,0 @@
package test
import (
"github.com/stretchr/testify/require"
"bytes"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"testing"
"mstore/app/router"
"mstore/app/server"
)
func MakeServer(t *testing.T) *server.Server {
var err error
srv, err := server.NewServer()
require.NoError(t, err)
err = srv.Configure()
require.NoError(t, err)
err = srv.Build()
require.NoError(t, err)
return srv
}
func TestFileLife(t *testing.T) {
}
func xxxTestFileExists(t *testing.T) {
var err error
srv := MakeServer(t)
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Head(routePath, hand.FileExists)
request, err := http.NewRequest("HEAD", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusNotFound, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func TestPutFile(t *testing.T) {
var err error
srv := MakeServer(t)
require.NotNil(t, srv)
reqPath := `/v3/api/file/foo/bare`
routePath := `/v3/api/file/{filepath}`
rout := router.NewRouter()
hand := srv.Handler()
require.NotNil(t, hand)
rout.Put(routePath, hand.PutFile)
datasize := 16
filedata := make([]byte, datasize)
_, err = rand.Read(filedata)
require.NoError(t, err)
source := bytes.NewReader(filedata)
request, err := http.NewRequest("PUT", reqPath, source)
require.NoError(t, err)
request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
request.Header.Set("Content-Type", "application/octet-stream")
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}
func xxxTestServiceHello(t *testing.T) {
var err error
srv := MakeServer(t)
require.NotNil(t, srv)
reqPath := "/service/hello"
routePath := "/service/hello"
rout := router.NewRouter()
hand := srv.Handler()
rout.Get(routePath, hand.SendHello)
request, err := http.NewRequest("GET", reqPath, nil)
require.NoError(t, err)
recorder := httptest.NewRecorder()
rout.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
fmt.Printf("Response code: %d\n", recorder.Code)
bodyReader := recorder.Body
bodyBytes, err := io.ReadAll(bodyReader)
fmt.Printf("Response body: %s\n", string(bodyBytes))
}