first draft of file storage

app/handler/file.go

@@ -1,6 +1,8 @@
 package handler
 
 import (
+	"io"
+
 	"mstore/app/operator"
 	"mstore/app/router"
 )
@@ -12,10 +14,18 @@ func (hand *Handler) FileExists(rctx *router.Context) {
 	params := &operator.FileExistsParams{
 		Filepath: filepath,
 	}
-	hand.logg.Debugf("filepath: %s", filepath)
 
-	code, _, _ := hand.oper.FileExists(params)
+	code, res, err := hand.oper.FileExists(params)
+	if err != nil {
+		hand.logg.Errorf("FileExists error: %v", err)
+		rctx.SetStatus(code)
+		return
+	}
+	rctx.SetHeader("Content-Type", res.ContentType)
+	rctx.SetHeader("Content-Length", res.ContentLength)
+	rctx.SetHeader("Content-Digest", res.ContentDigest)
 	rctx.SetStatus(code)
+
 }
 
 func (hand *Handler) PutFile(rctx *router.Context) {
@@ -31,12 +41,13 @@ func (hand *Handler) PutFile(rctx *router.Context) {
 		ContentType:   contentType,
 		Source:        rctx.Request.Body,
 	}
-	code, res, err := hand.oper.PutFile(params)
+	code, _, err := hand.oper.PutFile(params)
 	if err != nil {
-		hand.logg.Errorf("Error: %v", err)
+		hand.logg.Errorf("PutFile error: %v", err)
+		rctx.SetStatus(code)
+		return
 	}
 	rctx.SetStatus(code)
-	hand.SendResult(rctx, res)
 }
 
 func (hand *Handler) GetFile(rctx *router.Context) {
@@ -48,10 +59,25 @@ func (hand *Handler) GetFile(rctx *router.Context) {
 	}
 	hand.logg.Debugf("filepath: %s", filepath)
 
-	code, res, _ := hand.oper.GetFile(params)
+	code, res, err := hand.oper.GetFile(params)
+	if err != nil {
+		hand.logg.Errorf("PutFile error: %v", err)
+		rctx.SetStatus(code)
+		return
+	}
+	rctx.SetStatus(code)
 	rctx.SetHeader("Content-Type", res.ContentType)
 	rctx.SetHeader("Content-Length", res.ContentLength)
-	rctx.SetStatus(code)
+	rctx.SetHeader("Content-Digest", res.ContentDigest)
+
+	if res.Source != nil {
+		defer res.Source.Close()
+		_, err = io.Copy(rctx.Writer, res.Source)
+		if err != nil {
+			hand.logg.Errorf("GetFile error: %v", err)
+			return
+		}
+	}
 }
 
 func (hand *Handler) DeleteFile(rctx *router.Context) {
@@ -61,8 +87,9 @@ func (hand *Handler) DeleteFile(rctx *router.Context) {
 	params := &operator.DeleteFileParams{
 		Filepath: filepath,
 	}
-	hand.logg.Debugf("filepath: %s", filepath)
+	code, _, err := hand.oper.DeleteFile(params)
-
+	if err != nil {
-	code, _, _ := hand.oper.DeleteFile(params)
+		hand.logg.Errorf("GetFile error: %v", err)
+	}
 	rctx.SetStatus(code)
 }

app/operator/file.go

@@ -18,21 +18,28 @@ type FileExistsParams struct {
 	Dest     string
 }
 type FileExistsResult struct {
-	Descr *descr.File
+	ContentType   string
+	ContentLength string
+	ContentDigest string
 }
 
 func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) {
 	var err error
-	code := http.StatusNotFound
+	code := http.StatusOK
 	res := &FileExistsResult{}
 
 	filename := path.Base(param.Filepath)
 	collection := path.Dir(param.Filepath)
 
-	exist, file, err := oper.mdb.GetFileByCollection(collection, filename)
+	exist, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
-	if exist {
+	if !exist {
-		code = http.StatusOK
+		code = http.StatusNotFound
-		res.Descr = file
+		return code, res, err
+	}
+	res = &FileExistsResult{
+		ContentLength: strconv.FormatInt(fileDescr.Size, 10),
+		ContentType:   fileDescr.Type,
+		ContentDigest: fileDescr.Checksum,
 	}
 	return code, res, err
 }
@@ -44,9 +51,7 @@ type PutFileParams struct {
 	Filepath      string
 	Source        io.ReadCloser
 }
-type PutFileResult struct {
+type PutFileResult struct{}
-	Descr *descr.File
-}
 
 const defaultContentType = "application/octet-stream"
 
@@ -64,6 +69,8 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
 		contentType = defaultContentType
 	}
 
+	// TODO: convert file path to a unified and secure state
+
 	filename := path.Base(param.Filepath)
 	collection := path.Dir(param.Filepath)
 	oper.logg.Debugf("Put file %s %s", collection, filename)
@@ -74,14 +81,14 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
 		return code, res, err
 	}
 
-	exists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
+	descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
 	if err != nil {
 		code := http.StatusInternalServerError
 		return code, res, err
 	}
 
 	now := auxtool.TimeNow()
-	if exists {
+	if descrExists {
 		fileDescr.Size = size
 		fileDescr.Checksum = checksum
 		fileDescr.UpdatedAt = now
@@ -109,13 +116,11 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
 		}
 	}
 
-	err = oper.store.LinkFile(tmpname, collection, filename)
+	err = oper.store.HardlinkFile(tmpname, collection, filename)
 	if err != nil {
 		code := http.StatusInternalServerError
 		return code, res, err
 	}
-
-	res.Descr = fileDescr
 	code := http.StatusOK
 	return code, res, err
 }
@@ -127,12 +132,41 @@ type GetFileParams struct {
 type GetFileResult struct {
 	ContentType   string
 	ContentLength string
+	ContentDigest string
 	Source        io.ReadCloser
 }
 
 func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error) {
 	var err error
 	res := &GetFileResult{}
+
+	// TODO: convert file path to a unified and secure state
+
+	filename := path.Base(param.Filepath)
+	collection := path.Dir(param.Filepath)
+
+	oper.logg.Debugf("Gut file %s %s", collection, filename)
+
+	descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
+	if err != nil {
+		code := http.StatusInternalServerError
+		return code, res, err
+	}
+	if !descrExists {
+		code := http.StatusNotFound
+		return code, res, err
+	}
+	reader, err := oper.store.GetFileReader(collection, filename)
+	if err != nil {
+		code := http.StatusInternalServerError
+		return code, res, err
+	}
+	res = &GetFileResult{
+		ContentLength: strconv.FormatInt(fileDescr.Size, 10),
+		ContentType:   fileDescr.Type,
+		ContentDigest: fileDescr.Checksum,
+		Source:        reader,
+	}
 	code := http.StatusOK
 	return code, res, err
 }
@@ -147,5 +181,28 @@ func (oper *Operator) DeleteFile(param *DeleteFileParams) (int, *DeleteFileResul
 	var err error
 	res := &DeleteFileResult{}
 	code := http.StatusOK
+
+	filename := path.Base(param.Filepath)
+	collection := path.Dir(param.Filepath)
+
+	exist, _, err := oper.mdb.GetFileByCollection(collection, filename)
+	if err != nil {
+		code = http.StatusInternalServerError
+		return code, res, err
+	}
+	if exist {
+		err = oper.mdb.DeleteFileByCollection(collection, filename)
+		if err != nil {
+			code = http.StatusInternalServerError
+			return code, res, err
+		}
+	}
+	err = oper.store.DeleteFile(collection, filename)
+	if err != nil {
+		code = http.StatusInternalServerError
+		return code, res, err
+	}
+
 	return code, res, err
+
 }

app/router/router.go

@@ -46,6 +46,10 @@ func (rout *Router) Put(path string, handlerFunc HandlerFunc) {
 	rout.routeHandler.AddRoute("PUT", path, handlerFunc)
 }
 
+func (rout *Router) Delete(path string, handlerFunc HandlerFunc) {
+	rout.routeHandler.AddRoute("DELETE", path, handlerFunc)
+}
+
 func (rout *Router) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
 	rctx := NewContext(writer, req)
 	rout.routeHandler.ServeHTTP(rctx)

app/storage/storage.go

@@ -25,7 +25,20 @@ func NewStorage(basepath string) *Storage {
 	return res
 }
 
-func (store *Storage) LinkFile(tmpname, collection, filename string) error {
+func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
+	var err error
+	var res io.ReadCloser
+
+	filename = filepath.Join(store.basepath, collection, filename)
+	file, err := os.OpenFile(filename, os.O_RDONLY, 0)
+	if err != nil {
+		return res, err
+	}
+	res = file
+	return res, err
+}
+
+func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
 	var err error
 	dirname := filepath.Join(store.basepath, collection)
 	err = os.MkdirAll(dirname, 0750)
@@ -34,7 +47,7 @@ func (store *Storage) LinkFile(tmpname, collection, filename string) error {
 	}
 
 	filename = filepath.Join(store.basepath, collection, filename)
-	os.Remove(filename)
+	os.Remove(filename) // TODO
 
 	err = os.Link(tmpname, filename)
 	if err != nil {
@@ -74,3 +87,19 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
 
 	return tmppath, size, digest, err
 }
+
+func (store *Storage) DeleteFile(collection, filename string) error {
+	var err error
+	filename = filepath.Join(store.basepath, collection, filename)
+	err = os.Remove(filename)
+	if err != nil {
+		return err
+	}
+
+	dirname := filepath.Join(store.basepath, collection)
+	err = os.RemoveAll(dirname)
+	if err != nil {
+		return err
+	}
+	return err
+}

test/file_test.go

@@ -0,0 +1,251 @@
+package test
+
+import (
+	"github.com/stretchr/testify/require"
+
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"mstore/app/router"
+	"mstore/app/server"
+)
+
+func TestFileLife(t *testing.T) {
+	var tester FileTester
+	tester.MakeServer(t)
+	tester.TestServiceHello(t)
+	tester.TestPutFile(t)
+	tester.TestFileExists(t)
+	tester.TestGetFile(t)
+	tester.TestDeleteFile(t)
+	tester.TestFileNotExists(t)
+}
+
+type FileTester struct {
+	srv *server.Server
+}
+
+func (tester *FileTester) MakeServer(t *testing.T) {
+	var err error
+
+	fmt.Printf("=== MakeServer ===\n")
+
+	srv, err := server.NewServer()
+	require.NoError(t, err)
+
+	err = srv.Configure()
+	require.NoError(t, err)
+
+	err = srv.Build()
+	require.NoError(t, err)
+
+	tester.srv = srv
+}
+
+func (tester *FileTester) TestPutFile(t *testing.T) {
+	var err error
+
+	fmt.Printf("=== PutFile ===\n")
+
+	srv := tester.srv
+	require.NotNil(t, srv)
+
+	reqPath := `/v3/api/file/foo/bare`
+	routePath := `/v3/api/file/{filepath}`
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	require.NotNil(t, hand)
+
+	rout.Put(routePath, hand.PutFile)
+
+	datasize := 16
+	filedata := make([]byte, datasize)
+	_, err = rand.Read(filedata)
+	require.NoError(t, err)
+
+	filedata = []byte(hex.EncodeToString(filedata))
+
+	source := bytes.NewReader(filedata)
+
+	request, err := http.NewRequest("PUT", reqPath, source)
+	require.NoError(t, err)
+
+	request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
+	request.Header.Set("Content-Type", "application/octet-stream")
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusOK, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}
+
+func (tester *FileTester) TestFileExists(t *testing.T) {
+	var err error
+	fmt.Printf("=== FileExists ===\n")
+
+	srv := tester.srv
+
+	require.NotNil(t, srv)
+
+	reqPath := `/v3/api/file/foo/bare`
+	routePath := `/v3/api/file/{filepath}`
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	require.NotNil(t, hand)
+
+	rout.Head(routePath, hand.FileExists)
+
+	request, err := http.NewRequest("HEAD", reqPath, nil)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusOK, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}
+
+func (tester *FileTester) TestGetFile(t *testing.T) {
+	var err error
+
+	fmt.Printf("=== GetFile ===\n")
+
+	srv := tester.srv
+	require.NotNil(t, srv)
+
+	reqPath := `/v3/api/file/foo/bare`
+	routePath := `/v3/api/file/{filepath}`
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	require.NotNil(t, hand)
+
+	rout.Get(routePath, hand.GetFile)
+
+	request, err := http.NewRequest("GET", reqPath, nil)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusOK, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}
+
+func (tester *FileTester) TestDeleteFile(t *testing.T) {
+	var err error
+
+	fmt.Printf("=== DeleteFile ===\n")
+
+	srv := tester.srv
+	require.NotNil(t, srv)
+
+	reqPath := `/v3/api/file/foo/bare`
+	routePath := `/v3/api/file/{filepath}`
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	require.NotNil(t, hand)
+
+	rout.Delete(routePath, hand.DeleteFile)
+
+	request, err := http.NewRequest("DELETE", reqPath, nil)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusOK, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}
+
+func (tester *FileTester) TestFileNotExists(t *testing.T) {
+	var err error
+	fmt.Printf("=== FileNotExists ===\n")
+
+	srv := tester.srv
+
+	require.NotNil(t, srv)
+
+	reqPath := `/v3/api/file/foo/bare`
+	routePath := `/v3/api/file/{filepath}`
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	require.NotNil(t, hand)
+
+	rout.Head(routePath, hand.FileExists)
+
+	request, err := http.NewRequest("HEAD", reqPath, nil)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusNotFound, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}
+
+func (tester *FileTester) TestServiceHello(t *testing.T) {
+	var err error
+
+	fmt.Printf("=== ServiceHello ===\n")
+
+	srv := tester.srv
+	require.NotNil(t, srv)
+
+	reqPath := "/service/hello"
+	routePath := "/service/hello"
+
+	rout := router.NewRouter()
+	hand := srv.Handler()
+	rout.Get(routePath, hand.SendHello)
+
+	request, err := http.NewRequest("GET", reqPath, nil)
+	require.NoError(t, err)
+
+	recorder := httptest.NewRecorder()
+	rout.ServeHTTP(recorder, request)
+	require.Equal(t, http.StatusOK, recorder.Code)
+
+	fmt.Printf("Response code: %d\n", recorder.Code)
+
+	bodyReader := recorder.Body
+	bodyBytes, err := io.ReadAll(bodyReader)
+
+	fmt.Printf("Response body: %s\n", string(bodyBytes))
+}

test/main_test.go

@@ -1 +0,0 @@
-package test

test/svc_test.go

@@ -1,131 +0,0 @@
-package test
-
-import (
-	"github.com/stretchr/testify/require"
-
-	"bytes"
-	"fmt"
-	"io"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"mstore/app/router"
-	"mstore/app/server"
-)
-
-func MakeServer(t *testing.T) *server.Server {
-	var err error
-	srv, err := server.NewServer()
-	require.NoError(t, err)
-
-	err = srv.Configure()
-	require.NoError(t, err)
-
-	err = srv.Build()
-	require.NoError(t, err)
-
-	return srv
-}
-
-func TestFileLife(t *testing.T) {
-}
-
-func xxxTestFileExists(t *testing.T) {
-	var err error
-
-	srv := MakeServer(t)
-	require.NotNil(t, srv)
-
-	reqPath := `/v3/api/file/foo/bare`
-	routePath := `/v3/api/file/{filepath}`
-
-	rout := router.NewRouter()
-	hand := srv.Handler()
-	require.NotNil(t, hand)
-
-	rout.Head(routePath, hand.FileExists)
-
-	request, err := http.NewRequest("HEAD", reqPath, nil)
-	require.NoError(t, err)
-
-	recorder := httptest.NewRecorder()
-	rout.ServeHTTP(recorder, request)
-	require.Equal(t, http.StatusNotFound, recorder.Code)
-
-	fmt.Printf("Response code: %d\n", recorder.Code)
-
-	bodyReader := recorder.Body
-	bodyBytes, err := io.ReadAll(bodyReader)
-
-	fmt.Printf("Response body: %s\n", string(bodyBytes))
-}
-
-func TestPutFile(t *testing.T) {
-	var err error
-
-	srv := MakeServer(t)
-	require.NotNil(t, srv)
-
-	reqPath := `/v3/api/file/foo/bare`
-	routePath := `/v3/api/file/{filepath}`
-
-	rout := router.NewRouter()
-	hand := srv.Handler()
-	require.NotNil(t, hand)
-
-	rout.Put(routePath, hand.PutFile)
-
-	datasize := 16
-	filedata := make([]byte, datasize)
-	_, err = rand.Read(filedata)
-	require.NoError(t, err)
-
-	source := bytes.NewReader(filedata)
-
-	request, err := http.NewRequest("PUT", reqPath, source)
-	require.NoError(t, err)
-
-	request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
-	request.Header.Set("Content-Type", "application/octet-stream")
-
-	recorder := httptest.NewRecorder()
-	rout.ServeHTTP(recorder, request)
-	require.Equal(t, http.StatusOK, recorder.Code)
-
-	fmt.Printf("Response code: %d\n", recorder.Code)
-
-	bodyReader := recorder.Body
-	bodyBytes, err := io.ReadAll(bodyReader)
-
-	fmt.Printf("Response body: %s\n", string(bodyBytes))
-}
-
-func xxxTestServiceHello(t *testing.T) {
-	var err error
-
-	srv := MakeServer(t)
-	require.NotNil(t, srv)
-
-	reqPath := "/service/hello"
-	routePath := "/service/hello"
-
-	rout := router.NewRouter()
-	hand := srv.Handler()
-	rout.Get(routePath, hand.SendHello)
-
-	request, err := http.NewRequest("GET", reqPath, nil)
-	require.NoError(t, err)
-
-	recorder := httptest.NewRecorder()
-	rout.ServeHTTP(recorder, request)
-	require.Equal(t, http.StatusOK, recorder.Code)
-
-	fmt.Printf("Response code: %d\n", recorder.Code)
-
-	bodyReader := recorder.Body
-	bodyBytes, err := io.ReadAll(bodyReader)
-
-	fmt.Printf("Response body: %s\n", string(bodyBytes))
-}