app/imageoper, app/maindb: one image - one arch; app/storage: blob to name subdir
This commit is contained in:
+7
-2
@@ -40,7 +40,6 @@ EXTRA_mstorectl_SOURCES = \
|
|||||||
cmd/mstorectl/imagecmd/imageman.go \
|
cmd/mstorectl/imagecmd/imageman.go \
|
||||||
cmd/mstorectl/imagecmd/imagetags.go \
|
cmd/mstorectl/imagecmd/imagetags.go \
|
||||||
cmd/mstorectl/imagecmd/printresp.go \
|
cmd/mstorectl/imagecmd/printresp.go \
|
||||||
cmd/mstorectl/imagecmd/gcrimageconf.go \
|
|
||||||
cmd/mstorectl/imagecmd/pullimage.go \
|
cmd/mstorectl/imagecmd/pullimage.go \
|
||||||
cmd/mstorectl/imagecmd/pushimage.go
|
cmd/mstorectl/imagecmd/pushimage.go
|
||||||
|
|
||||||
@@ -128,7 +127,13 @@ EXTRA_mstored_SOURCES += \
|
|||||||
app/router/router.go \
|
app/router/router.go \
|
||||||
app/server/server.go \
|
app/server/server.go \
|
||||||
app/service/service.go \
|
app/service/service.go \
|
||||||
app/storage/storage.go \
|
\
|
||||||
|
app/storage/blob.go \
|
||||||
|
app/storage/file.go \
|
||||||
|
app/storage/hasher.go \
|
||||||
|
app/storage/helm.go \
|
||||||
|
app/storage/store.go \
|
||||||
|
\
|
||||||
pkg/accntcli/client.go \
|
pkg/accntcli/client.go \
|
||||||
pkg/accntcli/createacc.go \
|
pkg/accntcli/createacc.go \
|
||||||
pkg/accntcli/creategrant.go \
|
pkg/accntcli/creategrant.go \
|
||||||
|
|||||||
+2
-2
@@ -386,7 +386,6 @@ EXTRA_mstorectl_SOURCES = \
|
|||||||
cmd/mstorectl/imagecmd/imageman.go \
|
cmd/mstorectl/imagecmd/imageman.go \
|
||||||
cmd/mstorectl/imagecmd/imagetags.go \
|
cmd/mstorectl/imagecmd/imagetags.go \
|
||||||
cmd/mstorectl/imagecmd/printresp.go \
|
cmd/mstorectl/imagecmd/printresp.go \
|
||||||
cmd/mstorectl/imagecmd/gcrimageconf.go \
|
|
||||||
cmd/mstorectl/imagecmd/pullimage.go \
|
cmd/mstorectl/imagecmd/pullimage.go \
|
||||||
cmd/mstorectl/imagecmd/pushimage.go
|
cmd/mstorectl/imagecmd/pushimage.go
|
||||||
|
|
||||||
@@ -424,7 +423,8 @@ EXTRA_mstored_SOURCES = cmd/mstored/starter/starter.go \
|
|||||||
app/router/corsmw.go app/router/loggingmw.go \
|
app/router/corsmw.go app/router/loggingmw.go \
|
||||||
app/router/pathc.go app/router/recovermw.go \
|
app/router/pathc.go app/router/recovermw.go \
|
||||||
app/router/router.go app/server/server.go \
|
app/router/router.go app/server/server.go \
|
||||||
app/service/service.go app/storage/storage.go \
|
app/service/service.go app/storage/blob.go app/storage/file.go \
|
||||||
|
app/storage/hasher.go app/storage/helm.go app/storage/store.go \
|
||||||
pkg/accntcli/client.go pkg/accntcli/createacc.go \
|
pkg/accntcli/client.go pkg/accntcli/createacc.go \
|
||||||
pkg/accntcli/creategrant.go pkg/accntcli/delacc.go \
|
pkg/accntcli/creategrant.go pkg/accntcli/delacc.go \
|
||||||
pkg/accntcli/delgrant.go pkg/accntcli/getacc.go \
|
pkg/accntcli/delgrant.go pkg/accntcli/getacc.go \
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
|
|||||||
name, _ := rctx.GetSubpath("name")
|
name, _ := rctx.GetSubpath("name")
|
||||||
reference, _ := rctx.GetSubpath("reference")
|
reference, _ := rctx.GetSubpath("reference")
|
||||||
|
|
||||||
//hand.DumpHeaders("ManigestExists:\n", rctx)
|
hand.DumpHeaders("ManigestExists:\n", rctx)
|
||||||
|
|
||||||
params := &imageoper.ManifestExistsParams{
|
params := &imageoper.ManifestExistsParams{
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -48,6 +48,8 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
|
|||||||
rctx.SetHeader("Content-Type", res.ContentType)
|
rctx.SetHeader("Content-Type", res.ContentType)
|
||||||
rctx.SetHeader("Docker-Content-Digest", res.DockerContentDigest)
|
rctx.SetHeader("Docker-Content-Digest", res.DockerContentDigest)
|
||||||
}
|
}
|
||||||
|
hand.logg.Debugf(res.ContentType)
|
||||||
|
hand.logg.Debugf(res.DockerContentDigest)
|
||||||
rctx.SetStatus(code)
|
rctx.SetStatus(code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
//"net/http"
|
||||||
|
|
||||||
"mstore/app/imageoper"
|
"mstore/app/imageoper"
|
||||||
"mstore/app/router"
|
"mstore/app/router"
|
||||||
@@ -21,12 +21,14 @@ func (hand *Handler) GetVersion(rctx *router.Context) {
|
|||||||
params := &imageoper.GetVersionParams{}
|
params := &imageoper.GetVersionParams{}
|
||||||
|
|
||||||
//hand.DumpHeaders("GetVersion", rctx)
|
//hand.DumpHeaders("GetVersion", rctx)
|
||||||
|
/*
|
||||||
authorization := rctx.GetHeader("Authorization")
|
authorization := rctx.GetHeader("Authorization")
|
||||||
if authorization == "" {
|
if authorization == "" {
|
||||||
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
|
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
|
||||||
rctx.SetStatus(http.StatusUnauthorized)
|
rctx.SetStatus(http.StatusUnauthorized)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
ctx := rctx.GetContext()
|
ctx := rctx.GetContext()
|
||||||
_, code, err := hand.imop.GetVersion(ctx, params)
|
_, code, err := hand.imop.GetVersion(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ func (oper *Operator) BlobExists(ctx context.Context, operatorID string, params
|
|||||||
return res, http.StatusNotFound, err
|
return res, http.StatusNotFound, err
|
||||||
}
|
}
|
||||||
// Check blob file
|
// Check blob file
|
||||||
blobExists, _, err := oper.store.BlobExists(params.Digest)
|
blobExists, _, err := oper.store.BlobExists(params.Name, params.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,13 +59,13 @@ func (oper *Operator) DeleteBlob(ctx context.Context, operatorID string, params
|
|||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
// Removing the blob binary if usage == 0
|
// Removing the blob binary if usage == 0
|
||||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Digest)
|
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Name, params.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
if blobUsage == 0 {
|
if blobUsage == 0 {
|
||||||
oper.logg.Warningf("Deleting useless blob binary %s", params.Digest)
|
oper.logg.Warningf("Deleting useless blob binary %s", params.Digest)
|
||||||
oper.store.DeleteBlob(params.Digest)
|
oper.store.DeleteBlob(params.Name, params.Digest)
|
||||||
}
|
}
|
||||||
return res, http.StatusOK, err
|
return res, http.StatusOK, err
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-3
@@ -64,6 +64,7 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
|
|||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
/*
|
||||||
// Check manifest by name and reference
|
// Check manifest by name and reference
|
||||||
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -80,6 +81,19 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
|
|||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
// Check manifest by name and reference
|
||||||
|
exists, mandescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||||
|
if err != nil {
|
||||||
|
return res, http.StatusInternalServerError, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return res, http.StatusNotFound, err
|
||||||
|
}
|
||||||
|
err = oper.deleteManifestObjects(ctx, mandescr.Name, mandescr.Reference)
|
||||||
|
if err != nil {
|
||||||
|
return res, http.StatusInternalServerError, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Get blobs associated with the name
|
// Get blobs associated with the name
|
||||||
return res, http.StatusAccepted, err
|
return res, http.StatusAccepted, err
|
||||||
@@ -99,19 +113,19 @@ func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Check blob file
|
// Check blob file
|
||||||
exists, _, err := oper.store.BlobExists(blob.Digest)
|
exists, _, err := oper.store.BlobExists(name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if exists {
|
if exists {
|
||||||
// Check blob usage
|
// Check blob usage
|
||||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
|
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Delete if blob useless
|
// Delete if blob useless
|
||||||
if blobUsage == 0 {
|
if blobUsage == 0 {
|
||||||
err = oper.store.DeleteBlob(blob.Digest)
|
err = oper.store.DeleteBlob(name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
|
|||||||
oper.iLock.WaitAndLock(resName)
|
oper.iLock.WaitAndLock(resName)
|
||||||
defer oper.iLock.Done(resName)
|
defer oper.iLock.Done(resName)
|
||||||
|
|
||||||
blobExists, blobSize, err := oper.store.BlobExists(params.Digest)
|
blobExists, blobSize, err := oper.store.BlobExists(params.Name, params.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
@@ -57,7 +57,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
|
|||||||
return res, http.StatusNotFound, err
|
return res, http.StatusNotFound, err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, readCloser, err := oper.store.BlobReader(params.Digest)
|
_, readCloser, err := oper.store.BlobReader(params.Name, params.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
|
|||||||
res.Payload = manDescr.Payload
|
res.Payload = manDescr.Payload
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
/*
|
||||||
// Create index of manifests. Or not.
|
// Create index of manifests. Or not.
|
||||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -94,6 +95,20 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
|
|||||||
res.ContentType = oiiMediaType
|
res.ContentType = oiiMediaType
|
||||||
res.Payload = string(indexdata)
|
res.Payload = string(indexdata)
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||||
|
if err != nil {
|
||||||
|
return res, http.StatusInternalServerError, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return res, http.StatusNotFound, err
|
||||||
|
}
|
||||||
|
res.DockerContentDigest = manDescr.Digest
|
||||||
|
size := int64(len(manDescr.Payload))
|
||||||
|
res.ContentLength = strconv.FormatInt(size, 10)
|
||||||
|
res.ContentType = manDescr.ContentType
|
||||||
|
res.Payload = manDescr.Payload
|
||||||
|
|
||||||
}
|
}
|
||||||
return res, http.StatusOK, err
|
return res, http.StatusOK, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
|
|||||||
res.DockerContentDigest = man.Digest
|
res.DockerContentDigest = man.Digest
|
||||||
res.Exists = exist
|
res.Exists = exist
|
||||||
} else {
|
} else {
|
||||||
|
/*
|
||||||
// Create index of manifests
|
// Create index of manifests
|
||||||
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -70,6 +71,13 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
|
|||||||
if !exists {
|
if !exists {
|
||||||
return res, http.StatusNotFound, err
|
return res, http.StatusNotFound, err
|
||||||
}
|
}
|
||||||
|
if len(manDescrs) == 1 {
|
||||||
|
manDescr := manDescrs[0]
|
||||||
|
res.DockerContentDigest = manDescr.Digest
|
||||||
|
size := int64(len(manDescr.Payload))
|
||||||
|
res.ContentLength = strconv.FormatInt(size, 10)
|
||||||
|
res.ContentType = manDescr.ContentType
|
||||||
|
} else {
|
||||||
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
_, indexdata, err := indexFromManigestDescrs(manDescrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
@@ -81,5 +89,18 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
|
|||||||
res.ContentType = oiiMediaType
|
res.ContentType = oiiMediaType
|
||||||
res.Exists = true
|
res.Exists = true
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
|
||||||
|
if err != nil {
|
||||||
|
return res, http.StatusInternalServerError, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return res, http.StatusNotFound, err
|
||||||
|
}
|
||||||
|
res.DockerContentDigest = manDescr.Digest
|
||||||
|
size := int64(len(manDescr.Payload))
|
||||||
|
res.ContentLength = strconv.FormatInt(size, 10)
|
||||||
|
res.ContentType = manDescr.ContentType
|
||||||
|
}
|
||||||
return res, http.StatusOK, err
|
return res, http.StatusOK, err
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-12
@@ -111,6 +111,7 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
|||||||
name := params.Name
|
name := params.Name
|
||||||
reference := params.Reference
|
reference := params.Reference
|
||||||
|
|
||||||
|
/*
|
||||||
arch := inMan.Subject.Platform.Architecture
|
arch := inMan.Subject.Platform.Architecture
|
||||||
os := inMan.Subject.Platform.OS
|
os := inMan.Subject.Platform.OS
|
||||||
variant := inMan.Subject.Platform.Variant
|
variant := inMan.Subject.Platform.Variant
|
||||||
@@ -119,13 +120,17 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
manexist, exMandescr, err := oper.mdb.GetManifestByReference(ctx, name, reference)
|
||||||
|
if err != nil {
|
||||||
|
return res, http.StatusInternalServerError, err
|
||||||
|
}
|
||||||
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
|
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
|
||||||
inManDescr.Digest = digstr
|
inManDescr.Digest = digstr
|
||||||
// Always check layer files for availability
|
// Always check layer files for availability
|
||||||
var blobError error
|
var blobError error
|
||||||
for _, blobDescr := range inlayerdescrs {
|
for _, blobDescr := range inlayerdescrs {
|
||||||
blobExists, _, err := oper.store.BlobExists(blobDescr.Digest)
|
blobExists, _, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
@@ -151,7 +156,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
|||||||
return res, http.StatusCreated, err
|
return res, http.StatusCreated, err
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
exManData := []byte(exMandescr.Payload)
|
exManData := []byte(exMandescr.Payload)
|
||||||
exMan := &ocispec.Manifest{}
|
exMan := &ocispec.Manifest{}
|
||||||
err := json.Unmarshal(exManData, exMan)
|
err := json.Unmarshal(exManData, exMan)
|
||||||
@@ -163,25 +167,22 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Starting manifest and blobs transaction
|
// Starting manifest and blobs transaction
|
||||||
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
|
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
|
|
||||||
//goto end
|
|
||||||
for _, blob := range uselessBlobDescrs {
|
for _, blob := range uselessBlobDescrs {
|
||||||
exists, _, err := oper.store.BlobExists(blob.Digest)
|
exists, _, err := oper.store.BlobExists(blob.Name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
|
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
if exists && blobUsage == 0 {
|
if exists && blobUsage == 0 {
|
||||||
err = oper.store.DeleteBlob(blob.Digest)
|
err = oper.store.DeleteBlob(blob.Name, blob.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
}
|
}
|
||||||
@@ -202,9 +203,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//end:
|
|
||||||
|
|
||||||
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
|
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
|
||||||
return res, http.StatusCreated, err
|
return res, http.StatusCreated, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *
|
|||||||
res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest)
|
res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = oper.store.LinkUpload(params.Reference, params.Digest)
|
err = oper.store.LinkUpload(params.Reference, params.Name, params.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err)
|
err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err)
|
||||||
return res, http.StatusInternalServerError, err
|
return res, http.StatusInternalServerError, err
|
||||||
|
|||||||
+3
-3
@@ -103,12 +103,12 @@ func (db *Database) GetBlobsByReferense(ctx context.Context, name, reference str
|
|||||||
return blobs, err
|
return blobs, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Database) GetBlobUsage(ctx context.Context, digest string) (int64, error) {
|
func (db *Database) GetBlobUsage(ctx context.Context, name, digest string) (int64, error) {
|
||||||
var err error
|
var err error
|
||||||
var usage int64
|
var usage int64
|
||||||
count := make([]int64, 0)
|
count := make([]int64, 0)
|
||||||
request := `SELECT count(id) AS count FROM blobs WHERE digest = $1`
|
request := `SELECT count(id) AS count FROM blobs WHERE name = $1 AND digest = $2`
|
||||||
err = db.db.Select(&count, request, digest)
|
err = db.db.Select(&count, request, name, digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return usage, err
|
return usage, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -209,23 +209,24 @@ func (db *Database) GetManifestByDigest(ctx context.Context, name, digest string
|
|||||||
return exists, manifest, err
|
return exists, manifest, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Database) GetManifestsByReference(ctx context.Context, name, reference string) (bool, []descr.Manifest, error) {
|
func (db *Database) GetManifestByReference(ctx context.Context, name, reference string) (bool, descr.Manifest, error) {
|
||||||
var err error
|
var err error
|
||||||
exists := false
|
exists := false
|
||||||
|
manifest := descr.Manifest{}
|
||||||
manifests := make([]descr.Manifest, 0)
|
manifests := make([]descr.Manifest, 0)
|
||||||
request := `SELECT * FROM manifests WHERE name = $1 AND reference = $2 LIMIT 1`
|
request := `SELECT * FROM manifests WHERE name = $1 AND reference = $2 LIMIT 1`
|
||||||
err = db.db.Select(&manifests, request, name, reference)
|
err = db.db.Select(&manifests, request, name, reference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return exists, manifests, err
|
return exists, manifest, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(manifests) > 0 {
|
if len(manifests) > 0 {
|
||||||
|
manifest = manifests[0]
|
||||||
exists = true
|
exists = true
|
||||||
}
|
}
|
||||||
return exists, manifests, err
|
return exists, manifest, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Database) GetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
|
func (db *Database) xxxGetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
|
||||||
var err error
|
var err error
|
||||||
exists := false
|
exists := false
|
||||||
manifest := descr.Manifest{}
|
manifest := descr.Manifest{}
|
||||||
|
|||||||
@@ -43,8 +43,11 @@ const schema = `
|
|||||||
os VARCHAR(255) NOT NULL,
|
os VARCHAR(255) NOT NULL,
|
||||||
variant VARCHAR(255) NOT NULL
|
variant VARCHAR(255) NOT NULL
|
||||||
);
|
);
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index
|
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index01
|
||||||
ON manifests(name, reference, architecture, os, variant);
|
ON manifests(name, reference, architecture, os, variant);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index02
|
||||||
|
ON manifests(name, reference);
|
||||||
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS blobs (
|
CREATE TABLE IF NOT EXISTS blobs (
|
||||||
id VARCHAR(255) NOT NULL,
|
id VARCHAR(255) NOT NULL,
|
||||||
|
|||||||
@@ -0,0 +1,211 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||||
|
*
|
||||||
|
* This work is published and licensed under a Creative Commons
|
||||||
|
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||||
|
*
|
||||||
|
* Distribution of this work is permitted, but commercial use and
|
||||||
|
* modifications are strictly prohibited.
|
||||||
|
*/
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
upsubdir = "uploads"
|
||||||
|
blobsubdir = "blobs"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (store *Storage) makeUppath(upname string) string {
|
||||||
|
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeUpsubdir() string {
|
||||||
|
return filepath.Join(store.basepath, upsubdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeBlobpath(name, digstr string) string {
|
||||||
|
return filepath.Join(store.basepath, blobsubdir, name, digstr) + ".bin"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeBlobsubdir(name string) string {
|
||||||
|
return filepath.Join(store.basepath, blobsubdir, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
|
||||||
|
var err error
|
||||||
|
var recsize int64
|
||||||
|
|
||||||
|
uploadDir := store.makeUpsubdir()
|
||||||
|
_, err = os.Stat(uploadDir)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = os.MkdirAll(uploadDir, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return recsize, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return recsize, err
|
||||||
|
}
|
||||||
|
upPath := store.makeUppath(upID)
|
||||||
|
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return recsize, err
|
||||||
|
}
|
||||||
|
defer upFile.Close()
|
||||||
|
recsize, err = io.Copy(upFile, source)
|
||||||
|
if err != nil {
|
||||||
|
return recsize, err
|
||||||
|
}
|
||||||
|
return recsize, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) LinkUpload(reference, name, digest string) error {
|
||||||
|
var err error
|
||||||
|
upPath := store.makeUppath(reference)
|
||||||
|
|
||||||
|
blobdir := store.makeBlobsubdir(name)
|
||||||
|
_, err = os.Stat(blobdir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
err = os.MkdirAll(blobdir, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
blobPath := store.makeBlobpath(name, digest)
|
||||||
|
_, err = os.Stat(blobPath)
|
||||||
|
if err == nil {
|
||||||
|
err = os.Remove(blobPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = os.Link(upPath, blobPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = os.Remove(upPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) RemoveUpload(digest string) error {
|
||||||
|
var err error
|
||||||
|
upPath := store.makeUppath(digest)
|
||||||
|
err = os.Remove(upPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
|
||||||
|
var err error
|
||||||
|
var size int64
|
||||||
|
upPath := st.makeUppath(reference)
|
||||||
|
fileStat, err := os.Stat(upPath)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, 0, err
|
||||||
|
}
|
||||||
|
size = fileStat.Size()
|
||||||
|
return true, size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) WriteBlob(name, digstr string, source io.Reader) (int64, error) {
|
||||||
|
var err error
|
||||||
|
var size int64
|
||||||
|
blobDir := store.makeBlobsubdir(name)
|
||||||
|
_, err = os.Stat(blobDir)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = os.MkdirAll(blobDir, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
blobPath := store.makeBlobpath(name, digstr)
|
||||||
|
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
defer blobFile.Close()
|
||||||
|
size, err = io.Copy(blobFile, source)
|
||||||
|
if err != nil {
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *Storage) BlobExists(name, digest string) (bool, int64, error) {
|
||||||
|
var err error
|
||||||
|
var size int64
|
||||||
|
|
||||||
|
blobPath := st.makeBlobpath(name, digest)
|
||||||
|
fileStat, err := os.Stat(blobPath)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return false, 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, 0, err
|
||||||
|
}
|
||||||
|
size = fileStat.Size()
|
||||||
|
return true, size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) BlobReader(name, digest string) (int64, io.ReadCloser, error) {
|
||||||
|
var err error
|
||||||
|
var size int64
|
||||||
|
blobPath := store.makeBlobpath(name, digest)
|
||||||
|
nop := io.NopCloser(bytes.NewReader(nil))
|
||||||
|
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
return size, nop, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
file.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
filestat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return size, nop, err
|
||||||
|
}
|
||||||
|
size = filestat.Size()
|
||||||
|
return size, file, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) DeleteBlob(name, digest string) error {
|
||||||
|
var err error
|
||||||
|
blobPath := store.makeBlobpath(name, digest)
|
||||||
|
err = os.Remove(blobPath)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// TODO: remove dirs
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||||
|
*
|
||||||
|
* This work is published and licensed under a Creative Commons
|
||||||
|
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||||
|
*
|
||||||
|
* Distribution of this work is permitted, but commercial use and
|
||||||
|
* modifications are strictly prohibited.
|
||||||
|
*/
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"mstore/pkg/auxuuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
sha256prefix = "sha256:"
|
||||||
|
fileSubdir = "files"
|
||||||
|
tmpSubdir = "tmps"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (store *Storage) makeCollecionpath(collection string) string {
|
||||||
|
return filepath.Join(store.basepath, fileSubdir, collection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeFilepath(collection, filename string) string {
|
||||||
|
return filepath.Join(store.basepath, fileSubdir, collection, filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeTmppath(tmpName string) string {
|
||||||
|
return filepath.Join(store.basepath, tmpSubdir, tmpName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeTmpsubdir() string {
|
||||||
|
return filepath.Join(store.basepath, tmpSubdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeFilesubdir(collection, filename string) string {
|
||||||
|
return filepath.Join(store.basepath, fileSubdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
||||||
|
var err error
|
||||||
|
var res io.ReadCloser
|
||||||
|
|
||||||
|
filename = store.makeFilepath(collection, filename)
|
||||||
|
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
res = file
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
|
||||||
|
var err error
|
||||||
|
var size int64
|
||||||
|
var csum string
|
||||||
|
|
||||||
|
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
|
||||||
|
tmpPath := store.makeTmppath(tmpName)
|
||||||
|
|
||||||
|
tmpdirpath := store.makeTmpsubdir()
|
||||||
|
err = os.MkdirAll(tmpdirpath, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return tmpName, size, csum, err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
|
||||||
|
if err != nil {
|
||||||
|
return tmpName, size, csum, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
hasher := NewHasher()
|
||||||
|
mWriter := io.MultiWriter(file, hasher.Writer())
|
||||||
|
size, err = io.Copy(mWriter, source)
|
||||||
|
if err != nil {
|
||||||
|
return tmpName, size, csum, err
|
||||||
|
}
|
||||||
|
csum = hasher.Hex()
|
||||||
|
|
||||||
|
return tmpName, size, csum, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
dirname := store.makeCollecionpath(collection)
|
||||||
|
_, err = os.Stat(dirname)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
err = os.MkdirAll(dirname, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
filename = store.makeFilepath(collection, filename)
|
||||||
|
_, err = os.Stat(dirname)
|
||||||
|
if err == nil {
|
||||||
|
os.Remove(filename) // TODO: safe removing
|
||||||
|
}
|
||||||
|
tmpName = store.makeTmppath(tmpName)
|
||||||
|
err = os.Link(tmpName, filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = os.Remove(tmpName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) DeleteFile(collection, filename string) error {
|
||||||
|
var err error
|
||||||
|
filename = store.makeFilepath(collection, filename)
|
||||||
|
err = os.Remove(filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// TODO: clean removing
|
||||||
|
dirname := store.makeCollecionpath(collection)
|
||||||
|
os.RemoveAll(dirname)
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -1,341 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
|
||||||
*
|
|
||||||
* This work is published and licensed under a Creative Commons
|
|
||||||
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
|
||||||
*
|
|
||||||
* Distribution of this work is permitted, but commercial use and
|
|
||||||
* modifications are strictly prohibited.
|
|
||||||
*/
|
|
||||||
package storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"mstore/app/logger"
|
|
||||||
"mstore/pkg/auxuuid"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
sha256prefix = "sha256:"
|
|
||||||
fileSubdir = "files"
|
|
||||||
tmpSubdir = "tmps"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Storage struct {
|
|
||||||
basepath string
|
|
||||||
logg *logger.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewStorage(basepath string) *Storage {
|
|
||||||
res := &Storage{
|
|
||||||
basepath: basepath,
|
|
||||||
}
|
|
||||||
res.logg = logger.NewLoggerWithSubject("storage")
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeCollecionpath(collection string) string {
|
|
||||||
return filepath.Join(store.basepath, fileSubdir, collection)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeFilepath(collection, filename string) string {
|
|
||||||
return filepath.Join(store.basepath, fileSubdir, collection, filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeTmppath(tmpName string) string {
|
|
||||||
return filepath.Join(store.basepath, tmpSubdir, tmpName)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeTmpsubdir() string {
|
|
||||||
return filepath.Join(store.basepath, tmpSubdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeFilesubdir(collection, filename string) string {
|
|
||||||
return filepath.Join(store.basepath, fileSubdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
|
||||||
var err error
|
|
||||||
var res io.ReadCloser
|
|
||||||
|
|
||||||
filename = store.makeFilepath(collection, filename)
|
|
||||||
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
res = file
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
|
|
||||||
var err error
|
|
||||||
var size int64
|
|
||||||
var csum string
|
|
||||||
|
|
||||||
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
|
|
||||||
tmpPath := store.makeTmppath(tmpName)
|
|
||||||
|
|
||||||
tmpdirpath := store.makeTmpsubdir()
|
|
||||||
err = os.MkdirAll(tmpdirpath, 0750)
|
|
||||||
if err != nil {
|
|
||||||
return tmpName, size, csum, err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
|
|
||||||
if err != nil {
|
|
||||||
return tmpName, size, csum, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
hasher := NewHasher()
|
|
||||||
mWriter := io.MultiWriter(file, hasher.Writer())
|
|
||||||
size, err = io.Copy(mWriter, source)
|
|
||||||
if err != nil {
|
|
||||||
return tmpName, size, csum, err
|
|
||||||
}
|
|
||||||
csum = hasher.Hex()
|
|
||||||
|
|
||||||
return tmpName, size, csum, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
dirname := store.makeCollecionpath(collection)
|
|
||||||
_, err = os.Stat(dirname)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
err = os.MkdirAll(dirname, 0750)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
filename = store.makeFilepath(collection, filename)
|
|
||||||
_, err = os.Stat(dirname)
|
|
||||||
if err == nil {
|
|
||||||
os.Remove(filename) // TODO: safe removing
|
|
||||||
}
|
|
||||||
tmpName = store.makeTmppath(tmpName)
|
|
||||||
err = os.Link(tmpName, filename)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = os.Remove(tmpName)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) DeleteFile(collection, filename string) error {
|
|
||||||
var err error
|
|
||||||
filename = store.makeFilepath(collection, filename)
|
|
||||||
err = os.Remove(filename)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// TODO: clean removing
|
|
||||||
dirname := store.makeCollecionpath(collection)
|
|
||||||
os.RemoveAll(dirname)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
upsubdir = "uploads"
|
|
||||||
blobsubdir = "blobs"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (store *Storage) makeUppath(upname string) string {
|
|
||||||
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeUpsubdir() string {
|
|
||||||
return filepath.Join(store.basepath, upsubdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeBlobpath(upname string) string {
|
|
||||||
return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) makeBlobsubdir() string {
|
|
||||||
return filepath.Join(store.basepath, blobsubdir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
|
|
||||||
var err error
|
|
||||||
var recsize int64
|
|
||||||
|
|
||||||
uploadDir := store.makeUpsubdir()
|
|
||||||
_, err = os.Stat(uploadDir)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
err = os.MkdirAll(uploadDir, 0750)
|
|
||||||
if err != nil {
|
|
||||||
return recsize, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return recsize, err
|
|
||||||
}
|
|
||||||
upPath := store.makeUppath(upID)
|
|
||||||
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
|
|
||||||
if err != nil {
|
|
||||||
return recsize, err
|
|
||||||
}
|
|
||||||
defer upFile.Close()
|
|
||||||
recsize, err = io.Copy(upFile, source)
|
|
||||||
if err != nil {
|
|
||||||
return recsize, err
|
|
||||||
}
|
|
||||||
return recsize, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) LinkUpload(reference, digest string) error {
|
|
||||||
var err error
|
|
||||||
upPath := store.makeUppath(reference)
|
|
||||||
|
|
||||||
blobdir := store.makeBlobsubdir()
|
|
||||||
_, err = os.Stat(blobdir)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
err = os.MkdirAll(blobdir, 0750)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
blobPath := store.makeBlobpath(digest)
|
|
||||||
_, err = os.Stat(blobPath)
|
|
||||||
if err == nil {
|
|
||||||
err = os.Remove(blobPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = os.Link(upPath, blobPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = os.Remove(upPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) RemoveUpload(digest string) error {
|
|
||||||
var err error
|
|
||||||
upPath := store.makeUppath(digest)
|
|
||||||
err = os.Remove(upPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
|
|
||||||
var err error
|
|
||||||
var size int64
|
|
||||||
upPath := st.makeUppath(reference)
|
|
||||||
fileStat, err := os.Stat(upPath)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return false, 0, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, 0, err
|
|
||||||
}
|
|
||||||
size = fileStat.Size()
|
|
||||||
return true, size, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
|
|
||||||
var err error
|
|
||||||
var size int64
|
|
||||||
blobDir := store.makeBlobsubdir()
|
|
||||||
_, err = os.Stat(blobDir)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
err = os.MkdirAll(blobDir, 0750)
|
|
||||||
if err != nil {
|
|
||||||
return size, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return size, err
|
|
||||||
}
|
|
||||||
blobPath := store.makeBlobpath(digstr)
|
|
||||||
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
|
|
||||||
if err != nil {
|
|
||||||
return size, err
|
|
||||||
}
|
|
||||||
defer blobFile.Close()
|
|
||||||
size, err = io.Copy(blobFile, source)
|
|
||||||
if err != nil {
|
|
||||||
return size, err
|
|
||||||
}
|
|
||||||
return size, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
|
|
||||||
var err error
|
|
||||||
var size int64
|
|
||||||
|
|
||||||
blobPath := st.makeBlobpath(digest)
|
|
||||||
fileStat, err := os.Stat(blobPath)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return false, 0, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return false, 0, err
|
|
||||||
}
|
|
||||||
size = fileStat.Size()
|
|
||||||
return true, size, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
|
|
||||||
var err error
|
|
||||||
var size int64
|
|
||||||
blobPath := store.makeBlobpath(digest)
|
|
||||||
nop := io.NopCloser(bytes.NewReader(nil))
|
|
||||||
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
|
|
||||||
if err != nil {
|
|
||||||
return size, nop, err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
file.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
filestat, err := file.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return size, nop, err
|
|
||||||
}
|
|
||||||
size = filestat.Size()
|
|
||||||
return size, file, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (store *Storage) DeleteBlob(digest string) error {
|
|
||||||
var err error
|
|
||||||
blobPath := store.makeBlobpath(digest)
|
|
||||||
err = os.Remove(blobPath)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
|
||||||
|
*
|
||||||
|
* This work is published and licensed under a Creative Commons
|
||||||
|
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
|
||||||
|
*
|
||||||
|
* Distribution of this work is permitted, but commercial use and
|
||||||
|
* modifications are strictly prohibited.
|
||||||
|
*/
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"mstore/app/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Storage struct {
|
||||||
|
basepath string
|
||||||
|
logg *logger.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStorage(basepath string) *Storage {
|
||||||
|
res := &Storage{
|
||||||
|
basepath: basepath,
|
||||||
|
}
|
||||||
|
res.logg = logger.NewLoggerWithSubject("storage")
|
||||||
|
return res
|
||||||
|
}
|
||||||
@@ -27,7 +27,7 @@ spec:
|
|||||||
- name: config-volume
|
- name: config-volume
|
||||||
mountPath: /app/etc/mstore
|
mountPath: /app/etc/mstore
|
||||||
- name: db-volume
|
- name: db-volume
|
||||||
mountPath: /var/data
|
mountPath: /var/lib
|
||||||
volumes:
|
volumes:
|
||||||
- name: config-volume
|
- name: config-volume
|
||||||
configMap:
|
configMap:
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ require (
|
|||||||
github.com/google/go-containerregistry v0.21.2
|
github.com/google/go-containerregistry v0.21.2
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
github.com/jmoiron/sqlx v1.4.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.34
|
github.com/mattn/go-sqlite3 v1.14.37
|
||||||
github.com/opencontainers/go-digest v1.0.0
|
github.com/opencontainers/go-digest v1.0.0
|
||||||
github.com/opencontainers/image-spec v1.1.1
|
github.com/opencontainers/image-spec v1.1.1
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
|
|||||||
@@ -147,8 +147,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
|
|||||||
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
|
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
|
||||||
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg=
|
||||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
|
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
|
||||||
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
|
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
|
||||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||||
|
|||||||
+77
-52
@@ -1,7 +1,7 @@
|
|||||||
#ifndef USE_LIBSQLITE3
|
#ifndef USE_LIBSQLITE3
|
||||||
/******************************************************************************
|
/******************************************************************************
|
||||||
** This file is an amalgamation of many separate C source files from SQLite
|
** This file is an amalgamation of many separate C source files from SQLite
|
||||||
** version 3.51.2. By combining all the individual C code files into this
|
** version 3.51.3. By combining all the individual C code files into this
|
||||||
** single large file, the entire code can be compiled as a single translation
|
** single large file, the entire code can be compiled as a single translation
|
||||||
** unit. This allows many compilers to do optimizations that would not be
|
** unit. This allows many compilers to do optimizations that would not be
|
||||||
** possible if the files were compiled separately. Performance improvements
|
** possible if the files were compiled separately. Performance improvements
|
||||||
@@ -19,7 +19,7 @@
|
|||||||
** separate file. This file contains only code for the core SQLite library.
|
** separate file. This file contains only code for the core SQLite library.
|
||||||
**
|
**
|
||||||
** The content in this amalgamation comes from Fossil check-in
|
** The content in this amalgamation comes from Fossil check-in
|
||||||
** b270f8339eb13b504d0b2ba154ebca966b7d with changes in files:
|
** 737ae4a34738ffa0c3ff7f9bb18df914dd1c with changes in files:
|
||||||
**
|
**
|
||||||
**
|
**
|
||||||
*/
|
*/
|
||||||
@@ -468,12 +468,12 @@ extern "C" {
|
|||||||
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
|
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
|
||||||
** [sqlite_version()] and [sqlite_source_id()].
|
** [sqlite_version()] and [sqlite_source_id()].
|
||||||
*/
|
*/
|
||||||
#define SQLITE_VERSION "3.51.2"
|
#define SQLITE_VERSION "3.51.3"
|
||||||
#define SQLITE_VERSION_NUMBER 3051002
|
#define SQLITE_VERSION_NUMBER 3051003
|
||||||
#define SQLITE_SOURCE_ID "2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075"
|
#define SQLITE_SOURCE_ID "2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618"
|
||||||
#define SQLITE_SCM_BRANCH "branch-3.51"
|
#define SQLITE_SCM_BRANCH "branch-3.51"
|
||||||
#define SQLITE_SCM_TAGS "release version-3.51.2"
|
#define SQLITE_SCM_TAGS "release version-3.51.3"
|
||||||
#define SQLITE_SCM_DATETIME "2026-01-09T17:27:48.405Z"
|
#define SQLITE_SCM_DATETIME "2026-03-13T10:38:09.694Z"
|
||||||
|
|
||||||
/*
|
/*
|
||||||
** CAPI3REF: Run-Time Library Version Numbers
|
** CAPI3REF: Run-Time Library Version Numbers
|
||||||
@@ -14335,6 +14335,27 @@ struct fts5_api {
|
|||||||
#endif
|
#endif
|
||||||
#define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */
|
#define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */
|
||||||
|
|
||||||
|
/*
|
||||||
|
** Maximum size of any single memory allocation.
|
||||||
|
**
|
||||||
|
** This is not a limit on the total amount of memory used. This is
|
||||||
|
** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc().
|
||||||
|
**
|
||||||
|
** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391
|
||||||
|
** This provides a 256-byte safety margin for defense against 32-bit
|
||||||
|
** signed integer overflow bugs when computing memory allocation sizes.
|
||||||
|
** Paranoid applications might want to reduce the maximum allocation size
|
||||||
|
** further for an even larger safety margin. 0x3fffffff or 0x0fffffff
|
||||||
|
** or even smaller would be reasonable upper bounds on the size of a memory
|
||||||
|
** allocations for most applications.
|
||||||
|
*/
|
||||||
|
#ifndef SQLITE_MAX_ALLOCATION_SIZE
|
||||||
|
# define SQLITE_MAX_ALLOCATION_SIZE 2147483391
|
||||||
|
#endif
|
||||||
|
#if SQLITE_MAX_ALLOCATION_SIZE>2147483391
|
||||||
|
# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391
|
||||||
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
** This is the maximum number of
|
** This is the maximum number of
|
||||||
**
|
**
|
||||||
@@ -21665,6 +21686,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList
|
|||||||
Expr*,ExprList*,u32,Expr*);
|
Expr*,ExprList*,u32,Expr*);
|
||||||
SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*);
|
SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*);
|
||||||
SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*);
|
SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*);
|
||||||
|
SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect);
|
||||||
SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*);
|
SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*);
|
||||||
SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
|
SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
|
||||||
SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
|
SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
|
||||||
@@ -31311,27 +31333,6 @@ static void mallocWithAlarm(int n, void **pp){
|
|||||||
*pp = p;
|
*pp = p;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
** Maximum size of any single memory allocation.
|
|
||||||
**
|
|
||||||
** This is not a limit on the total amount of memory used. This is
|
|
||||||
** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc().
|
|
||||||
**
|
|
||||||
** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391
|
|
||||||
** This provides a 256-byte safety margin for defense against 32-bit
|
|
||||||
** signed integer overflow bugs when computing memory allocation sizes.
|
|
||||||
** Paranoid applications might want to reduce the maximum allocation size
|
|
||||||
** further for an even larger safety margin. 0x3fffffff or 0x0fffffff
|
|
||||||
** or even smaller would be reasonable upper bounds on the size of a memory
|
|
||||||
** allocations for most applications.
|
|
||||||
*/
|
|
||||||
#ifndef SQLITE_MAX_ALLOCATION_SIZE
|
|
||||||
# define SQLITE_MAX_ALLOCATION_SIZE 2147483391
|
|
||||||
#endif
|
|
||||||
#if SQLITE_MAX_ALLOCATION_SIZE>2147483391
|
|
||||||
# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
** Allocate memory. This routine is like sqlite3_malloc() except that it
|
** Allocate memory. This routine is like sqlite3_malloc() except that it
|
||||||
** assumes the memory subsystem has already been initialized.
|
** assumes the memory subsystem has already been initialized.
|
||||||
@@ -31555,8 +31556,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){
|
|||||||
sqlite3_free(pOld); /* IMP: R-26507-47431 */
|
sqlite3_free(pOld); /* IMP: R-26507-47431 */
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if( nBytes>=0x7fffff00 ){
|
if( nBytes>SQLITE_MAX_ALLOCATION_SIZE ){
|
||||||
/* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
nOld = sqlite3MallocSize(pOld);
|
nOld = sqlite3MallocSize(pOld);
|
||||||
@@ -69011,6 +69011,17 @@ static int walCheckpoint(
|
|||||||
&& (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
|
&& (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
|
||||||
){
|
){
|
||||||
u32 nBackfill = pInfo->nBackfill;
|
u32 nBackfill = pInfo->nBackfill;
|
||||||
|
WalIndexHdr *pLive = (WalIndexHdr*)walIndexHdr(pWal);
|
||||||
|
|
||||||
|
/* Now that read-lock slot 0 is locked, check that the wal has not been
|
||||||
|
** wrapped since the header was read for this checkpoint. If it was, then
|
||||||
|
** there was no work to do anyway. In this case the
|
||||||
|
** (pInfo->nBackfill<pWal->hdr.mxFrame) test above only passed because
|
||||||
|
** pInfo->nBackfill had already been set to 0 by the writer that wrapped
|
||||||
|
** the wal file. It would also be dangerous to proceed, as there may be
|
||||||
|
** fewer than pWal->hdr.mxFrame valid frames in the wal file. */
|
||||||
|
int bChg = memcmp(pLive->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt));
|
||||||
|
if( 0==bChg ){
|
||||||
pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
|
pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
|
||||||
|
|
||||||
/* Sync the WAL to disk */
|
/* Sync the WAL to disk */
|
||||||
@@ -69032,13 +69043,15 @@ static int walCheckpoint(
|
|||||||
** must be corruption somewhere. */
|
** must be corruption somewhere. */
|
||||||
rc = SQLITE_CORRUPT_BKPT;
|
rc = SQLITE_CORRUPT_BKPT;
|
||||||
}else{
|
}else{
|
||||||
sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq);
|
sqlite3OsFileControlHint(
|
||||||
|
pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Iterate through the contents of the WAL, copying data to the db file */
|
/* Iterate through the contents of the WAL, copying data to the
|
||||||
|
** db file */
|
||||||
while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
|
while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
|
||||||
i64 iOffset;
|
i64 iOffset;
|
||||||
assert( walFramePgno(pWal, iFrame)==iDbpage );
|
assert( walFramePgno(pWal, iFrame)==iDbpage );
|
||||||
@@ -69075,6 +69088,7 @@ static int walCheckpoint(
|
|||||||
AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
|
AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Release the reader lock held while backfilling */
|
/* Release the reader lock held while backfilling */
|
||||||
walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
|
walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
|
||||||
@@ -71122,6 +71136,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint(
|
|||||||
|
|
||||||
/* Copy data from the log to the database file. */
|
/* Copy data from the log to the database file. */
|
||||||
if( rc==SQLITE_OK ){
|
if( rc==SQLITE_OK ){
|
||||||
|
sqlite3FaultSim(660);
|
||||||
if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
|
if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
|
||||||
rc = SQLITE_CORRUPT_BKPT;
|
rc = SQLITE_CORRUPT_BKPT;
|
||||||
}else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){
|
}else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){
|
||||||
@@ -77557,7 +77572,7 @@ static int accessPayload(
|
|||||||
|
|
||||||
getCellInfo(pCur);
|
getCellInfo(pCur);
|
||||||
aPayload = pCur->info.pPayload;
|
aPayload = pCur->info.pPayload;
|
||||||
assert( offset+amt <= pCur->info.nPayload );
|
assert( (u64)offset+(u64)amt <= (u64)pCur->info.nPayload );
|
||||||
|
|
||||||
assert( aPayload > pPage->aData );
|
assert( aPayload > pPage->aData );
|
||||||
if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){
|
if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){
|
||||||
@@ -86031,7 +86046,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(
|
|||||||
){
|
){
|
||||||
int rc;
|
int rc;
|
||||||
pMem->flags = MEM_Null;
|
pMem->flags = MEM_Null;
|
||||||
if( sqlite3BtreeMaxRecordSize(pCur)<offset+amt ){
|
testcase( amt==SQLITE_MAX_ALLOCATION_SIZE-1 );
|
||||||
|
testcase( amt==SQLITE_MAX_ALLOCATION_SIZE );
|
||||||
|
if( amt>=SQLITE_MAX_ALLOCATION_SIZE ){
|
||||||
|
return SQLITE_NOMEM_BKPT;
|
||||||
|
}
|
||||||
|
if( (u64)amt + (u64)offset > (u64)sqlite3BtreeMaxRecordSize(pCur) ){
|
||||||
return SQLITE_CORRUPT_BKPT;
|
return SQLITE_CORRUPT_BKPT;
|
||||||
}
|
}
|
||||||
if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){
|
if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){
|
||||||
@@ -93444,7 +93464,7 @@ static int valueFromValueList(
|
|||||||
Mem sMem; /* Raw content of current row */
|
Mem sMem; /* Raw content of current row */
|
||||||
memset(&sMem, 0, sizeof(sMem));
|
memset(&sMem, 0, sizeof(sMem));
|
||||||
sz = sqlite3BtreePayloadSize(pRhs->pCsr);
|
sz = sqlite3BtreePayloadSize(pRhs->pCsr);
|
||||||
rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem);
|
rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,sz,&sMem);
|
||||||
if( rc==SQLITE_OK ){
|
if( rc==SQLITE_OK ){
|
||||||
u8 *zBuf = (u8*)sMem.z;
|
u8 *zBuf = (u8*)sMem.z;
|
||||||
u32 iSerial;
|
u32 iSerial;
|
||||||
@@ -111186,6 +111206,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
|
|||||||
return WRC_Abort;
|
return WRC_Abort;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* If the SELECT statement contains ON clauses that were moved into
|
||||||
|
** the WHERE clause, go through and verify that none of the terms
|
||||||
|
** in the ON clauses reference tables to the right of the ON clause. */
|
||||||
|
if( (p->selFlags & SF_OnToWhere) ){
|
||||||
|
sqlite3SelectCheckOnClauses(pParse, p);
|
||||||
|
if( pParse->nErr ) return WRC_Abort;
|
||||||
|
}
|
||||||
|
|
||||||
/* Advance to the next term of the compound
|
/* Advance to the next term of the compound
|
||||||
*/
|
*/
|
||||||
p = p->pPrior;
|
p = p->pPrior;
|
||||||
@@ -154357,7 +154385,7 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){
|
|||||||
** Check all ON clauses in pSelect to verify that they do not reference
|
** Check all ON clauses in pSelect to verify that they do not reference
|
||||||
** columns to the right.
|
** columns to the right.
|
||||||
*/
|
*/
|
||||||
static void selectCheckOnClauses(Parse *pParse, Select *pSelect){
|
SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){
|
||||||
Walker w;
|
Walker w;
|
||||||
CheckOnCtx sCtx;
|
CheckOnCtx sCtx;
|
||||||
assert( pSelect->selFlags & SF_OnToWhere );
|
assert( pSelect->selFlags & SF_OnToWhere );
|
||||||
@@ -154500,18 +154528,6 @@ SQLITE_PRIVATE int sqlite3Select(
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* If the SELECT statement contains ON clauses that were moved into
|
|
||||||
** the WHERE clause, go through and verify that none of the terms
|
|
||||||
** in the ON clauses reference tables to the right of the ON clause.
|
|
||||||
** Do this now, after name resolution, but before query flattening
|
|
||||||
*/
|
|
||||||
if( p->selFlags & SF_OnToWhere ){
|
|
||||||
selectCheckOnClauses(pParse, p);
|
|
||||||
if( pParse->nErr ){
|
|
||||||
goto select_end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* If the SF_UFSrcCheck flag is set, then this function is being called
|
/* If the SF_UFSrcCheck flag is set, then this function is being called
|
||||||
** as part of populating the temp table for an UPDATE...FROM statement.
|
** as part of populating the temp table for an UPDATE...FROM statement.
|
||||||
** In this case, it is an error if the target object (pSrc->a[0]) name
|
** In this case, it is an error if the target object (pSrc->a[0]) name
|
||||||
@@ -164678,6 +164694,15 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop(
|
|||||||
sqlite3ExprDup(pParse->db, pTerm->pExpr, 0));
|
sqlite3ExprDup(pParse->db, pTerm->pExpr, 0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if( pLevel->iIdxCur ){
|
||||||
|
/* pSubWhere may contain expressions that read from an index on the
|
||||||
|
** table on the RHS of the right join. All such expressions first test
|
||||||
|
** if the index is pointing at a NULL row, and if so, read from the
|
||||||
|
** table cursor instead. So ensure that the index cursor really is
|
||||||
|
** pointing at a NULL row here, so that no values are read from it during
|
||||||
|
** the scan of the RHS of the RIGHT join below. */
|
||||||
|
sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
|
||||||
|
}
|
||||||
pFrom = &uSrc.sSrc;
|
pFrom = &uSrc.sSrc;
|
||||||
pFrom->nSrc = 1;
|
pFrom->nSrc = 1;
|
||||||
pFrom->nAlloc = 1;
|
pFrom->nAlloc = 1;
|
||||||
@@ -224164,7 +224189,7 @@ static int rbuDeltaApply(
|
|||||||
/* ERROR: copy exceeds output file size */
|
/* ERROR: copy exceeds output file size */
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if( (int)(ofst+cnt) > lenSrc ){
|
if( (u64)ofst+(u64)cnt > (u64)lenSrc ){
|
||||||
/* ERROR: copy extends past end of input */
|
/* ERROR: copy extends past end of input */
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -252451,7 +252476,7 @@ static void fts5DoSecureDelete(
|
|||||||
int iSegid = pSeg->pSeg->iSegid;
|
int iSegid = pSeg->pSeg->iSegid;
|
||||||
u8 *aPg = pSeg->pLeaf->p;
|
u8 *aPg = pSeg->pLeaf->p;
|
||||||
int nPg = pSeg->pLeaf->nn;
|
int nPg = pSeg->pLeaf->nn;
|
||||||
int iPgIdx = pSeg->pLeaf->szLeaf;
|
int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */
|
||||||
|
|
||||||
u64 iDelta = 0;
|
u64 iDelta = 0;
|
||||||
int iNextOff = 0;
|
int iNextOff = 0;
|
||||||
@@ -252530,7 +252555,7 @@ static void fts5DoSecureDelete(
|
|||||||
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
|
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
|
||||||
}
|
}
|
||||||
assert_nc( iSOP==pSeg->iLeafOffset );
|
assert_nc( iSOP==pSeg->iLeafOffset );
|
||||||
iNextOff = pSeg->iLeafOffset + pSeg->nPos;
|
iNextOff = iSOP + pSeg->nPos;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -260349,7 +260374,7 @@ static void fts5SourceIdFunc(
|
|||||||
){
|
){
|
||||||
assert( nArg==0 );
|
assert( nArg==0 );
|
||||||
UNUSED_PARAM2(nArg, apUnused);
|
UNUSED_PARAM2(nArg, apUnused);
|
||||||
sqlite3_result_text(pCtx, "fts5: 2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075", -1, SQLITE_TRANSIENT);
|
sqlite3_result_text(pCtx, "fts5: 2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618", -1, SQLITE_TRANSIENT);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|||||||
+5
-5
@@ -147,12 +147,12 @@ extern "C" {
|
|||||||
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
|
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
|
||||||
** [sqlite_version()] and [sqlite_source_id()].
|
** [sqlite_version()] and [sqlite_source_id()].
|
||||||
*/
|
*/
|
||||||
#define SQLITE_VERSION "3.51.2"
|
#define SQLITE_VERSION "3.51.3"
|
||||||
#define SQLITE_VERSION_NUMBER 3051002
|
#define SQLITE_VERSION_NUMBER 3051003
|
||||||
#define SQLITE_SOURCE_ID "2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075"
|
#define SQLITE_SOURCE_ID "2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618"
|
||||||
#define SQLITE_SCM_BRANCH "branch-3.51"
|
#define SQLITE_SCM_BRANCH "branch-3.51"
|
||||||
#define SQLITE_SCM_TAGS "release version-3.51.2"
|
#define SQLITE_SCM_TAGS "release version-3.51.3"
|
||||||
#define SQLITE_SCM_DATETIME "2026-01-09T17:27:48.405Z"
|
#define SQLITE_SCM_DATETIME "2026-03-13T10:38:09.694Z"
|
||||||
|
|
||||||
/*
|
/*
|
||||||
** CAPI3REF: Run-Time Library Version Numbers
|
** CAPI3REF: Run-Time Library Version Numbers
|
||||||
|
|||||||
+17
-10
@@ -1783,15 +1783,18 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
|
|||||||
|
|
||||||
// Close the connection.
|
// Close the connection.
|
||||||
func (c *SQLiteConn) Close() error {
|
func (c *SQLiteConn) Close() error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.db == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
runtime.SetFinalizer(c, nil)
|
||||||
rv := C.sqlite3_close_v2(c.db)
|
rv := C.sqlite3_close_v2(c.db)
|
||||||
if rv != C.SQLITE_OK {
|
if rv != C.SQLITE_OK {
|
||||||
return c.lastError()
|
return lastError(c.db)
|
||||||
}
|
}
|
||||||
deleteHandles(c)
|
deleteHandles(c)
|
||||||
c.mu.Lock()
|
|
||||||
c.db = nil
|
c.db = nil
|
||||||
c.mu.Unlock()
|
|
||||||
runtime.SetFinalizer(c, nil)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1932,16 +1935,18 @@ func (s *SQLiteStmt) Close() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
s.closed = true
|
s.closed = true
|
||||||
if !s.c.dbConnOpen() {
|
runtime.SetFinalizer(s, nil)
|
||||||
|
conn := s.c
|
||||||
|
stmt := s.s
|
||||||
|
s.s = nil
|
||||||
|
s.c = nil
|
||||||
|
if !conn.dbConnOpen() {
|
||||||
return errors.New("sqlite statement with already closed database connection")
|
return errors.New("sqlite statement with already closed database connection")
|
||||||
}
|
}
|
||||||
rv := C.sqlite3_finalize(s.s)
|
rv := C.sqlite3_finalize(stmt)
|
||||||
s.s = nil
|
|
||||||
if rv != C.SQLITE_OK {
|
if rv != C.SQLITE_OK {
|
||||||
return s.c.lastError()
|
return conn.lastError()
|
||||||
}
|
}
|
||||||
s.c = nil
|
|
||||||
runtime.SetFinalizer(s, nil)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1958,6 +1963,8 @@ func (s *SQLiteStmt) bind(args []driver.NamedValue) error {
|
|||||||
return s.c.lastError()
|
return s.c.lastError()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
C.sqlite3_clear_bindings(s.s)
|
||||||
|
|
||||||
bindIndices := make([][3]int, len(args))
|
bindIndices := make([][3]int, len(args))
|
||||||
prefixes := []string{":", "@", "$"}
|
prefixes := []string{":", "@", "$"}
|
||||||
for i, v := range args {
|
for i, v := range args {
|
||||||
|
|||||||
+2
-14
@@ -17,7 +17,6 @@ import "C"
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"reflect"
|
|
||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -43,14 +42,8 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
|
|||||||
return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz)
|
return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz)
|
||||||
}
|
}
|
||||||
|
|
||||||
cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
|
|
||||||
Data: uintptr(unsafe.Pointer(ptr)),
|
|
||||||
Len: int(sz),
|
|
||||||
Cap: int(sz),
|
|
||||||
}))
|
|
||||||
|
|
||||||
res := make([]byte, int(sz))
|
res := make([]byte, int(sz))
|
||||||
copy(res, cBuf)
|
copy(res, unsafe.Slice((*byte)(unsafe.Pointer(ptr)), int(sz)))
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,12 +60,7 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
|
|||||||
defer C.free(unsafe.Pointer(zSchema))
|
defer C.free(unsafe.Pointer(zSchema))
|
||||||
|
|
||||||
tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b))))
|
tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b))))
|
||||||
cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
|
copy(unsafe.Slice((*byte)(unsafe.Pointer(tmpBuf)), len(b)), b)
|
||||||
Data: uintptr(unsafe.Pointer(tmpBuf)),
|
|
||||||
Len: len(b),
|
|
||||||
Cap: len(b),
|
|
||||||
}))
|
|
||||||
copy(cBuf, b)
|
|
||||||
|
|
||||||
rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)),
|
rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)),
|
||||||
C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE)
|
C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE)
|
||||||
|
|||||||
Vendored
+1
-1
@@ -235,7 +235,7 @@ github.com/liggitt/tabwriter
|
|||||||
github.com/mailru/easyjson/buffer
|
github.com/mailru/easyjson/buffer
|
||||||
github.com/mailru/easyjson/jlexer
|
github.com/mailru/easyjson/jlexer
|
||||||
github.com/mailru/easyjson/jwriter
|
github.com/mailru/easyjson/jwriter
|
||||||
# github.com/mattn/go-sqlite3 v1.14.34
|
# github.com/mattn/go-sqlite3 v1.14.37
|
||||||
## explicit; go 1.19
|
## explicit; go 1.19
|
||||||
github.com/mattn/go-sqlite3
|
github.com/mattn/go-sqlite3
|
||||||
# github.com/mitchellh/go-homedir v1.1.0
|
# github.com/mitchellh/go-homedir v1.1.0
|
||||||
|
|||||||
Reference in New Issue
Block a user