ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

app/imageoper, app/maindb: one image - one arch; app/storage: blob to name subdir

Browse Source
This commit is contained in:
Олег Бородин
2026-03-17 09:40:53 +02:00
parent 13b1905e05
commit 791e04cdf1
28 changed files with 680 additions and 569 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.am
+7 -2
View File
@@ -40,7 +40,6 @@ EXTRA_mstorectl_SOURCES = \
cmd/mstorectl/imagecmd/imageman.go \
cmd/mstorectl/imagecmd/imagetags.go \
cmd/mstorectl/imagecmd/printresp.go \
cmd/mstorectl/imagecmd/gcrimageconf.go \
cmd/mstorectl/imagecmd/pullimage.go \
cmd/mstorectl/imagecmd/pushimage.go
@@ -128,7 +127,13 @@ EXTRA_mstored_SOURCES += \
app/router/router.go \
app/server/server.go \
app/service/service.go \
app/storage/storage.go \
\
app/storage/blob.go \
app/storage/file.go \
app/storage/hasher.go \
app/storage/helm.go \
app/storage/store.go \
\
pkg/accntcli/client.go \
pkg/accntcli/createacc.go \
pkg/accntcli/creategrant.go \
Makefile.in
+2 -2
View File
@@ -386,7 +386,6 @@ EXTRA_mstorectl_SOURCES = \
cmd/mstorectl/imagecmd/imageman.go \
cmd/mstorectl/imagecmd/imagetags.go \
cmd/mstorectl/imagecmd/printresp.go \
cmd/mstorectl/imagecmd/gcrimageconf.go \
cmd/mstorectl/imagecmd/pullimage.go \
cmd/mstorectl/imagecmd/pushimage.go
@@ -424,7 +423,8 @@ EXTRA_mstored_SOURCES = cmd/mstored/starter/starter.go \
app/router/corsmw.go app/router/loggingmw.go \
app/router/pathc.go app/router/recovermw.go \
app/router/router.go app/server/server.go \
app/service/service.go app/storage/storage.go \
app/service/service.go app/storage/blob.go app/storage/file.go \
app/storage/hasher.go app/storage/helm.go app/storage/store.go \
pkg/accntcli/client.go pkg/accntcli/createacc.go \
pkg/accntcli/creategrant.go pkg/accntcli/delacc.go \
pkg/accntcli/delgrant.go pkg/accntcli/getacc.go \
app/handler/manifest.go
+3 -1
View File
@@ -21,7 +21,7 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
name, _ := rctx.GetSubpath("name")
reference, _ := rctx.GetSubpath("reference")
//hand.DumpHeaders("ManigestExists:\n", rctx)
hand.DumpHeaders("ManigestExists:\n", rctx)
params := &imageoper.ManifestExistsParams{
Name: name,
@@ -48,6 +48,8 @@ func (hand *Handler) ManifestExists(rctx *router.Context) {
rctx.SetHeader("Content-Type", res.ContentType)
rctx.SetHeader("Docker-Content-Digest", res.DockerContentDigest)
}
hand.logg.Debugf(res.ContentType)
hand.logg.Debugf(res.DockerContentDigest)
rctx.SetStatus(code)
}
app/handler/version.go
+9 -7
View File
@@ -10,7 +10,7 @@
package handler
import (
"net/http"
//"net/http"
"mstore/app/imageoper"
"mstore/app/router"
@@ -21,12 +21,14 @@ func (hand *Handler) GetVersion(rctx *router.Context) {
params := &imageoper.GetVersionParams{}
//hand.DumpHeaders("GetVersion", rctx)
authorization := rctx.GetHeader("Authorization")
if authorization == "" {
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
rctx.SetStatus(http.StatusUnauthorized)
return
}
/*
authorization := rctx.GetHeader("Authorization")
if authorization == "" {
rctx.SetHeader("WWW-Authenticate", `Basic realm="mstore"`)
rctx.SetStatus(http.StatusUnauthorized)
return
}
*/
ctx := rctx.GetContext()
_, code, err := hand.imop.GetVersion(ctx, params)
if err != nil {
app/imageoper/blobexist.go
+1 -1
View File
@@ -52,7 +52,7 @@ func (oper *Operator) BlobExists(ctx context.Context, operatorID string, params
return res, http.StatusNotFound, err
}
// Check blob file
blobExists, _, err := oper.store.BlobExists(params.Digest)
blobExists, _, err := oper.store.BlobExists(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
app/imageoper/delblob.go
+2 -2
View File
@@ -59,13 +59,13 @@ func (oper *Operator) DeleteBlob(ctx context.Context, operatorID string, params
return res, http.StatusInternalServerError, err
}
// Removing the blob binary if usage == 0
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if blobUsage == 0 {
oper.logg.Warningf("Deleting useless blob binary %s", params.Digest)
oper.store.DeleteBlob(params.Digest)
oper.store.DeleteBlob(params.Name, params.Digest)
}
return res, http.StatusOK, err
}
app/imageoper/delman.go
+25 -11
View File
@@ -64,21 +64,35 @@ func (oper *Operator) DeleteManifest(ctx context.Context, params *DeleteManifest
return res, http.StatusInternalServerError, err
}
} else {
/*
// Check manifest by name and reference
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
reference = params.Reference
for _, mandescr := range mandescrs {
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
*/
// Check manifest by name and reference
exists, mandescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
exists, mandescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
reference = params.Reference
for _, mandescr := range mandescrs {
reference = mandescr.Reference
err = oper.deleteManifestObjects(ctx, params.Name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
err = oper.deleteManifestObjects(ctx, mandescr.Name, mandescr.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
}
// Get blobs associated with the name
@@ -99,19 +113,19 @@ func (oper *Operator) deleteManifestObjects(ctx context.Context, name, reference
return err
}
// Check blob file
exists, _, err := oper.store.BlobExists(blob.Digest)
exists, _, err := oper.store.BlobExists(name, blob.Digest)
if err != nil {
return err
}
if exists {
// Check blob usage
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
if err != nil {
return err
}
// Delete if blob useless
if blobUsage == 0 {
err = oper.store.DeleteBlob(blob.Digest)
err = oper.store.DeleteBlob(name, blob.Digest)
if err != nil {
return err
}
app/imageoper/getblob.go
+2 -2
View File
@@ -45,7 +45,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
oper.iLock.WaitAndLock(resName)
defer oper.iLock.Done(resName)
blobExists, blobSize, err := oper.store.BlobExists(params.Digest)
blobExists, blobSize, err := oper.store.BlobExists(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -57,7 +57,7 @@ func (oper *Operator) GetBlob(ctx context.Context, operatorID string, params *Ge
return res, http.StatusNotFound, err
}
_, readCloser, err := oper.store.BlobReader(params.Digest)
_, readCloser, err := oper.store.BlobReader(params.Name, params.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
app/imageoper/getman.go
+36 -21
View File
@@ -66,34 +66,49 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
res.Payload = manDescr.Payload
} else {
// Create index of manifests. Or not.
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
/*
// Create index of manifests. Or not.
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr = manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Payload = string(indexdata)
}
*/
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr = manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
res.Payload = manDescr.Payload
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Payload = string(indexdata)
}
}
return res, http.StatusOK, err
}
app/imageoper/manexist.go
+32 -11
View File
@@ -62,24 +62,45 @@ func (oper *Operator) ManifestExists(ctx context.Context, params *ManifestExists
res.DockerContentDigest = man.Digest
res.Exists = exist
} else {
// Create index of manifests
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
/*
// Create index of manifests
exists, manDescrs, err := oper.mdb.GetManifestsByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
if len(manDescrs) == 1 {
manDescr := manDescrs[0]
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = manDescr.ContentType
} else {
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Exists = true
}
*/
exists, manDescr, err := oper.mdb.GetManifestByReference(ctx, params.Name, params.Reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
if !exists {
return res, http.StatusNotFound, err
}
_, indexdata, err := indexFromManigestDescrs(manDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
digobj := ocidigest.FromBytes(indexdata)
res.DockerContentDigest = digobj.String()
size := int64(len(indexdata))
res.DockerContentDigest = manDescr.Digest
size := int64(len(manDescr.Payload))
res.ContentLength = strconv.FormatInt(size, 10)
res.ContentType = oiiMediaType
res.Exists = true
res.ContentType = manDescr.ContentType
}
return res, http.StatusOK, err
}
app/imageoper/putman.go
+14 -16
View File
@@ -111,21 +111,26 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
name := params.Name
reference := params.Reference
arch := inMan.Subject.Platform.Architecture
os := inMan.Subject.Platform.OS
variant := inMan.Subject.Platform.Variant
/*
arch := inMan.Subject.Platform.Architecture
os := inMan.Subject.Platform.OS
variant := inMan.Subject.Platform.Variant
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
manexist, exMandescr, err := oper.mdb.GetManifestsByReferenceArchitecture(ctx, name, reference, arch, os, variant)
if err != nil {
return res, http.StatusInternalServerError, err
}
*/
manexist, exMandescr, err := oper.mdb.GetManifestByReference(ctx, name, reference)
if err != nil {
return res, http.StatusInternalServerError, err
}
inManDescr, inlayerdescrs, err := descrsFromManifest(name, reference, inMan, inManData)
inManDescr.Digest = digstr
// Always check layer files for availability
var blobError error
for _, blobDescr := range inlayerdescrs {
blobExists, _, err := oper.store.BlobExists(blobDescr.Digest)
blobExists, _, err := oper.store.BlobExists(blobDescr.Name, blobDescr.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -151,7 +156,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
return res, http.StatusCreated, err
}
*/
exManData := []byte(exMandescr.Payload)
exMan := &ocispec.Manifest{}
err := json.Unmarshal(exManData, exMan)
@@ -163,25 +167,22 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
if err != nil {
return res, http.StatusInternalServerError, err
}
// Starting manifest and blobs transaction
err = oper.mdb.UpdateManifestWithBlobs(ctx, &inManDescr, addedBlobDescrs, uselessBlobDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
}
//goto end
for _, blob := range uselessBlobDescrs {
exists, _, err := oper.store.BlobExists(blob.Digest)
exists, _, err := oper.store.BlobExists(blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Digest)
blobUsage, err := oper.mdb.GetBlobUsage(ctx, blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
if exists && blobUsage == 0 {
err = oper.store.DeleteBlob(blob.Digest)
err = oper.store.DeleteBlob(blob.Name, blob.Digest)
if err != nil {
return res, http.StatusInternalServerError, err
}
@@ -202,9 +203,6 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
}
}
}
//end:
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
return res, http.StatusCreated, err
}
app/imageoper/putupload.go
+1 -1
View File
@@ -80,7 +80,7 @@ func (oper *Operator) PutUpload(ctx context.Context, operatorID string, params *
res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest)
}
err = oper.store.LinkUpload(params.Reference, params.Digest)
err = oper.store.LinkUpload(params.Reference, params.Name, params.Digest)
if err != nil {
err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err)
return res, http.StatusInternalServerError, err
app/maindb/blob.go
+3 -3
View File
@@ -103,12 +103,12 @@ func (db *Database) GetBlobsByReferense(ctx context.Context, name, reference str
return blobs, err
}
func (db *Database) GetBlobUsage(ctx context.Context, digest string) (int64, error) {
func (db *Database) GetBlobUsage(ctx context.Context, name, digest string) (int64, error) {
var err error
var usage int64
count := make([]int64, 0)
request := `SELECT count(id) AS count FROM blobs WHERE digest = $1`
err = db.db.Select(&count, request, digest)
request := `SELECT count(id) AS count FROM blobs WHERE name = $1 AND digest = $2`
err = db.db.Select(&count, request, name, digest)
if err != nil {
return usage, err
}
app/maindb/manifest.go
+6 -5
View File
@@ -209,23 +209,24 @@ func (db *Database) GetManifestByDigest(ctx context.Context, name, digest string
return exists, manifest, err
}
func (db *Database) GetManifestsByReference(ctx context.Context, name, reference string) (bool, []descr.Manifest, error) {
func (db *Database) GetManifestByReference(ctx context.Context, name, reference string) (bool, descr.Manifest, error) {
var err error
exists := false
manifest := descr.Manifest{}
manifests := make([]descr.Manifest, 0)
request := `SELECT * FROM manifests WHERE name = $1 AND reference = $2 LIMIT 1`
err = db.db.Select(&manifests, request, name, reference)
if err != nil {
return exists, manifests, err
return exists, manifest, err
}
if len(manifests) > 0 {
manifest = manifests[0]
exists = true
}
return exists, manifests, err
return exists, manifest, err
}
func (db *Database) GetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
func (db *Database) xxxGetManifestsByReferenceArchitecture(ctx context.Context, name, reference, architecture, os, variant string) (bool, descr.Manifest, error) {
var err error
exists := false
manifest := descr.Manifest{}
app/maindb/scheme.go
+4 -1
View File
@@ -43,8 +43,11 @@ const schema = `
os VARCHAR(255) NOT NULL,
variant VARCHAR(255) NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index01
ON manifests(name, reference, architecture, os, variant);
CREATE UNIQUE INDEX IF NOT EXISTS manifests_index02
ON manifests(name, reference);
CREATE TABLE IF NOT EXISTS blobs (
id VARCHAR(255) NOT NULL,
app/storage/blob.go
+211
View File
@@ -0,0 +1,211 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"io"
"os"
"path/filepath"
)
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(name, digstr string) string {
return filepath.Join(store.basepath, blobsubdir, name, digstr) + ".bin"
}
func (store *Storage) makeBlobsubdir(name string) string {
return filepath.Join(store.basepath, blobsubdir, name)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, name, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir(name)
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(name, digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(name, digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir(name)
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(name, digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(name, digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(name, digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(name, digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(name, digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(name, digest string) error {
var err error
blobPath := store.makeBlobpath(name, digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
// TODO: remove dirs
return err
}
app/storage/file.go
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"fmt"
"io"
"os"
"path/filepath"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
app/storage/helmmeta.go → app/storage/helm.go
View File
app/storage/storage.go
-341
View File
@@ -1,341 +0,0 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"mstore/app/logger"
"mstore/pkg/auxuuid"
)
const (
sha256prefix = "sha256:"
fileSubdir = "files"
tmpSubdir = "tmps"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}
func (store *Storage) makeCollecionpath(collection string) string {
return filepath.Join(store.basepath, fileSubdir, collection)
}
func (store *Storage) makeFilepath(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir, collection, filename)
}
func (store *Storage) makeTmppath(tmpName string) string {
return filepath.Join(store.basepath, tmpSubdir, tmpName)
}
func (store *Storage) makeTmpsubdir() string {
return filepath.Join(store.basepath, tmpSubdir)
}
func (store *Storage) makeFilesubdir(collection, filename string) string {
return filepath.Join(store.basepath, fileSubdir)
}
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
var err error
var res io.ReadCloser
filename = store.makeFilepath(collection, filename)
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
if err != nil {
return res, err
}
res = file
return res, err
}
func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, error) {
var err error
var size int64
var csum string
tmpName := fmt.Sprintf("%s.tmp", auxuuid.NewUUID())
tmpPath := store.makeTmppath(tmpName)
tmpdirpath := store.makeTmpsubdir()
err = os.MkdirAll(tmpdirpath, 0750)
if err != nil {
return tmpName, size, csum, err
}
file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return tmpName, size, csum, err
}
defer file.Close()
hasher := NewHasher()
mWriter := io.MultiWriter(file, hasher.Writer())
size, err = io.Copy(mWriter, source)
if err != nil {
return tmpName, size, csum, err
}
csum = hasher.Hex()
return tmpName, size, csum, err
}
func (store *Storage) HardlinkFile(tmpName, collection, filename string) error {
var err error
dirname := store.makeCollecionpath(collection)
_, err = os.Stat(dirname)
if os.IsNotExist(err) {
err = os.MkdirAll(dirname, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
filename = store.makeFilepath(collection, filename)
_, err = os.Stat(dirname)
if err == nil {
os.Remove(filename) // TODO: safe removing
}
tmpName = store.makeTmppath(tmpName)
err = os.Link(tmpName, filename)
if err != nil {
return err
}
err = os.Remove(tmpName)
if err != nil {
return err
}
return err
}
func (store *Storage) DeleteFile(collection, filename string) error {
var err error
filename = store.makeFilepath(collection, filename)
err = os.Remove(filename)
if err != nil {
return err
}
// TODO: clean removing
dirname := store.makeCollecionpath(collection)
os.RemoveAll(dirname)
return err
}
const (
upsubdir = "uploads"
blobsubdir = "blobs"
)
func (store *Storage) makeUppath(upname string) string {
return filepath.Join(store.basepath, upsubdir, upname) + ".bin"
}
func (store *Storage) makeUpsubdir() string {
return filepath.Join(store.basepath, upsubdir)
}
func (store *Storage) makeBlobpath(upname string) string {
return filepath.Join(store.basepath, blobsubdir, upname) + ".bin"
}
func (store *Storage) makeBlobsubdir() string {
return filepath.Join(store.basepath, blobsubdir)
}
func (store *Storage) WriteUpload(upID string, source io.Reader) (int64, error) {
var err error
var recsize int64
uploadDir := store.makeUpsubdir()
_, err = os.Stat(uploadDir)
if os.IsNotExist(err) {
err = os.MkdirAll(uploadDir, 0750)
if err != nil {
return recsize, err
}
}
if err != nil {
return recsize, err
}
upPath := store.makeUppath(upID)
upFile, err := os.OpenFile(upPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return recsize, err
}
defer upFile.Close()
recsize, err = io.Copy(upFile, source)
if err != nil {
return recsize, err
}
return recsize, err
}
func (store *Storage) LinkUpload(reference, digest string) error {
var err error
upPath := store.makeUppath(reference)
blobdir := store.makeBlobsubdir()
_, err = os.Stat(blobdir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(blobdir, 0750)
if err != nil {
return err
}
}
if err != nil {
return err
}
blobPath := store.makeBlobpath(digest)
_, err = os.Stat(blobPath)
if err == nil {
err = os.Remove(blobPath)
if err != nil {
return err
}
}
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
err = os.Link(upPath, blobPath)
if err != nil {
return err
}
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (store *Storage) RemoveUpload(digest string) error {
var err error
upPath := store.makeUppath(digest)
err = os.Remove(upPath)
if err != nil {
return err
}
return err
}
func (st *Storage) UploadExists(name, reference string) (bool, int64, error) {
var err error
var size int64
upPath := st.makeUppath(reference)
fileStat, err := os.Stat(upPath)
if errors.Is(err, os.ErrNotExist) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) WriteBlob(digstr string, source io.Reader) (int64, error) {
var err error
var size int64
blobDir := store.makeBlobsubdir()
_, err = os.Stat(blobDir)
if os.IsNotExist(err) {
err = os.MkdirAll(blobDir, 0750)
if err != nil {
return size, err
}
}
if err != nil {
return size, err
}
blobPath := store.makeBlobpath(digstr)
blobFile, err := os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return size, err
}
defer blobFile.Close()
size, err = io.Copy(blobFile, source)
if err != nil {
return size, err
}
return size, err
}
func (st *Storage) BlobExists(digest string) (bool, int64, error) {
var err error
var size int64
blobPath := st.makeBlobpath(digest)
fileStat, err := os.Stat(blobPath)
if os.IsNotExist(err) {
return false, 0, nil
}
if err != nil {
return false, 0, err
}
size = fileStat.Size()
return true, size, err
}
func (store *Storage) BlobReader(digest string) (int64, io.ReadCloser, error) {
var err error
var size int64
blobPath := store.makeBlobpath(digest)
nop := io.NopCloser(bytes.NewReader(nil))
file, err := os.OpenFile(blobPath, os.O_RDONLY, 0)
if err != nil {
return size, nop, err
}
defer func() {
if err != nil {
file.Close()
}
}()
filestat, err := file.Stat()
if err != nil {
return size, nop, err
}
size = filestat.Size()
return size, file, err
}
func (store *Storage) DeleteBlob(digest string) error {
var err error
blobPath := store.makeBlobpath(digest)
err = os.Remove(blobPath)
if os.IsNotExist(err) {
err = nil
}
if err != nil {
return err
}
return err
}
app/storage/store.go
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package storage
import (
"mstore/app/logger"
)
type Storage struct {
basepath string
logg *logger.Logger
}
func NewStorage(basepath string) *Storage {
res := &Storage{
basepath: basepath,
}
res.logg = logger.NewLoggerWithSubject("storage")
return res
}
chart/templates/deployment.yaml
+1 -1
View File
@@ -27,7 +27,7 @@ spec:
- name: config-volume
mountPath: /app/etc/mstore
- name: db-volume
mountPath: /var/data
mountPath: /var/lib
volumes:
- name: config-volume
configMap:
go.mod
+1 -1
View File
@@ -6,7 +6,7 @@ require (
github.com/google/go-containerregistry v0.21.2
github.com/google/uuid v1.6.0
github.com/jmoiron/sqlx v1.4.0
github.com/mattn/go-sqlite3 v1.14.34
github.com/mattn/go-sqlite3 v1.14.37
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.1
github.com/spf13/cobra v1.10.2
go.sum
+2 -2
View File
@@ -147,8 +147,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg=
github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c
Generated Vendored
+133 -108
View File
@@ -1,7 +1,7 @@
#ifndef USE_LIBSQLITE3
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
** version 3.51.2. By combining all the individual C code files into this
** version 3.51.3. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a single translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
@@ -19,7 +19,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
** b270f8339eb13b504d0b2ba154ebca966b7d with changes in files:
** 737ae4a34738ffa0c3ff7f9bb18df914dd1c with changes in files:
**
**
*/
@@ -468,12 +468,12 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
#define SQLITE_VERSION "3.51.2"
#define SQLITE_VERSION_NUMBER 3051002
#define SQLITE_SOURCE_ID "2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075"
#define SQLITE_VERSION "3.51.3"
#define SQLITE_VERSION_NUMBER 3051003
#define SQLITE_SOURCE_ID "2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618"
#define SQLITE_SCM_BRANCH "branch-3.51"
#define SQLITE_SCM_TAGS "release version-3.51.2"
#define SQLITE_SCM_DATETIME "2026-01-09T17:27:48.405Z"
#define SQLITE_SCM_TAGS "release version-3.51.3"
#define SQLITE_SCM_DATETIME "2026-03-13T10:38:09.694Z"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -14335,6 +14335,27 @@ struct fts5_api {
#endif
#define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */
/*
** Maximum size of any single memory allocation.
**
** This is not a limit on the total amount of memory used. This is
** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc().
**
** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391
** This provides a 256-byte safety margin for defense against 32-bit
** signed integer overflow bugs when computing memory allocation sizes.
** Paranoid applications might want to reduce the maximum allocation size
** further for an even larger safety margin. 0x3fffffff or 0x0fffffff
** or even smaller would be reasonable upper bounds on the size of a memory
** allocations for most applications.
*/
#ifndef SQLITE_MAX_ALLOCATION_SIZE
# define SQLITE_MAX_ALLOCATION_SIZE 2147483391
#endif
#if SQLITE_MAX_ALLOCATION_SIZE>2147483391
# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391
#endif
/*
** This is the maximum number of
**
@@ -21665,6 +21686,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList
Expr*,ExprList*,u32,Expr*);
SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*);
SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*);
SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect);
SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*);
SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
@@ -31311,27 +31333,6 @@ static void mallocWithAlarm(int n, void **pp){
*pp = p;
}
/*
** Maximum size of any single memory allocation.
**
** This is not a limit on the total amount of memory used. This is
** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc().
**
** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391
** This provides a 256-byte safety margin for defense against 32-bit
** signed integer overflow bugs when computing memory allocation sizes.
** Paranoid applications might want to reduce the maximum allocation size
** further for an even larger safety margin. 0x3fffffff or 0x0fffffff
** or even smaller would be reasonable upper bounds on the size of a memory
** allocations for most applications.
*/
#ifndef SQLITE_MAX_ALLOCATION_SIZE
# define SQLITE_MAX_ALLOCATION_SIZE 2147483391
#endif
#if SQLITE_MAX_ALLOCATION_SIZE>2147483391
# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391
#endif
/*
** Allocate memory. This routine is like sqlite3_malloc() except that it
** assumes the memory subsystem has already been initialized.
@@ -31555,8 +31556,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){
sqlite3_free(pOld); /* IMP: R-26507-47431 */
return 0;
}
if( nBytes>=0x7fffff00 ){
/* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */
if( nBytes>SQLITE_MAX_ALLOCATION_SIZE ){
return 0;
}
nOld = sqlite3MallocSize(pOld);
@@ -69011,68 +69011,82 @@ static int walCheckpoint(
&& (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
){
u32 nBackfill = pInfo->nBackfill;
pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
WalIndexHdr *pLive = (WalIndexHdr*)walIndexHdr(pWal);
/* Sync the WAL to disk */
rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
/* Now that read-lock slot 0 is locked, check that the wal has not been
** wrapped since the header was read for this checkpoint. If it was, then
** there was no work to do anyway. In this case the
** (pInfo->nBackfill<pWal->hdr.mxFrame) test above only passed because
** pInfo->nBackfill had already been set to 0 by the writer that wrapped
** the wal file. It would also be dangerous to proceed, as there may be
** fewer than pWal->hdr.mxFrame valid frames in the wal file. */
int bChg = memcmp(pLive->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt));
if( 0==bChg ){
pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
/* If the database may grow as a result of this checkpoint, hint
** about the eventual size of the db file to the VFS layer.
*/
if( rc==SQLITE_OK ){
i64 nReq = ((i64)mxPage * szPage);
i64 nSize; /* Current size of database file */
sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0);
rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
if( rc==SQLITE_OK && nSize<nReq ){
if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){
/* If the size of the final database is larger than the current
** database plus the amount of data in the wal file, plus the
** maximum size of the pending-byte page (65536 bytes), then
** must be corruption somewhere. */
rc = SQLITE_CORRUPT_BKPT;
}else{
sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq);
}
}
/* Sync the WAL to disk */
rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
}
/* Iterate through the contents of the WAL, copying data to the db file */
while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
i64 iOffset;
assert( walFramePgno(pWal, iFrame)==iDbpage );
SEH_INJECT_FAULT;
if( AtomicLoad(&db->u1.isInterrupted) ){
rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
break;
}
if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
continue;
}
iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
/* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
if( rc!=SQLITE_OK ) break;
iOffset = (iDbpage-1)*(i64)szPage;
testcase( IS_BIG_INT(iOffset) );
rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
if( rc!=SQLITE_OK ) break;
}
sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0);
/* If work was actually accomplished... */
if( rc==SQLITE_OK ){
if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
i64 szDb = pWal->hdr.nPage*(i64)szPage;
testcase( IS_BIG_INT(szDb) );
rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
if( rc==SQLITE_OK ){
rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
}
}
/* If the database may grow as a result of this checkpoint, hint
** about the eventual size of the db file to the VFS layer.
*/
if( rc==SQLITE_OK ){
AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
i64 nReq = ((i64)mxPage * szPage);
i64 nSize; /* Current size of database file */
sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0);
rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
if( rc==SQLITE_OK && nSize<nReq ){
if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){
/* If the size of the final database is larger than the current
** database plus the amount of data in the wal file, plus the
** maximum size of the pending-byte page (65536 bytes), then
** must be corruption somewhere. */
rc = SQLITE_CORRUPT_BKPT;
}else{
sqlite3OsFileControlHint(
pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
}
}
}
/* Iterate through the contents of the WAL, copying data to the
** db file */
while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
i64 iOffset;
assert( walFramePgno(pWal, iFrame)==iDbpage );
SEH_INJECT_FAULT;
if( AtomicLoad(&db->u1.isInterrupted) ){
rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
break;
}
if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
continue;
}
iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
/* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
if( rc!=SQLITE_OK ) break;
iOffset = (iDbpage-1)*(i64)szPage;
testcase( IS_BIG_INT(iOffset) );
rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
if( rc!=SQLITE_OK ) break;
}
sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0);
/* If work was actually accomplished... */
if( rc==SQLITE_OK ){
if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
i64 szDb = pWal->hdr.nPage*(i64)szPage;
testcase( IS_BIG_INT(szDb) );
rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
if( rc==SQLITE_OK ){
rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
}
}
if( rc==SQLITE_OK ){
AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
}
}
}
@@ -71122,6 +71136,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint(
/* Copy data from the log to the database file. */
if( rc==SQLITE_OK ){
sqlite3FaultSim(660);
if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
rc = SQLITE_CORRUPT_BKPT;
}else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){
@@ -77557,7 +77572,7 @@ static int accessPayload(
getCellInfo(pCur);
aPayload = pCur->info.pPayload;
assert( offset+amt <= pCur->info.nPayload );
assert( (u64)offset+(u64)amt <= (u64)pCur->info.nPayload );
assert( aPayload > pPage->aData );
if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){
@@ -86031,7 +86046,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(
){
int rc;
pMem->flags = MEM_Null;
if( sqlite3BtreeMaxRecordSize(pCur)<offset+amt ){
testcase( amt==SQLITE_MAX_ALLOCATION_SIZE-1 );
testcase( amt==SQLITE_MAX_ALLOCATION_SIZE );
if( amt>=SQLITE_MAX_ALLOCATION_SIZE ){
return SQLITE_NOMEM_BKPT;
}
if( (u64)amt + (u64)offset > (u64)sqlite3BtreeMaxRecordSize(pCur) ){
return SQLITE_CORRUPT_BKPT;
}
if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){
@@ -93444,7 +93464,7 @@ static int valueFromValueList(
Mem sMem; /* Raw content of current row */
memset(&sMem, 0, sizeof(sMem));
sz = sqlite3BtreePayloadSize(pRhs->pCsr);
rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem);
rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,sz,&sMem);
if( rc==SQLITE_OK ){
u8 *zBuf = (u8*)sMem.z;
u32 iSerial;
@@ -111186,6 +111206,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
return WRC_Abort;
}
/* If the SELECT statement contains ON clauses that were moved into
** the WHERE clause, go through and verify that none of the terms
** in the ON clauses reference tables to the right of the ON clause. */
if( (p->selFlags & SF_OnToWhere) ){
sqlite3SelectCheckOnClauses(pParse, p);
if( pParse->nErr ) return WRC_Abort;
}
/* Advance to the next term of the compound
*/
p = p->pPrior;
@@ -154357,7 +154385,7 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){
** Check all ON clauses in pSelect to verify that they do not reference
** columns to the right.
*/
static void selectCheckOnClauses(Parse *pParse, Select *pSelect){
SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){
Walker w;
CheckOnCtx sCtx;
assert( pSelect->selFlags & SF_OnToWhere );
@@ -154500,18 +154528,6 @@ SQLITE_PRIVATE int sqlite3Select(
}
#endif
/* If the SELECT statement contains ON clauses that were moved into
** the WHERE clause, go through and verify that none of the terms
** in the ON clauses reference tables to the right of the ON clause.
** Do this now, after name resolution, but before query flattening
*/
if( p->selFlags & SF_OnToWhere ){
selectCheckOnClauses(pParse, p);
if( pParse->nErr ){
goto select_end;
}
}
/* If the SF_UFSrcCheck flag is set, then this function is being called
** as part of populating the temp table for an UPDATE...FROM statement.
** In this case, it is an error if the target object (pSrc->a[0]) name
@@ -164678,6 +164694,15 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop(
sqlite3ExprDup(pParse->db, pTerm->pExpr, 0));
}
}
if( pLevel->iIdxCur ){
/* pSubWhere may contain expressions that read from an index on the
** table on the RHS of the right join. All such expressions first test
** if the index is pointing at a NULL row, and if so, read from the
** table cursor instead. So ensure that the index cursor really is
** pointing at a NULL row here, so that no values are read from it during
** the scan of the RHS of the RIGHT join below. */
sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
}
pFrom = &uSrc.sSrc;
pFrom->nSrc = 1;
pFrom->nAlloc = 1;
@@ -224164,7 +224189,7 @@ static int rbuDeltaApply(
/* ERROR: copy exceeds output file size */
return -1;
}
if( (int)(ofst+cnt) > lenSrc ){
if( (u64)ofst+(u64)cnt > (u64)lenSrc ){
/* ERROR: copy extends past end of input */
return -1;
}
@@ -252451,7 +252476,7 @@ static void fts5DoSecureDelete(
int iSegid = pSeg->pSeg->iSegid;
u8 *aPg = pSeg->pLeaf->p;
int nPg = pSeg->pLeaf->nn;
int iPgIdx = pSeg->pLeaf->szLeaf;
int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */
u64 iDelta = 0;
int iNextOff = 0;
@@ -252530,7 +252555,7 @@ static void fts5DoSecureDelete(
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
}
assert_nc( iSOP==pSeg->iLeafOffset );
iNextOff = pSeg->iLeafOffset + pSeg->nPos;
iNextOff = iSOP + pSeg->nPos;
}
}
@@ -260349,7 +260374,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "fts5: 2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075", -1, SQLITE_TRANSIENT);
sqlite3_result_text(pCtx, "fts5: 2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618", -1, SQLITE_TRANSIENT);
}
/*
vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h
Generated Vendored
+5 -5
View File
@@ -147,12 +147,12 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
#define SQLITE_VERSION "3.51.2"
#define SQLITE_VERSION_NUMBER 3051002
#define SQLITE_SOURCE_ID "2026-01-09 17:27:48 b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075"
#define SQLITE_VERSION "3.51.3"
#define SQLITE_VERSION_NUMBER 3051003
#define SQLITE_SOURCE_ID "2026-03-13 10:38:09 737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618"
#define SQLITE_SCM_BRANCH "branch-3.51"
#define SQLITE_SCM_TAGS "release version-3.51.2"
#define SQLITE_SCM_DATETIME "2026-01-09T17:27:48.405Z"
#define SQLITE_SCM_TAGS "release version-3.51.3"
#define SQLITE_SCM_DATETIME "2026-03-13T10:38:09.694Z"
/*
** CAPI3REF: Run-Time Library Version Numbers
vendor/github.com/mattn/go-sqlite3/sqlite3.go
Generated Vendored
+17 -10
View File
@@ -1783,15 +1783,18 @@ func (d *SQLiteDriver) Open(dsn string) (driver.Conn, error) {
// Close the connection.
func (c *SQLiteConn) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.db == nil {
return nil
}
runtime.SetFinalizer(c, nil)
rv := C.sqlite3_close_v2(c.db)
if rv != C.SQLITE_OK {
return c.lastError()
return lastError(c.db)
}
deleteHandles(c)
c.mu.Lock()
c.db = nil
c.mu.Unlock()
runtime.SetFinalizer(c, nil)
return nil
}
@@ -1932,16 +1935,18 @@ func (s *SQLiteStmt) Close() error {
return nil
}
s.closed = true
if !s.c.dbConnOpen() {
runtime.SetFinalizer(s, nil)
conn := s.c
stmt := s.s
s.s = nil
s.c = nil
if !conn.dbConnOpen() {
return errors.New("sqlite statement with already closed database connection")
}
rv := C.sqlite3_finalize(s.s)
s.s = nil
rv := C.sqlite3_finalize(stmt)
if rv != C.SQLITE_OK {
return s.c.lastError()
return conn.lastError()
}
s.c = nil
runtime.SetFinalizer(s, nil)
return nil
}
@@ -1958,6 +1963,8 @@ func (s *SQLiteStmt) bind(args []driver.NamedValue) error {
return s.c.lastError()
}
C.sqlite3_clear_bindings(s.s)
bindIndices := make([][3]int, len(args))
prefixes := []string{":", "@", "$"}
for i, v := range args {
vendor/github.com/mattn/go-sqlite3/sqlite3_opt_serialize.go
Generated Vendored
+2 -14
View File
@@ -17,7 +17,6 @@ import "C"
import (
"fmt"
"math"
"reflect"
"unsafe"
)
@@ -43,14 +42,8 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) {
return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz)
}
cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(ptr)),
Len: int(sz),
Cap: int(sz),
}))
res := make([]byte, int(sz))
copy(res, cBuf)
copy(res, unsafe.Slice((*byte)(unsafe.Pointer(ptr)), int(sz)))
return res, nil
}
@@ -67,12 +60,7 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error {
defer C.free(unsafe.Pointer(zSchema))
tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b))))
cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(tmpBuf)),
Len: len(b),
Cap: len(b),
}))
copy(cBuf, b)
copy(unsafe.Slice((*byte)(unsafe.Pointer(tmpBuf)), len(b)), b)
rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)),
C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE)
vendor/modules.txt
Vendored
+1 -1
View File
@@ -235,7 +235,7 @@ github.com/liggitt/tabwriter
github.com/mailru/easyjson/buffer
github.com/mailru/easyjson/jlexer
github.com/mailru/easyjson/jwriter
# github.com/mattn/go-sqlite3 v1.14.34
# github.com/mattn/go-sqlite3 v1.14.37
## explicit; go 1.19
github.com/mattn/go-sqlite3
# github.com/mitchellh/go-homedir v1.1.0