Repository navigation
Expand file tree
/
Copy pathsdkdb.go
More file actions
147 lines (140 loc) · 5.17 KB
/
Copy pathsdkdb.go
File metadata and controls
147 lines (140 loc) · 5.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
package anysyncsdk
import (
"context"
"errors"
"io/fs"
"os"
"runtime"
"time"
anystore "github.com/anyproto/any-store/v2"
"go.uber.org/zap"
)
// sdkDBCopySuffix names the copy of sdk.db in the latest on-disk format
// until it takes the file's place.
const sdkDBCopySuffix = ".migrate"
// The two steps after the source is closed that decide which file is in
// place; variables so a test can fail them.
var (
renameFile = os.Rename
syncDir = syncDirOf
)
// sdkDBConfig is the configuration sdk.db is opened with, and the one a
// copy of it is written with. Open fills in defaults, so each open gets
// its own.
func sdkDBConfig() *anystore.Config {
return &anystore.Config{
UseGlobalPageBuffer: true,
// One shared DB serves every space's reads; keep the engine's
// CPU scaling but never drop below 8 concurrent readers.
ReadConcurrency: max(runtime.NumCPU(), 8),
// Without the idle flush, checkpoints only happen when the WAL
// hits the commit-path threshold (10000 frames ≈ 40 MB) — a busy
// DB sits under a huge WAL that must be replayed on every open.
// Idle checkpointing keeps it bounded; the sentinel adds a
// quick-check after unclean shutdowns.
Durability: anystore.DurabilityConfig{
AutoFlush: true,
IdleAfter: 20 * time.Second,
FlushMode: anystore.FlushModeCheckpointPassive,
Sentinel: true,
},
}
}
// openSDKDB opens sdk.db. A file in an older on-disk format is replaced by
// a copy in the latest one before anyone else holds the handle; no older
// any-store release reads the result. When the copy cannot be made or put
// in place the old file stays: the open goes on with it and the next open
// tries again. A cancelled open fails, with the old file or the complete
// copy in place.
func openSDKDB(ctx context.Context, path string) (anystore.DB, error) {
InitPageBuffer()
db, err := anystore.Open(ctx, path, sdkDBConfig())
if err != nil {
return nil, err
}
format, err := anystore.SchemaFormat(db)
if err != nil {
_ = db.Close()
return nil, err
}
if format >= anystore.LatestSchemaFormat {
return db, nil
}
log.Info("sdk.db: copying to the latest on-disk format",
zap.Int("from", format), zap.Int("to", anystore.LatestSchemaFormat))
copied, report, err := replaceWithLatestFormat(ctx, db, path)
switch {
case err == nil:
log.Info("sdk.db: on-disk format updated",
zap.Int("collections", report.Collections), zap.Int("documents", report.Documents),
zap.Int("indexes", report.Indexes), zap.Duration("took", report.Duration))
return copied, nil
case ctx.Err() != nil:
if copied != nil {
_ = copied.Close()
}
return nil, err
case copied != nil:
log.Warn("sdk.db: not copied to the latest on-disk format, staying on the old one", zap.Error(err))
return copied, nil
default:
// The source is closed: path holds the old database or the copy.
log.Warn("sdk.db: replacing the file with its copy did not finish, opening what is in place", zap.Error(err))
return anystore.Open(ctx, path, sdkDBConfig())
}
}
// replaceWithLatestFormat puts a copy of db in the latest on-disk format in
// the place of the file at path, and returns the database opened from it.
// db is the only handle on the file and nothing writes through it: the
// host opens a data dir once at a time and keeps other processes off the
// file. The steps and their order are any-store's
// (docs/migration-format-5-to-6.md): a crash at any of them leaves the old
// file with its log, or the copy.
//
// On an error before the source is closed it returns db, still open. After
// that it returns nil, and path is to be opened again.
func replaceWithLatestFormat(ctx context.Context, db anystore.DB, path string) (anystore.DB, anystore.CopyReport, error) {
tmp := path + sdkDBCopySuffix
// A copy an interrupted run left can be older than the database.
if err := os.Remove(tmp); err != nil && !errors.Is(err, fs.ErrNotExist) {
return db, anystore.CopyReport{}, err
}
// ErrCopyPublished: the copy is complete and only a step after it
// failed; the directory is synced again below.
report, err := anystore.CopyToLatestFormat(ctx, db, tmp, sdkDBConfig())
if err != nil && !errors.Is(err, anystore.ErrCopyPublished) {
return db, anystore.CopyReport{}, err
}
// Close drops a checkpoint error, so checkpoint first.
if err := db.Flush(ctx, 0, anystore.FlushModeCheckpointTruncate); err != nil {
_ = os.Remove(tmp)
return db, anystore.CopyReport{}, err
}
if err := db.Close(); err != nil {
_ = os.Remove(tmp)
return nil, anystore.CopyReport{}, err
}
// The copy must never sit beside the old file's log.
for _, sidecar := range []string{path + "-wal", path + "-wal-shm"} {
if err := os.Remove(sidecar); err != nil && !errors.Is(err, fs.ErrNotExist) {
_ = os.Remove(tmp)
return nil, anystore.CopyReport{}, err
}
}
if err := syncDir(path); err != nil {
_ = os.Remove(tmp)
return nil, anystore.CopyReport{}, err
}
if err := renameFile(tmp, path); err != nil {
_ = os.Remove(tmp)
return nil, anystore.CopyReport{}, err
}
if err := syncDir(path); err != nil {
return nil, anystore.CopyReport{}, err
}
copied, err := anystore.Open(ctx, path, sdkDBConfig())
if err != nil {
return nil, anystore.CopyReport{}, err
}
return copied, report, nil
}