Skip to content

Commit fce8578

Browse files
amitksingh1490autofix-ci[bot]tusharmath
authored
fix(forge_repo): retry database pool creation and connection access (#2608)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Tushar Mathur <tusharmath@gmail.com>
1 parent 02aef23 commit fce8578

3 files changed

Lines changed: 79 additions & 31 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/forge_repo/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ forge_fs.workspace = true
1414
forge_template.workspace = true
1515
forge_json_repair.workspace = true
1616
anyhow.workspace = true
17+
backon.workspace = true
1718
futures.workspace = true
1819
async-trait.workspace = true
1920
serde.workspace = true

‎crates/forge_repo/src/database/pool.rs‎

Lines changed: 77 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use std::path::PathBuf;
33
use std::time::Duration;
44

55
use anyhow::Result;
6+
use backon::{BlockingRetryable, ExponentialBuilder};
67
use diesel::prelude::*;
78
use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool, PooledConnection};
89
use diesel::sqlite::SqliteConnection;
@@ -20,6 +21,7 @@ pub struct PoolConfig {
2021
pub min_idle: Option<u32>,
2122
pub connection_timeout: Duration,
2223
pub idle_timeout: Option<Duration>,
24+
pub max_retries: usize,
2325
pub database_path: PathBuf,
2426
}
2527

@@ -28,15 +30,17 @@ impl PoolConfig {
2830
Self {
2931
max_size: 5,
3032
min_idle: Some(1),
31-
connection_timeout: Duration::from_secs(30),
33+
connection_timeout: Duration::from_secs(5),
3234
idle_timeout: Some(Duration::from_secs(600)), // 10 minutes
35+
max_retries: 5,
3336
database_path,
3437
}
3538
}
3639
}
3740

3841
pub struct DatabasePool {
3942
pool: DbPool,
43+
max_retries: usize,
4044
}
4145

4246
impl DatabasePool {
@@ -61,14 +65,65 @@ impl DatabasePool {
6165
.run_pending_migrations(MIGRATIONS)
6266
.map_err(|e| anyhow::anyhow!("Failed to run database migrations: {e}"))?;
6367

64-
Ok(Self { pool })
68+
Ok(Self { pool, max_retries: 5 })
6569
}
6670

6771
pub fn get_connection(&self) -> Result<PooledSqliteConnection> {
68-
self.pool.get().map_err(|e| {
69-
warn!(error = %e, "Failed to get connection from pool");
70-
anyhow::anyhow!("Failed to get connection from pool: {e}")
71-
})
72+
Self::retry_with_backoff(
73+
self.max_retries,
74+
"Failed to get connection from pool, retrying",
75+
|| {
76+
self.pool
77+
.get()
78+
.map_err(|e| anyhow::anyhow!("Failed to get connection from pool: {e}"))
79+
},
80+
)
81+
}
82+
83+
/// Retries a blocking database pool operation with exponential backoff.
84+
fn retry_with_backoff<T>(
85+
max_retries: usize,
86+
message: &'static str,
87+
operation: impl FnMut() -> Result<T>,
88+
) -> Result<T> {
89+
operation
90+
.retry(
91+
ExponentialBuilder::default()
92+
.with_min_delay(Duration::from_secs(1))
93+
.with_max_times(max_retries)
94+
.with_jitter(),
95+
)
96+
.sleep(std::thread::sleep)
97+
.notify(|err, dur| {
98+
warn!(
99+
error = %err,
100+
retry_after_ms = dur.as_millis() as u64,
101+
"{}",
102+
message
103+
);
104+
})
105+
.call()
106+
}
107+
}
108+
// Configure SQLite for better concurrency ref: https://docs.diesel.rs/master/diesel/sqlite/struct.SqliteConnection.html#concurrency
109+
#[derive(Debug)]
110+
struct SqliteCustomizer;
111+
112+
impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for SqliteCustomizer {
113+
fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), diesel::r2d2::Error> {
114+
diesel::sql_query("PRAGMA busy_timeout = 30000;")
115+
.execute(conn)
116+
.map_err(diesel::r2d2::Error::QueryError)?;
117+
diesel::sql_query("PRAGMA journal_mode = WAL;")
118+
.execute(conn)
119+
.map_err(diesel::r2d2::Error::QueryError)?;
120+
diesel::sql_query("PRAGMA synchronous = NORMAL;")
121+
.execute(conn)
122+
.map_err(diesel::r2d2::Error::QueryError)?;
123+
diesel::sql_query("PRAGMA wal_autocheckpoint = 1000;")
124+
.execute(conn)
125+
.map_err(diesel::r2d2::Error::QueryError)?;
126+
Ok(())
72127
}
73128
}
74129

@@ -83,36 +138,27 @@ impl TryFrom<PoolConfig> for DatabasePool {
83138
std::fs::create_dir_all(parent)?;
84139
}
85140

141+
// Retry pool creation with exponential backoff to handle transient
142+
// failures such as another process holding an exclusive lock on the
143+
// SQLite database file.
144+
DatabasePool::retry_with_backoff(
145+
config.max_retries,
146+
"Failed to create database pool, retrying",
147+
|| Self::build_pool(&config),
148+
)
149+
}
150+
}
151+
152+
impl DatabasePool {
153+
/// Builds the connection pool and runs migrations.
154+
fn build_pool(config: &PoolConfig) -> Result<Self> {
86155
let database_url = config.database_path.to_string_lossy().to_string();
87156
let manager = ConnectionManager::<SqliteConnection>::new(&database_url);
88157

89-
// Configure SQLite for better concurrency ref: https://docs.diesel.rs/master/diesel/sqlite/struct.SqliteConnection.html#concurrency
90-
#[derive(Debug)]
91-
struct SqliteCustomizer;
92-
impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for SqliteCustomizer {
93-
fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), diesel::r2d2::Error> {
94-
diesel::sql_query("PRAGMA busy_timeout = 30000;")
95-
.execute(conn)
96-
.map_err(diesel::r2d2::Error::QueryError)?;
97-
diesel::sql_query("PRAGMA journal_mode = WAL;")
98-
.execute(conn)
99-
.map_err(diesel::r2d2::Error::QueryError)?;
100-
diesel::sql_query("PRAGMA synchronous = NORMAL;")
101-
.execute(conn)
102-
.map_err(diesel::r2d2::Error::QueryError)?;
103-
diesel::sql_query("PRAGMA wal_autocheckpoint = 1000;")
104-
.execute(conn)
105-
.map_err(diesel::r2d2::Error::QueryError)?;
106-
Ok(())
107-
}
108-
}
109-
110-
let customizer = SqliteCustomizer;
111-
112158
let mut builder = Pool::builder()
113159
.max_size(config.max_size)
114160
.connection_timeout(config.connection_timeout)
115-
.connection_customizer(Box::new(customizer));
161+
.connection_customizer(Box::new(SqliteCustomizer));
116162

117163
if let Some(min_idle) = config.min_idle {
118164
builder = builder.min_idle(Some(min_idle));
@@ -138,6 +184,6 @@ impl TryFrom<PoolConfig> for DatabasePool {
138184
})?;
139185

140186
debug!(database_path = %config.database_path.display(), "created connection pool");
141-
Ok(Self { pool })
187+
Ok(Self { pool, max_retries: config.max_retries })
142188
}
143189
}

0 commit comments

Comments
 (0)