[WIP] Add DB compression

.github/workflows/bedrock.yml

@@ -62,6 +62,8 @@ jobs:
     - name: Mark failure if debugging
       if: runner.debug == '1'
       run: exit 1
+    - name: Install zstd
+      run: apt-get update -y && apt-get install -y libzstd-dev
     - name: Build Bedrock
       run: "./ci_build.sh"
     - name: Upload binaries
@@ -94,6 +96,8 @@ jobs:
       with:
         repository: Expensify/Bedrock
         path: .
+    - name: Install zstd
+      run: apt-get update -y && apt-get install -y libzstd-dev
     - name: Download binaries
       uses: ./.github/actions/composite/download-binaries
     - name: Setup tmate session
@@ -129,6 +133,8 @@ jobs:
       with:
         repository: Expensify/Bedrock
         path: .
+    - name: Install zstd
+      run: apt-get update -y && apt-get install -y libzstd-dev
     - name: Download binaries
       uses: ./.github/actions/composite/download-binaries
     - name: Setup tmate session

BedrockPlugin.cpp

@@ -81,6 +81,10 @@ void BedrockPlugin::upgradeDatabase(SQLite& db)
 {
 }
 
+void BedrockPlugin::initializeFromDB(SQLite& db)
+{
+}
+
 bool BedrockPlugin::shouldLockCommitPageOnConflict(const string& conflictLocation) const
 {
     return true;

BedrockPlugin.h

@@ -38,6 +38,10 @@ class BedrockPlugin {
     // Called at some point during initiation to allow the plugin to verify/change the database schema.
     virtual void upgradeDatabase(SQLite& db);
 
+    // Called once after upgradeDatabase has completed, allowing plugins to read from the DB at startup.
+    // This runs on the sync thread before any commands are processed.
+    virtual void initializeFromDB(SQLite& db);
+
     // The plugin can register any number of timers it wants. When any of them `ding`, then the `timerFired`
     // function will be called, and passed the timer that is dinging.
     set<SStopwatch*> timers;

BedrockServer.cpp

@@ -106,10 +106,16 @@ void BedrockServer::sync()
 
     // We use fewer FDs on test machines that have other resource restrictions in place.
 
+    SQLite::journalZstdDictionaryID = args.calc("-journalZstdDictionaryID");
     SINFO("Setting dbPool size to: " << _dbPoolSize);
     _dbPool = make_shared<SQLitePool>(_dbPoolSize, args["-db"], args.calc("-cacheSize"), args.calc("-maxJournalSize"), journalTables, mmapSizeGB, args.isSet("-newDBsUseHctree"), args["-checkpointMode"]);
     SQLite& db = _dbPool->getBase();
 
+    // Allow plugins to read from the DB at startup.
+    for (auto plugin : plugins) {
+        plugin.second->initializeFromDB(db);
+    }
+
     // Initialize the command processor.
     BedrockCore core(db, *this);
 

Makefile

@@ -22,7 +22,7 @@ INTERMEDIATEDIR = .build
 
 # We use the same library paths and required libraries for all binaries.
 LIBPATHS =-L$(PROJECT) -Lmbedtls/library
-LIBRARIES =-Wl,--start-group -lbedrock -lstuff -Wl,--end-group -ldl -lpcre2-8 -lpthread -lmbedtls -lmbedx509 -lmbedcrypto -lz -lm
+LIBRARIES =-Wl,--start-group -lbedrock -lstuff -Wl,--end-group -ldl -lpcre2-8 -lpthread -lmbedtls -lmbedx509 -lmbedcrypto -lz -lzstd -lm
 
 # These targets aren't actual files.
 .PHONY: all test clustertest clean testplugin

ci_style.sh

@@ -50,7 +50,7 @@ do
     fi
 
     # Counts occurrences of std:: that aren't in comments and have a leading space (except if it's inside pointer brackets, eg: <std::thing>)
-    RETURN_VAL=$(sed -n '/^.*\/\/.*/!s/ std:://p; /^.* std::.*\/\//s/ std:://p; /^.*\<.*std::.*\>/s/std:://p;' "${FILENAME}" | wc -l)
+    RETURN_VAL=$(sed -n '/^.*\/\/.*/!s/ std:://p; /^.* std::.*\/\//s/ std:://p; /^.*\<.* std::.*\>/s/ std:://p;' "${FILENAME}" | wc -l)
     if [[ $RETURN_VAL -gt 0 ]] && [[ "$FAILED" != "true" ]]; then
         echo -e "${RED}${OUT} failed style checks, do not use std:: prefix.${RESET}"
         EXIT_VAL=$RETURN_VAL

docker/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA9EF27F && \
 # Update and install dependencies
 RUN apt-fast update && apt-fast install -y \
     wget gnupg software-properties-common lsb-release ccache zlib1g-dev rsyslog cmake \
-    libpcre2-dev libpcre3-dev libsodium-dev libgpgme11-dev libstdc++-13-dev make \
+    libpcre2-dev libpcre3-dev libsodium-dev libgpgme11-dev libstdc++-13-dev libzstd-dev make \
     linux-headers-generic git clang-18 lldb-18 lld-18 clangd-18 clang-tidy-18 \
     clang-format-18 clang-tools-18 llvm-18-dev llvm-18-tools libomp-18-dev libc++-18-dev \
     libc++abi-18-dev libclang-common-18-dev libclang-18-dev libclang-cpp18-dev libunwind-18-dev
@@ -69,7 +69,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 # Install necessary packages
 RUN apt-get update && apt-get install -y software-properties-common && \
-    apt-get install -y build-essential libpcre++ zlib1g && \
+    apt-get install -y build-essential libpcre++ zlib1g libzstd1 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     rm -rf /etc/apt/sources.list.d/*

main.cpp

@@ -15,6 +15,7 @@
 #include <plugins/DB.h>
 #include <plugins/Jobs.h>
 #include <plugins/MySQL.h>
+#include <plugins/Compression.h>
 #include <libstuff/libstuff.h>
 #include <libstuff/SSSLState.h>
 #include <sqlitecluster/SQLite.h>
@@ -103,6 +104,9 @@ set<string> loadPlugins(SData& args)
     BedrockPlugin::g_registeredPluginList.emplace(make_pair("MYSQL", [](BedrockServer& s){
         return new BedrockPlugin_MySQL(s);
     }));
+    BedrockPlugin::g_registeredPluginList.emplace(make_pair("ZSTD", [](BedrockServer& s){
+        return new BedrockPlugin_Zstd(s);
+    }));
 
     for (string pluginName : plugins) {
         // If it's one of our standard plugins, just move on to the next one.
@@ -324,11 +328,12 @@ int main(int argc, char* argv[])
     SETDEFAULT("-controlPort", "localhost:9999");
     SETDEFAULT("-nodeName", SGetHostName());
     SETDEFAULT("-cacheSize", SToStr(0));
-    SETDEFAULT("-plugins", "db,jobs,cache,mysql");
+    SETDEFAULT("-plugins", "db,jobs,cache,mysql,zstd");
     SETDEFAULT("-priority", "100");
     SETDEFAULT("-maxJournalSize", "1000000");
     SETDEFAULT("-queryLog", "queryLog.csv");
     SETDEFAULT("-enableMultiWrite", "true");
+    SETDEFAULT("-journalZstdDictionaryID", "0");
 
     // We default to PASSIVE checkpoint everywhere as that has been the value proven to work fine for many years.
     SETDEFAULT("-checkpointMode", "PASSIVE");