fix(database/gdb): Fix N+1 query problem and add batch query control

contrib/drivers/mysql/testdata/with_tpl_detail_meta_soft_delete.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS %s (
+    id int(10) unsigned NOT NULL AUTO_INCREMENT,
+    detail_id int(10) unsigned NOT NULL,
+    meta_key varchar(50) NOT NULL,
+    meta_value varchar(100) NOT NULL,
+    sort_order int(10) unsigned NOT NULL DEFAULT 0,
+    deleted_at datetime default NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

contrib/drivers/mysql/testdata/with_tpl_score_details_soft_delete.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS %s (
+    id int(10) unsigned NOT NULL AUTO_INCREMENT,
+    score_id int(10) unsigned NOT NULL,
+    detail_info varchar(100) NOT NULL,
+    rank int(10) unsigned NOT NULL DEFAULT 0,
+    deleted_at datetime default NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

contrib/drivers/mysql/testdata/with_tpl_user_detail.sql

@@ -1,5 +1,6 @@
 CREATE TABLE IF NOT EXISTS %s (
     uid int(10) unsigned NOT NULL AUTO_INCREMENT,
     address varchar(45) NOT NULL,
+    deleted_at datetime default NULL,
     PRIMARY KEY (uid)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

contrib/drivers/mysql/testdata/with_tpl_user_scores_soft_delete.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS %s (
+    id int(10) unsigned NOT NULL AUTO_INCREMENT,
+    uid int(10) unsigned NOT NULL,
+    score int(10) unsigned NOT NULL,
+    priority int(10) unsigned NOT NULL DEFAULT 0,
+    deleted_at datetime default NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

contrib/drivers/mysql/testdata/with_tpl_user_soft_delete.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS %s (
+    id int(10) unsigned NOT NULL AUTO_INCREMENT,
+    name varchar(45) NOT NULL,
+    status int(10) unsigned NOT NULL DEFAULT 1,
+    deleted_at datetime default NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

database/gdb/gdb_func.go

@@ -66,6 +66,9 @@ const (
 	OrmTagForWithOrder    = "order"
 	OrmTagForWithUnscoped = "unscoped"
 	OrmTagForDo           = "do"
+	OrmTagForChunkName    = "chunkName"
+	OrmTagForChunkSize    = "chunkSize"
+	OrmTagForChunkMinRows = "chunkMinRows"
 )
 
 var (

database/gdb/gdb_model.go

@@ -17,45 +17,56 @@ import (
 
 // Model is core struct implementing the DAO for ORM.
 type Model struct {
-	db              DB                // Underlying DB interface.
-	tx              TX                // Underlying TX interface.
-	rawSql          string            // rawSql is the raw SQL string which marks a raw SQL based Model not a table based Model.
-	schema          string            // Custom database schema.
-	linkType        int               // Mark for operation on master or slave.
-	tablesInit      string            // Table names when model initialization.
-	tables          string            // Operation table names, which can be more than one table names and aliases, like: "user", "user u", "user u, user_detail ud".
-	fields          []any             // Operation fields, multiple fields joined using char ','.
-	fieldsEx        []any             // Excluded operation fields, it here uses slice instead of string type for quick filtering.
-	withArray       []any             // Arguments for With feature.
-	withAll         bool              // Enable model association operations on all objects that have "with" tag in the struct.
-	extraArgs       []any             // Extra custom arguments for sql, which are prepended to the arguments before sql committed to underlying driver.
-	whereBuilder    *WhereBuilder     // Condition builder for where operation.
-	groupBy         string            // Used for "group by" statement.
-	orderBy         string            // Used for "order by" statement.
-	having          []any             // Used for "having..." statement.
-	start           int               // Used for "select ... start, limit ..." statement.
-	limit           int               // Used for "select ... start, limit ..." statement.
-	option          int               // Option for extra operation features.
-	offset          int               // Offset statement for some databases grammar.
-	partition       string            // Partition table partition name.
-	data            any               // Data for operation, which can be type of map/[]map/struct/*struct/string, etc.
-	batch           int               // Batch number for batch Insert/Replace/Save operations.
-	filter          bool              // Filter data and where key-value pairs according to the fields of the table.
-	distinct        string            // Force the query to only return distinct results.
-	lockInfo        string            // Lock for update or in shared lock.
-	cacheEnabled    bool              // Enable sql result cache feature, which is mainly for indicating cache duration(especially 0) usage.
-	cacheOption     CacheOption       // Cache option for query statement.
-	pageCacheOption []CacheOption     // Cache option for paging query statement.
-	hookHandler     HookHandler       // Hook functions for model hook feature.
-	unscoped        bool              // Disables soft deleting features when select/delete operations.
-	safe            bool              // If true, it clones and returns a new model object whenever operation done; or else it changes the attribute of current model.
-	onDuplicate     any               // onDuplicate is used for on Upsert clause.
-	onDuplicateEx   any               // onDuplicateEx is used for excluding some columns on Upsert clause.
-	onConflict      any               // onConflict is used for conflict keys on Upsert clause.
-	tableAliasMap   map[string]string // Table alias to true table name, usually used in join statements.
-	softTimeOption  SoftTimeOption    // SoftTimeOption is the option to customize soft time feature for Model.
-	shardingConfig  ShardingConfig    // ShardingConfig for database/table sharding feature.
-	shardingValue   any               // Sharding value for sharding feature.
+	db              DB                        // Underlying DB interface.
+	tx              TX                        // Underlying TX interface.
+	rawSql          string                    // rawSql is the raw SQL string which marks a raw SQL based Model not a table based Model.
+	schema          string                    // Custom database schema.
+	linkType        int                       // Mark for operation on master or slave.
+	tablesInit      string                    // Table names when model initialization.
+	tables          string                    // Operation table names, which can be more than one table names and aliases, like: "user", "user u", "user u, user_detail ud".
+	fields          []any                     // Operation fields, multiple fields joined using char ','.
+	fieldsEx        []any                     // Excluded operation fields, it here uses slice instead of string type for quick filtering.
+	withArray       []any                     // Arguments for With feature.
+	withAll         bool                      // Enable model association operations on all objects that have "with" tag in the struct.
+	extraArgs       []any                     // Extra custom arguments for sql, which are prepended to the arguments before sql committed to underlying driver.
+	whereBuilder    *WhereBuilder             // Condition builder for where operation.
+	groupBy         string                    // Used for "group by" statement.
+	orderBy         string                    // Used for "order by" statement.
+	having          []any                     // Used for "having..." statement.
+	start           int                       // Used for "select ... start, limit ..." statement.
+	limit           int                       // Used for "select ... start, limit ..." statement.
+	option          int                       // Option for extra operation features.
+	offset          int                       // Offset statement for some databases grammar.
+	partition       string                    // Partition table partition name.
+	data            any                       // Data for operation, which can be type of map/[]map/struct/*struct/string, etc.
+	batch           int                       // Batch number for batch Insert/Replace/Save operations.
+	filter          bool                      // Filter data and where key-value pairs according to the fields of the table.
+	distinct        string                    // Force the query to only return distinct results.
+	lockInfo        string                    // Lock for update or in shared lock.
+	cacheEnabled    bool                      // Enable sql result cache feature, which is mainly for indicating cache duration(especially 0) usage.
+	cacheOption     CacheOption               // Cache option for query statement.
+	pageCacheOption []CacheOption             // Cache option for paging query statement.
+	hookHandler     HookHandler               // Hook functions for model hook feature.
+	unscoped        bool                      // Disables soft deleting features when select/delete operations.
+	safe            bool                      // If true, it clones and returns a new model object whenever operation done; or else it changes the attribute of current model.
+	onDuplicate     any                       // onDuplicate is used for on Upsert clause.
+	onDuplicateEx   any                       // onDuplicateEx is used for excluding some columns on Upsert clause.
+	onConflict      any                       // onConflict is used for conflict keys on Upsert clause.
+	tableAliasMap   map[string]string         // Table alias to true table name, usually used in join statements.
+	softTimeOption  SoftTimeOption            // SoftTimeOption is the option to customize soft time feature for Model.
+	shardingConfig  ShardingConfig            // ShardingConfig for database/table sharding feature.
+	shardingValue   any                       // Sharding value for sharding feature.
+	withOptions     map[ChunkName]*WithOption // withOptions is the batch association configuration indexed by chunkName.
+}
+
+// ChunkName is a type alias for chunk group identifier used in WithOptions.
+type ChunkName = string
+
+// WithOption is the configuration for batch association operations.
+type WithOption struct {
+	ChunkName    ChunkName // ChunkName is used to match the chunkName in the tag (for grouping fields).
+	ChunkSize    int       // ChunkSize is the size of each chunk (0 means no chunking, -1 means use default).
+	ChunkMinRows int       // ChunkMinRows is the minimum number of rows to trigger chunking (0 means always chunk, -1 means use default).
 }
 
 // ModelHandler is a function that handles given Model and returns a new Model that is custom modified.
@@ -303,6 +314,13 @@ func (m *Model) Clone() *Model {
 		newModel.having = make([]any, n)
 		copy(newModel.having, m.having)
 	}
+	if n := len(m.withOptions); n > 0 {
+		newModel.withOptions = make(map[ChunkName]*WithOption, n)
+		for k, v := range m.withOptions {
+			optCopy := *v
+			newModel.withOptions[k] = &optCopy
+		}
+	}
 	return newModel
 }
 

database/gdb/gdb_model_struct_cache.go

@@ -0,0 +1,85 @@
+// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://github.com/gogf/gf.
+
+package gdb
+
+import (
+	"reflect"
+
+	"github.com/gogf/gf/v2/container/gmap"
+	"github.com/gogf/gf/v2/os/gstructs"
+)
+
+// Global cache for struct information using gmap.KVMap.
+// This cache is shared by both single-struct and batch With operations.
+// Uses NewKVMapWithChecker to handle typed nil issue for pointer values.
+var (
+	structInfoChecker = func(v *modelStructCacheItem) bool { return v == nil }
+	structInfoCache   = gmap.NewKVMapWithChecker[reflect.Type, *modelStructCacheItem](structInfoChecker, true)
+)
+
+// modelStructCacheItem holds cached struct information.
+// Only caches static information (field metadata from gstructs).
+// Tag parsing is done dynamically to maintain flexibility.
+//
+// IMPORTANT: The Field.Value in cached fields is a zero-value instance.
+// Only use Field.Field (StructField), Field.Type(), Field.Name(), Field.Tag() etc.
+// DO NOT use Field.Value.Interface() to get actual values from real instances.
+type modelStructCacheItem struct {
+	fields []gstructs.Field // All fields from gstructs (static metadata)
+}
+
+// buildStructCacheItem creates a modelStructCacheItem from a struct type.
+// It extracts field information using gstructs and builds the fields slice.
+func buildStructCacheItem(structType reflect.Type) (*modelStructCacheItem, error) {
+	// Use gstructs to get field information
+	fieldMap, err := gstructs.FieldMap(gstructs.FieldMapInput{
+		Pointer:         reflect.New(structType).Interface(),
+		RecursiveOption: gstructs.RecursiveOptionEmbeddedNoTag,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Build cache item with fields slice
+	info := &modelStructCacheItem{
+		fields: make([]gstructs.Field, 0, len(fieldMap)),
+	}
+	for _, field := range fieldMap {
+		info.fields = append(info.fields, field)
+	}
+
+	return info, nil
+}
+
+// getCachedStructInfo gets or creates cached struct information.
+// It uses gmap.KVMap's GetOrSetFuncLock for thread-safe lazy initialization.
+// This function is used by both single-struct and batch With operations.
+func getCachedStructInfo(structType reflect.Type) (*modelStructCacheItem, error) {
+	// Use GetOrSetFuncLock to ensure thread-safe lazy initialization
+	// The function is only executed once per key, even under concurrent access
+	cached := structInfoCache.GetOrSetFuncLock(structType, func() *modelStructCacheItem {
+		info, err := buildStructCacheItem(structType)
+		if err != nil {
+			return nil // Return nil on error, will not be cached
+		}
+		return info
+	})
+
+	// Check if the cached value is nil (error case)
+	if cached == nil {
+		// Re-execute to get the actual error
+		return buildStructCacheItem(structType)
+	}
+
+	return cached, nil
+}
+
+// ClearModelStructCache clears the model struct cache.
+// This is usually not needed unless you're dynamically loading many struct types.
+func ClearModelStructCache() {
+	structInfoCache.Clear()
+}