python main.py <command> [pipeline_id] [options]功能: 分析原始数据和种子数据,生成处理蓝图
python main.py understand <pipeline_id> [--verbose]参数:
pipeline_id: Pipeline标识符--verbose: 详细输出模式
输出:
data/understanding_results/<pipeline_id>.json
示例:
python main.py understand text_sft_synthesis功能: 基于理解结果,智能编排数据处理pipeline
python main.py orchestrate <pipeline_id> [--verbose]参数:
pipeline_id: Pipeline标识符--verbose: 详细输出模式
输出:
data/orchestration_results/<pipeline_id>.json
示例:
python main.py orchestrate text_sft_synthesis功能: 基于编排结果,迭代生成可执行的算子代码
python main.py generate-operators <pipeline_id> [--verbose]参数:
pipeline_id: Pipeline标识符--verbose: 详细输出模式
输出:
data/generated_pipelines/<pipeline_id>/data/run_pipeline_results/<pipeline_id>/
示例:
python main.py generate-operators text_sft_synthesis功能: 运行生成的pipeline,获取处理结果
python main.py run-pipelines <pipeline_id> [--verbose]参数:
pipeline_id: Pipeline标识符--verbose: 详细输出模式
输出:
data/run_pipeline_results/<pipeline_id>/final_processed_data.jsonl
示例:
python main.py run-pipelines text_sft_synthesis功能: 运行完整的DataFlow+处理流程
python main.py run-full <pipeline_id> [--verbose]参数:
pipeline_id: Pipeline标识符--verbose: 详细输出模式
流程:
- 结构化理解
- Pipeline编排
- 算子生成
- Pipeline执行
示例:
python main.py run-full text_sft_synthesis功能: 查看系统或特定pipeline的状态
python main.py status [pipeline_id] [--verbose]参数:
pipeline_id: 可选,特定pipeline标识符--verbose: 详细输出模式
示例:
python main.py status # 查看系统状态
python main.py status text_sft_synthesis # 查看特定pipeline状态功能: 列出可用的数据集
python main.py list [--verbose]参数:
--verbose: 详细输出模式
示例:
python main.py list功能: 运行快速开始示例
python main.py quick-start [--verbose]参数:
--verbose: 详细输出模式
示例:
python main.py quick-start文件: core/config_manager.py
功能: 统一配置管理
class ConfigManager:
def __init__(self, config_file: str = "config/config.json")
def get(self, key: str, default: Any = None) -> Any
def set(self, key: str, value: Any) -> None
def load_api_keys(self) -> Dict[str, Any]示例:
from core.config_manager import ConfigManager
config = ConfigManager()
llm_config = config.get('llm', {})
api_keys = config.load_api_keys()文件: core/llm_service.py
功能: LLM服务封装
class LLMService:
def __init__(self, config: Dict[str, Any])
def call_llm(self, prompt: str, **kwargs) -> str
def call_llm_with_system_prompt(self, system_prompt: str, user_prompt: str, **kwargs) -> str示例:
from core.llm_service import LLMService
llm = LLMService(config)
response = llm.call_llm_with_system_prompt(
system_prompt="你是一个专业的文本分析专家",
user_prompt="请分析以下文本..."
)文件: core/path_manager.py
功能: 路径管理
class PathManager:
def __init__(self, base_path: str = None)
def get_data_path(self, subdir: str = None) -> Path
def get_run_pipeline_results_path(self) -> str
def get_generated_pipelines_path(self) -> str示例:
from core.path_manager import PathManager
path_manager = PathManager()
data_path = path_manager.get_data_path("understanding_results")
results_path = path_manager.get_run_pipeline_results_path()文件: core/logger.py
功能: 统一日志服务
class LoggerService:
def __init__(self, name: str = "dataflow_plus")
def info(self, message: str) -> None
def warning(self, message: str) -> None
def error(self, message: str) -> None
def debug(self, message: str) -> None示例:
from core.logger import LoggerService
logger = LoggerService("my_module")
logger.info("处理开始")
logger.error("处理失败")文件: subsystems/structured_understanding/understanding_analyzer.py
class UnderstandingAnalyzer:
def __init__(self, llm_service: LLMService, path_manager: PathManager)
def analyze_data(self, pipeline_id: str) -> Dict[str, Any]
def _analyze_file_formats(self, raw_data: List[Dict], seed_data: List[Dict]) -> Dict[str, Any]
def _analyze_quality_differences(self, raw_data: List[Dict], seed_data: List[Dict]) -> Dict[str, Any]文件: subsystems/pipeline_orchestration/main.py
class PipelineOrchestration:
def __init__(self, llm_service: LLMService, path_manager: PathManager)
def orchestrate_pipeline(self, pipeline_id: str) -> Dict[str, Any]
def _free_fitting_stage(self, understanding_result: Dict[str, Any]) -> Dict[str, Any]
def _template_combination_stage(self, free_fitting_result: Dict[str, Any]) -> Dict[str, Any]
def _constrained_search_stage(self, template_result: Dict[str, Any]) -> Dict[str, Any]文件: subsystems/operator_generation/generate_operators.py
class OperatorGeneration:
def __init__(self, config_manager: ConfigManager)
def generate_operators(self, orchestration_result: Dict[str, Any]) -> Dict[str, Any]文件: subsystems/operator_generation/pipeline_generator.py
class PipelineGenerator:
def __init__(self, llm_service: LLMService)
def generate_iterative_pipeline(self, steps: List[Dict], intermediate_results: List[Dict], step_index: int) -> str
def generate_final_pipeline(self, all_steps: List[Dict], all_intermediate_results: List[Dict]) -> str
def _operator_requires_llm(self, operator_name: str) -> bool
def _get_llm_task_config(self, operator_name: str) -> Dict[str, Any]文件: subsystems/operator_generation/pipeline_executor.py
class PipelineExecutor:
def __init__(self, run_results_dir: str)
def run_pipeline_and_get_result(self, pipeline_file: str, pipeline_id: str, step_number: str, step_dir: str = None, intermediate_results: List[Dict] = None) -> Dict[str, Any]
def run_final_pipeline_and_get_data(self, pipeline_file: str) -> List[Dict]文件: subsystems/operator_generation/result_manager.py
class ResultManager:
def __init__(self, run_results_dir: str, generated_pipelines_dir: str)
def save_step_result(self, step_dir: str, step_result: Dict[str, Any]) -> None
def save_pipeline_code(self, pipeline_dir: str, pipeline_id: str, pipeline_code: str) -> str
def save_final_data(self, pipeline_id: str, final_processed_data: List[Dict], understanding_result: Dict[str, Any]) -> str
def create_pipeline_directory(self, pipeline_id: str) -> str
def create_step_directory(self, pipeline_dir: str, step_number: int) -> str原始数据 (datasets/<pipeline_id>/raw_data/):
{
"difficulty": "Easy|Medium|Hard",
"instruction": "问题文本",
"output": "回答文本"
}种子数据 (datasets/<pipeline_id>/seed_data/):
{
"difficulty": "Easy|Medium|Hard",
"instruction": "问题文本",
"output": "高质量回答文本",
"AlpagasusScore": 4.0-5.0
}处理结果 (data/run_pipeline_results/<pipeline_id>/final_processed_data.jsonl):
{
"difficulty": "Easy|Medium|Hard",
"instruction": "问题文本",
"output": "Enhanced: 增强后的高质量回答",
"AlpagasusScore": 4.0-5.0
}结构化理解结果 (data/understanding_results/<pipeline_id>.json):
{
"pipeline_id": "text_sft_synthesis",
"basic_information": {
"文件格式分析": {...},
"质量差异分析": {...},
"处理目标": {...}
},
"field_analysis": {
"key_fields": [...],
"field_mappings": {...}
}
}Pipeline编排结果 (data/orchestration_results/<pipeline_id>.json):
{
"pipeline_id": "text_sft_synthesis",
"free_fitting": {...},
"template_combination": {...},
"constrained_search": {
"optimized_pipeline": [
{
"step_id": "step_1",
"operator": "io.read_jsonl_file",
"input_keys": [...],
"output_keys": [...],
"parameters": {...}
}
]
}
}文件: config/config.json
{
"llm": {
"provider": "custom",
"model": "gpt-3.5-turbo",
"temperature": 0.7,
"max_tokens": 1000,
"timeout": 30
},
"pipeline": {
"max_steps": 10,
"timeout": 300,
"parallel_execution": false
},
"logging": {
"level": "INFO",
"file": "logs/dataflow_plus.log",
"max_size": "10MB",
"backup_count": 5
}
}文件: config/api_keys.json
{
"openai": {
"api_key": "sk-...",
"base_url": "https://api.openai.com/v1"
},
"custom": {
"api_key": "your-api-key",
"base_url": "https://your-api-endpoint.com/v1"
}
}# 1. 在算子注册表中注册
# data/operator_registry.json
{
"operators": {
"custom": {
"my_operator": {
"description": "我的自定义算子",
"input_keys": ["input"],
"output_keys": ["output"],
"parameters": {
"param1": {"type": "string", "default": "default"}
},
"category": "processing",
"requires_llm": false,
"dependencies": []
}
}
}
}
# 2. 实现算子函数
def my_operator(data: List[Dict], param1: str = "default") -> List[Dict]:
"""custom.my_operator - 我的自定义算子"""
# 实现逻辑
return processed_data# 1. 在算子注册表中标记LLM支持
{
"requires_llm": true,
"llm_task": {
"system_prompt": "你是一个专业的...",
"user_prompt_template": "请处理:{data}",
"expected_output_format": "json",
"response_fields": ["result"]
}
}
# 2. 在pipeline生成器中实现LLM调用
def my_llm_operator(data: List[Dict]) -> List[Dict]:
"""custom.my_llm_operator - LLM增强算子"""
enhanced_data = []
for item in data:
# LLM调用逻辑
enhanced_item = call_llm_api(system_prompt, user_prompt)
enhanced_data.append(enhanced_item)
return enhanced_dataclass DataFlowPlusError(Exception):
"""DataFlow+基础异常"""
pass
class ConfigurationError(DataFlowPlusError):
"""配置错误"""
pass
class LLMServiceError(DataFlowPlusError):
"""LLM服务错误"""
pass
class PipelineExecutionError(DataFlowPlusError):
"""Pipeline执行错误"""
passtry:
result = llm_service.call_llm(prompt)
except LLMServiceError as e:
logger.error(f"LLM调用失败: {e}")
# 使用fallback逻辑
result = fallback_function()class PerformanceMonitor:
def __init__(self):
self.metrics = {}
def start_timer(self, operation: str) -> None:
"""开始计时"""
pass
def end_timer(self, operation: str) -> float:
"""结束计时,返回耗时"""
pass
def record_metric(self, name: str, value: float) -> None:
"""记录性能指标"""
passmonitor = PerformanceMonitor()
monitor.start_timer("pipeline_execution")
# 执行pipeline
execution_time = monitor.end_timer("pipeline_execution")
monitor.record_metric("total_execution_time", execution_time)