IterativePipeline runs feature engineering across multiple rounds. Each round: 1. The router selects agents 2. Agent memory provides context from past rounds 3. CorePipeline generates and evaluates features 4. Memory is updated with results 5. Router performance is updated
This notebook demonstrates the full iterative cycle and the 3-tier memory system.
import os
import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
print("Iterative Pipeline & Memory Demo")Iterative Pipeline & Memory DemoX, y = make_classification(
n_samples=300, n_features=7, n_informative=4, random_state=42
)
df = pd.DataFrame(X, columns=[f"var_{i+1}" for i in range(X.shape[1])])
df["target"] = y
X_train, X_test, y_train, y_test = train_test_split(
df.drop(columns=["target"]), df["target"],
test_size=0.3, random_state=42, stratify=df["target"]
)
print(f"Train: {X_train.shape}, Test: {X_test.shape}")
X_train.head()Train: (210, 7), Test: (90, 7)| var_1 | var_2 | var_3 | var_4 | var_5 | var_6 | var_7 | |
|---|---|---|---|---|---|---|---|
| 183 | -1.830494 | 2.591218 | -1.610339 | -1.656433 | 0.921472 | -1.599557 | -2.297226 |
| 284 | 0.543193 | 0.133050 | -0.510729 | -0.045881 | 0.161825 | -1.168057 | 1.797557 |
| 128 | -1.782045 | -0.558475 | -1.248607 | -0.522402 | 0.316726 | 0.545798 | -0.789814 |
| 135 | 0.160007 | 0.871449 | -1.761100 | 1.521330 | -0.232812 | -1.209466 | -0.411545 |
| 15 | 0.534833 | 0.017506 | -0.023910 | -0.197430 | -0.089851 | -0.390762 | 0.701202 |
from feature_forge.data.registry import DatasetRegistry
registry = DatasetRegistry()
print(f"Available datasets: {registry.list()}")
try:
ds = registry.load("titanic")
titanic_train = ds["train"]
target_col = ds["target"]
print(f"Titanic train: {titanic_train.shape}, target: {target_col}")
titanic_train.head()
except Exception as exc:
print(f"Titanic dataset unavailable (needs Kaggle API): {exc}")
titanic_train = NoneAvailable datasets: ['titanic', 'house_prices']
Titanic dataset unavailable (needs Kaggle API): kaggle package not installed. Run: uv pip install kagglefrom feature_forge.config import LLMConfig, Settings
from feature_forge.pipeline.iterative import IterativePipeline
from feature_forge.llm.providers.deepseek import DeepSeekProvider
config = Settings(
task="classification",
metric="auc",
n_rounds=2,
llm=LLMConfig(
model="deepseek-chat",
api_key=os.environ.get("FF_LLM__API_KEY", ""),
max_concurrent_calls=2,
),
)
llm = DeepSeekProvider(
model=config.llm.model,
api_key=config.llm.api_key.get_secret_value() if config.llm.api_key else None,
)
pipeline = IterativePipeline(config=config, llm_client=llm)
result = {}
try:
result = await pipeline.run(X_train, y_train, X_test, task_description="binary classification")
print(f"Pipeline complete: {len(result.get('selected_features', []))} features selected")
except Exception as exc:
print(f"Pipeline error: {exc}"){"n_rounds": 2, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:05:45.585805Z", "span": null}
{"round_idx": 0, "total_rounds": 2, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:05:45.586579Z", "span": null}
{"strategy": "hybrid", "round_idx": 0, "selected_agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "event": "router_select_agents", "level": "info", "timestamp": "2026-05-11T13:05:45.586925Z", "span": null}
{"round_idx": 0, "agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:05:45.588940Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.589813Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.590234Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.591166Z", "span": null}
{"agent": "cross_compositional", "path": "memory_files/agent_memories/cross_compositional_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.591475Z", "span": null}
{"path": "memory_files/agent_memories/aggregation_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.592161Z", "span": null}
{"agent": "aggregation", "path": "memory_files/agent_memories/aggregation_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.592517Z", "span": null}
{"path": "memory_files/agent_memories/temporal_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.593110Z", "span": null}
{"agent": "temporal", "path": "memory_files/agent_memories/temporal_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.593480Z", "span": null}
{"path": "memory_files/agent_memories/local_transform_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.593918Z", "span": null}
{"agent": "local_transform", "path": "memory_files/agent_memories/local_transform_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.594180Z", "span": null}
{"path": "memory_files/agent_memories/local_pattern_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:45.594876Z", "span": null}
{"agent": "local_pattern", "path": "memory_files/agent_memories/local_pattern_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:05:45.595048Z", "span": null}
{"agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "num_agents": 6, "train_shape": [210, 7], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:05:45.595233Z", "span": null}
{"agent": "unary", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.596641Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.596849Z", "span": null}
{"agent": "cross_compositional", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.674254Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.674607Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:45.806240Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:45.806566Z", "span": null}
{"agent": "aggregation", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.807533Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.807773Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:45.809663Z", "span": null}
{"agent": "cross_compositional", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:45.810093Z", "span": null}
{"agent": "temporal", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.811173Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.811398Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:45.923916Z", "span": null}
{"agent": "temporal", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:45.924270Z", "span": null}
{"agent": "local_transform", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.925272Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.925474Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:45.987970Z", "span": null}
{"agent": "aggregation", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:45.988326Z", "span": null}
{"agent": "local_pattern", "num_columns": 7, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:45.989289Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:45.989532Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.061938Z", "span": null}
{"agent": "local_transform", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.062235Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.172307Z", "span": null}
{"agent": "local_pattern", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.172659Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.173205Z", "span": null}
{"agent": "cross_compositional", "error": "cross_compositional LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.173399Z", "span": null}
{"agent": "aggregation", "error": "aggregation LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.173627Z", "span": null}
{"agent": "temporal", "error": "temporal LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.173884Z", "span": null}
{"agent": "local_transform", "error": "local_transform LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.174136Z", "span": null}
{"agent": "local_pattern", "error": "local_pattern LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.174751Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:05:46.174943Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.176923Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.178222Z", "span": null}
{"path": "memory_files/agent_memories/aggregation_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.179295Z", "span": null}
{"path": "memory_files/agent_memories/temporal_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.180232Z", "span": null}
{"path": "memory_files/agent_memories/local_transform_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.181812Z", "span": null}
{"path": "memory_files/agent_memories/local_pattern_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.183189Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 597.0, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:05:46.183538Z", "span": null}
{"round_idx": 1, "total_rounds": 2, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:05:46.183756Z", "span": null}
{"strategy": "hybrid", "round_idx": 1, "selected_agents": ["cross_compositional", "unary", "local_pattern", "aggregation", "local_transform", "temporal"], "event": "router_select_agents", "level": "info", "timestamp": "2026-05-11T13:05:46.183958Z", "span": null}
{"round_idx": 1, "agents": ["cross_compositional", "unary", "local_pattern", "aggregation", "local_transform", "temporal"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:05:46.185389Z", "span": null}
{"agents": ["cross_compositional", "unary", "local_pattern", "aggregation", "local_transform", "temporal"], "num_agents": 6, "train_shape": [210, 7], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:05:46.185656Z", "span": null}
{"agent": "cross_compositional", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.186471Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.186700Z", "span": null}
{"agent": "unary", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.188404Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.188558Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.295397Z", "span": null}
{"agent": "cross_compositional", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.295783Z", "span": null}
{"agent": "local_pattern", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.297115Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.297395Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.299399Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.299925Z", "span": null}
{"agent": "aggregation", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.301573Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.301934Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.409184Z", "span": null}
{"agent": "aggregation", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.409638Z", "span": null}
{"agent": "local_transform", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.411536Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.411760Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.420624Z", "span": null}
{"agent": "local_pattern", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.421132Z", "span": null}
{"agent": "temporal", "num_columns": 7, "round_idx": 1, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:05:46.422288Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.422614Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.512072Z", "span": null}
{"agent": "local_transform", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.512439Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.549872Z", "span": null}
{"agent": "temporal", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:05:46.550142Z", "span": null}
{"agent": "cross_compositional", "error": "cross_compositional LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.550469Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.550614Z", "span": null}
{"agent": "local_pattern", "error": "local_pattern LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.550756Z", "span": null}
{"agent": "aggregation", "error": "aggregation LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.550880Z", "span": null}
{"agent": "local_transform", "error": "local_transform LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.551109Z", "span": null}
{"agent": "temporal", "error": "temporal LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:05:46.551262Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:05:46.551428Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.552594Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.553736Z", "span": null}
{"path": "memory_files/agent_memories/local_pattern_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.555211Z", "span": null}
{"path": "memory_files/agent_memories/aggregation_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.556041Z", "span": null}
{"path": "memory_files/agent_memories/local_transform_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.557500Z", "span": null}
{"path": "memory_files/agent_memories/temporal_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.558416Z", "span": null}
{"round_idx": 1, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 375.1, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:05:46.558875Z", "span": null}
{"total_rounds": 2, "total_features": 0, "latency_ms": 973.3, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:05:46.559083Z", "span": null}
Pipeline complete: 0 features selectedprint(f"Selected features: {result.get('selected_features', [])}")
print(f"Round summaries: {len(result.get('round_summaries', []))}")
for i, summary in enumerate(result.get("round_summaries", [])):
print(f"\n--- Round {i} ---")
print(f" Agents: {summary.get('agents', [])}")
print(f" Features generated: {summary.get('num_features_generated', 0)}")
print(f" Features selected: {summary.get('num_features_selected', 0)}")
print(f" Baseline: {summary.get('baseline_score', 0):.4f}")Selected features: []
Round summaries: 2
--- Round 0 ---
Agents: ['unary', 'cross_compositional', 'aggregation', 'temporal', 'local_transform', 'local_pattern']
Features generated: 0
Features selected: 0
Baseline: 0.0000
--- Round 1 ---
Agents: ['cross_compositional', 'unary', 'local_pattern', 'aggregation', 'local_transform', 'temporal']
Features generated: 0
Features selected: 0
Baseline: 0.0000from feature_forge.memory.base import AgentMemory
from feature_forge.memory.conceptual import ConceptualMemory
# Per-agent memory
mem = AgentMemory(agent_name="unary", memory_path="/tmp/test_unary_memory.json")
mem.record_procedure(
base_columns=["var_1"],
transform="log1p",
feature_name="log_var_1",
ty="numerical",
description="log1p transform of var_1",
round_idx=0,
)
mem.record_feedback(
feature_name="log_var_1",
metric="auc",
value=0.03,
effective=True,
round_idx=0,
base=["var_1"],
ty="numerical",
)
mem.save()
print("Procedural memory:")
print(mem.procedural)
print("\nFeedback memory:")
print(mem.feedback)
print("\nStats:")
print(mem.stats){"path": "/tmp/test_unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:05:46.569147Z", "span": null}
{"path": "/tmp/test_unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:05:46.569739Z", "span": null}
Procedural memory:
[{'base_columns': ['var_1'], 'transform': 'log1p', 'feature_name': 'log_var_1', 'type': 'numerical', 'description': 'log1p transform of var_1', 'agent_name': 'unary', 'round_idx': 0}]
Feedback memory:
[{'feature_name': 'log_var_1', 'metric': 'auc', 'value': 0.03, 'effective': True, 'round_idx': 0, 'agent_name': 'unary', 'base_columns': ['var_1'], 'type': 'numerical'}]
Stats:
{}conceptual = ConceptualMemory(llm)
summary = await conceptual.summarize_agent(mem)
print(f"Conceptual summary: {summary}")
# Global summary across all agents
global_summary = await conceptual.summarize_global({"unary": mem})
print(f"Global summary: {global_summary}"){"agent": "unary", "num_effective": 1, "event": "conceptual_summarize_start", "level": "info", "timestamp": "2026-05-11T13:05:46.574057Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.6, "max_tokens": 1024, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:05:46.574333Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:05:46.721974Z", "span": null}--------------------------------------------------------------------------- AuthenticationError Traceback (most recent call last) File ~/Desktop/personal/feature_forge/src/feature_forge/llm/providers/openai.py:59, in OpenAIProvider.complete(self, messages, temperature, max_tokens, **kwargs) 58 try: ---> 59 response = await self._client.chat.completions.create( 60 model=self.model, 61 messages=messages, # type: ignore[arg-type] 62 temperature=temperature, 63 max_tokens=max_tokens, 64 **kwargs, 65 ) 66 except Exception as exc: File ~/Desktop/personal/feature_forge/.venv/lib/python3.13/site-packages/openai/resources/chat/completions/completions.py:2700, in AsyncCompletions.create(self, messages, model, audio, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, metadata, modalities, n, parallel_tool_calls, prediction, presence_penalty, prompt_cache_key, prompt_cache_retention, reasoning_effort, response_format, safety_identifier, seed, service_tier, stop, store, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, verbosity, web_search_options, extra_headers, extra_query, extra_body, timeout) 2699 validate_response_format(response_format) -> 2700 return await self._post( 2701 "/chat/completions", 2702 body=await async_maybe_transform( 2703 { 2704 "messages": messages, 2705 "model": model, 2706 "audio": audio, 2707 "frequency_penalty": frequency_penalty, 2708 "function_call": function_call, 2709 "functions": functions, 2710 "logit_bias": logit_bias, 2711 "logprobs": logprobs, 2712 "max_completion_tokens": max_completion_tokens, 2713 "max_tokens": max_tokens, 2714 "metadata": metadata, 2715 "modalities": modalities, 2716 "n": n, 2717 "parallel_tool_calls": parallel_tool_calls, 2718 "prediction": prediction, 2719 "presence_penalty": presence_penalty, 2720 "prompt_cache_key": prompt_cache_key, 2721 "prompt_cache_retention": prompt_cache_retention, 2722 "reasoning_effort": reasoning_effort, 2723 "response_format": response_format, 2724 "safety_identifier": safety_identifier, 2725 "seed": seed, 2726 "service_tier": service_tier, 2727 "stop": stop, 2728 "store": store, 2729 "stream": stream, 2730 "stream_options": stream_options, 2731 "temperature": temperature, 2732 "tool_choice": tool_choice, 2733 "tools": tools, 2734 "top_logprobs": top_logprobs, 2735 "top_p": top_p, 2736 "user": user, 2737 "verbosity": verbosity, 2738 "web_search_options": web_search_options, 2739 }, 2740 completion_create_params.CompletionCreateParamsStreaming 2741 if stream 2742 else completion_create_params.CompletionCreateParamsNonStreaming, 2743 ), 2744 options=make_request_options( 2745 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout 2746 ), 2747 cast_to=ChatCompletion, 2748 stream=stream or False, 2749 stream_cls=AsyncStream[ChatCompletionChunk], 2750 ) File ~/Desktop/personal/feature_forge/.venv/lib/python3.13/site-packages/openai/_base_client.py:1884, in AsyncAPIClient.post(self, path, cast_to, body, content, files, options, stream, stream_cls) 1881 opts = FinalRequestOptions.construct( 1882 method="post", url=path, json_data=body, content=content, files=await async_to_httpx_files(files), **options 1883 ) -> 1884 return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls) File ~/Desktop/personal/feature_forge/.venv/lib/python3.13/site-packages/openai/_base_client.py:1669, in AsyncAPIClient.request(self, cast_to, options, stream, stream_cls) 1668 log.debug("Re-raising status error") -> 1669 raise self._make_status_error_from_response(err.response) from None 1671 break AuthenticationError: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}} The above exception was the direct cause of the following exception: LLMError Traceback (most recent call last) Cell In[7], line 2 1 conceptual = ConceptualMemory(llm) ----> 2 summary = await conceptual.summarize_agent(mem) 3 print(f"Conceptual summary: {summary}") 4 5 # Global summary across all agents File ~/Desktop/personal/feature_forge/src/feature_forge/memory/conceptual.py:83, in ConceptualMemory.summarize_agent(self, memory, min_effective) 69 system_prompt = ( 70 f"You are {memory.agent_name} agent, an expert feature engineering assistant. " 71 "You will receive a list of effective features and statistics about their patterns. " (...) 74 "Avoid any irrelevant information." 75 ) 76 user_prompt = ( 77 f"Here are the effective feature examples:\n\n{examples_text}\n\n" 78 f"Here are the statistics about effective features:\n\n{stats_text}\n\n" 79 "Based on both the examples and the statistics, summarize 1 to 3 concise and actionable " 80 "conceptual rules to optimize future feature generation. Rules should be in clear bullet points." 81 ) ---> 83 response = await self.llm_client.complete( 84 messages=[ 85 {"role": "system", "content": system_prompt}, 86 {"role": "user", "content": user_prompt}, 87 ], 88 temperature=0.6, 89 max_tokens=1024, 90 ) 91 memory.conceptual_summary = response.content 92 memory.record_conceptual(memory.conceptual_summary) File ~/Desktop/personal/feature_forge/src/feature_forge/llm/providers/deepseek.py:127, in DeepSeekProvider.complete(self, messages, temperature, max_tokens, **kwargs) 119 async def complete( 120 self, 121 messages: list[dict[str, str]], (...) 124 **kwargs: Any, 125 ) -> LLMResponse: 126 """Standard completion via DeepSeek's OpenAI-compatible API.""" --> 127 return await super().complete( 128 messages=messages, 129 temperature=temperature, 130 max_tokens=max_tokens, 131 **kwargs, 132 ) File ~/Desktop/personal/feature_forge/src/feature_forge/llm/providers/openai.py:68, in OpenAIProvider.complete(self, messages, temperature, max_tokens, **kwargs) 66 except Exception as exc: 67 logger.error("llm_error", provider=self.provider_name, model=self.model, error=str(exc)) ---> 68 raise LLMError(f"OpenAI API error: {exc}") from exc 70 choice = response.choices[0] 71 content = choice.message.content or "" LLMError: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}
agent_gains = result.get("agent_gains", {})
if agent_gains:
for agent_name, gains_list in agent_gains.items():
print(f"{agent_name}: {len(gains_list)} gain records")unary: 2 gain records
cross_compositional: 2 gain records
aggregation: 2 gain records
temporal: 2 gain records
local_transform: 2 gain records
local_pattern: 2 gain recordsimport matplotlib.pyplot as plt
rounds = result.get("round_summaries", [])
if rounds:
df_rounds = pd.DataFrame([
{
"round": i,
"generated": r.get("num_features_generated", 0),
"selected": r.get("num_features_selected", 0),
"baseline": r.get("baseline_score", 0),
}
for i, r in enumerate(rounds)
])
fig, ax1 = plt.subplots(figsize=(8, 4))
x = np.arange(len(df_rounds))
width = 0.35
ax1.bar(x - width/2, df_rounds["generated"], width, color="steelblue", label="Generated")
ax1.bar(x + width/2, df_rounds["selected"], width, color="coral", label="Selected")
ax1.set_xlabel("Round")
ax1.set_ylabel("Feature Count")
ax1.set_xticks(x)
ax1.set_xticklabels(df_rounds["round"])
plt.title("Iterative Pipeline: Features per Round")
plt.legend()
plt.tight_layout()
plt.show()
IterativePipeline orchestrates N rounds of generation, evaluation, and memory updates