Artifacts & Dashboard

Unified artifact access, comparison, diff, and offline HTML dashboards.

Published

May 11, 2026

Introduction

Every feature engineering method in Feature Forge implements ArtifactExporter, providing a uniform interface to scripts, DataFrames, and metadata. This notebook demonstrates artifact schemas, comparison, diff, dashboard generation, and cross-method gain analysis.

Setup

Code

import os
import warnings
warnings.filterwarnings("ignore")

from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification

print("Artifacts & Dashboard Demo")

Artifacts & Dashboard Demo

Load Data

Code

X, y = make_classification(
    n_samples=200, n_features=5, n_informative=3, random_state=42
)
df = pd.DataFrame(X, columns=[f"c{i+1}" for i in range(X.shape[1])])
df["target"] = y

X_train = df.drop(columns=["target"])
y_train = df["target"]

ArtifactConfig

Code

from feature_forge.artifacts import ArtifactConfig

mem_cfg = ArtifactConfig(storage_mode="memory")
print(f"Memory config: {mem_cfg}")

disk_cfg = ArtifactConfig(storage_mode="disk", storage_format="parquet")
print(f"Disk config: {disk_cfg}")

Memory config: ArtifactConfig(storage_mode='memory', storage_format='parquet', spill_threshold_bytes=52428800, storage_dir='.feature_forge_artifacts')
Disk config: ArtifactConfig(storage_mode='disk', storage_format='parquet', spill_threshold_bytes=52428800, storage_dir='.feature_forge_artifacts')

ArtifactExporter Interface

Code

from feature_forge.api import MALMASFeatureEngineer
from feature_forge.config import LLMConfig, Settings

config = Settings(
    task="classification",
    n_rounds=1,
    llm=LLMConfig(
        model="deepseek-chat",
        api_key=os.environ.get("FF_LLM__API_KEY", ""),
    ),
)

fe = MALMASFeatureEngineer(config=config, artifact_config=ArtifactConfig())
fe.fit(X_train, y_train)

print(f"Generated scripts: {len(fe.generated_scripts)}")
print(f"Feature metadata records: {len(fe.feature_metadata)}")
print(f"Provenance records: {len(fe.provenance_records)}")

{"mode": "full", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:29.142382Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.143824Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:29.144334Z", "span": null}
{"strategy": "hybrid", "round_idx": 0, "selected_agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "event": "router_select_agents", "level": "info", "timestamp": "2026-05-11T13:06:29.144593Z", "span": null}
{"round_idx": 0, "agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:29.145987Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.146519Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.146793Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.147249Z", "span": null}
{"agent": "cross_compositional", "path": "memory_files/agent_memories/cross_compositional_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.147539Z", "span": null}
{"path": "memory_files/agent_memories/aggregation_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.148101Z", "span": null}
{"agent": "aggregation", "path": "memory_files/agent_memories/aggregation_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.148260Z", "span": null}
{"path": "memory_files/agent_memories/temporal_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.148647Z", "span": null}
{"agent": "temporal", "path": "memory_files/agent_memories/temporal_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.148889Z", "span": null}
{"path": "memory_files/agent_memories/local_transform_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.149525Z", "span": null}
{"agent": "local_transform", "path": "memory_files/agent_memories/local_transform_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.149686Z", "span": null}
{"path": "memory_files/agent_memories/local_pattern_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.150287Z", "span": null}
{"agent": "local_pattern", "path": "memory_files/agent_memories/local_pattern_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.150510Z", "span": null}
{"agents": ["unary", "cross_compositional", "aggregation", "temporal", "local_transform", "local_pattern"], "num_agents": 6, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.150705Z", "span": null}
{"agent": "unary", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.152292Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.152544Z", "span": null}
{"agent": "cross_compositional", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.242699Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.243159Z", "span": null}
{"agent": "aggregation", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.244442Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.244727Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.384786Z", "span": null}
{"agent": "aggregation", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.385207Z", "span": null}
{"agent": "temporal", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.386169Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.386466Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.388260Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.388540Z", "span": null}
{"agent": "local_transform", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.389448Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.389620Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.391108Z", "span": null}
{"agent": "cross_compositional", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.391275Z", "span": null}
{"agent": "local_pattern", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.392313Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.392508Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.509102Z", "span": null}
{"agent": "temporal", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.509398Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.509995Z", "span": null}
{"agent": "local_pattern", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.510343Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.511943Z", "span": null}
{"agent": "local_transform", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.512183Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.512663Z", "span": null}
{"agent": "cross_compositional", "error": "cross_compositional LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.512971Z", "span": null}
{"agent": "aggregation", "error": "aggregation LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.513466Z", "span": null}
{"agent": "temporal", "error": "temporal LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.513807Z", "span": null}
{"agent": "local_transform", "error": "local_transform LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.514215Z", "span": null}
{"agent": "local_pattern", "error": "local_pattern LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.514532Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.514763Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.516924Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.518403Z", "span": null}
{"path": "memory_files/agent_memories/aggregation_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.519781Z", "span": null}
{"path": "memory_files/agent_memories/temporal_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.520955Z", "span": null}
{"path": "memory_files/agent_memories/local_transform_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.522986Z", "span": null}
{"path": "memory_files/agent_memories/local_pattern_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.524845Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 381.1, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.525442Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 381.8, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.525636Z", "span": null}
{"num_selected_features": 0, "latency_ms": 383.8, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.526123Z", "span": null}
Generated scripts: 0
Feature metadata records: 0
Provenance records: 0

Schema

Code

from feature_forge.artifacts.schema import ArtifactBundle, FeatureMetadata, ProvenanceRecord

meta = FeatureMetadata(
    name="test_feat",
    method="malmas",
    agent="unary",
    code="df['test'] = df['c1'] * 2",
)
print(f"FeatureMetadata: {meta}")

prov = ProvenanceRecord(
    feature_name="test_feat",
    source_method="malmas",
    source_agent="unary",
    round_index=0,
)
print(f"ProvenanceRecord: {prov}")

FeatureMetadata: name='test_feat' method='malmas' round=None agent='unary' iteration=None code="df['test'] = df['c1'] * 2" gain=None importance=None provenance={}
ProvenanceRecord: feature_name='test_feat' source_method='malmas' source_agent='unary' round_index=0 iteration_index=None prompt=None generated_code=None cv_gain=None timestamp='2026-05-11T13:06:29.532020+00:00'

Artifact Diff

Code

from feature_forge.artifacts.diff import ArtifactDiff

bundle_a = ArtifactBundle(
    method_name="method_a",
    generated_scripts=["code_a", "code_b"],
    feature_metadata=[meta],
)
bundle_b = ArtifactBundle(
    method_name="method_b",
    generated_scripts=["code_b", "code_c"],
    feature_metadata=[meta],
)

diff = ArtifactDiff({"a": bundle_a, "b": bundle_b})
print(f"Summary: {diff.summary()}")
print("\nOverlap matrix:")
print(diff.overlap_matrix())

Summary: {'total_unique_features': 1, 'shared_across_all': 1, 'shared_feature_names': ['test_feat'], 'per_method': {'a': {'total_features': 1, 'unique_features': 0, 'unique_feature_names': [], 'mean_gain': None, 'max_gain': None}, 'b': {'total_features': 1, 'unique_features': 0, 'unique_feature_names': [], 'mean_gain': None, 'max_gain': None}}}

Overlap matrix:
           a  b
feature        
test_feat  1  1

Comparison Utility

Code

from feature_forge.artifacts import compare_methods

fe_a = MALMASFeatureEngineer(config=config, mode="unary")
fe_b = MALMASFeatureEngineer(config=config, mode="cross_compositional")

try:
    fe_a.fit(X_train, y_train)
    fe_b.fit(X_train, y_train)

    comparison = compare_methods(
        methods={"unary": fe_a, "cross": fe_b},
        X_train=X_train,
        y_train=y_train,
    )
    for name, artifacts in comparison.items():
        if "error" not in artifacts:
            print(f"{name}: {len(artifacts.get('feature_codes', []))} codes")
except Exception as exc:
    print(f"Comparison skipped: {exc}")

{"mode": "unary", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:29.569849Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.571409Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:29.571840Z", "span": null}
{"round_idx": 0, "agents": ["unary"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:29.572388Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.573455Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.573826Z", "span": null}
{"agents": ["unary"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.574389Z", "span": null}
{"agent": "unary", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.575748Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.576222Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.708264Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.708613Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.709047Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.709288Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.711262Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 140.3, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.712136Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 141.0, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.712463Z", "span": null}
{"num_selected_features": 0, "latency_ms": 143.3, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.713133Z", "span": null}
{"mode": "cross_compositional", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:29.713561Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.714855Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:29.715398Z", "span": null}
{"round_idx": 0, "agents": ["cross_compositional"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:29.716187Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.716918Z", "span": null}
{"agent": "cross_compositional", "path": "memory_files/agent_memories/cross_compositional_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.717247Z", "span": null}
{"agents": ["cross_compositional"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.717604Z", "span": null}
{"agent": "cross_compositional", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.718715Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.719079Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:29.923056Z", "span": null}
{"agent": "cross_compositional", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:29.923431Z", "span": null}
{"agent": "cross_compositional", "error": "cross_compositional LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:29.923936Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.924124Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:29.925688Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 210.8, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.926241Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 211.8, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.926644Z", "span": null}
{"num_selected_features": 0, "latency_ms": 213.6, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:29.927097Z", "span": null}
{"mode": "unary", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:29.927463Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.928518Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:29.928774Z", "span": null}
{"round_idx": 0, "agents": ["unary"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:29.929252Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:29.929907Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:29.930152Z", "span": null}
{"agents": ["unary"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:29.930381Z", "span": null}
{"agent": "unary", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:29.931493Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:29.931707Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:30.034187Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:30.034575Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:30.035118Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.035320Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:30.036959Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 108.7, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.037446Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 109.3, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.037826Z", "span": null}
{"num_selected_features": 0, "latency_ms": 110.8, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.038214Z", "span": null}
{"mode": "cross_compositional", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:30.038503Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.039425Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:30.039623Z", "span": null}
{"round_idx": 0, "agents": ["cross_compositional"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:30.039936Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:30.040584Z", "span": null}
{"agent": "cross_compositional", "path": "memory_files/agent_memories/cross_compositional_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:30.040950Z", "span": null}
{"agents": ["cross_compositional"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.041220Z", "span": null}
{"agent": "cross_compositional", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:30.042095Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:30.042325Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:30.150675Z", "span": null}
{"agent": "cross_compositional", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:30.150984Z", "span": null}
{"agent": "cross_compositional", "error": "cross_compositional LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:30.151281Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.151531Z", "span": null}
{"path": "memory_files/agent_memories/cross_compositional_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:30.152976Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 114.0, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.153607Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 114.4, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.153852Z", "span": null}
{"num_selected_features": 0, "latency_ms": 115.8, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.154345Z", "span": null}
unary: 0 codes
cross: 0 codes

Dashboard Generation

Code

from feature_forge.artifacts import ArtifactDashboard

bundles = {}
for name in ["a", "b"]:
    try:
        fe_mode = MALMASFeatureEngineer(config=config, mode="unary")
        fe_mode.fit(X_train, y_train)
        arts = fe_mode.get_artifacts()
        bundles[name] = ArtifactBundle(
            method_name=name,
            generated_scripts=arts.get("feature_codes", []),
            provenance_records=arts.get("provenance", []),
        )
    except Exception as exc:
        print(f"Skipping {name}: {exc}")

if bundles:
    dash = ArtifactDashboard(bundles)
    report_path = "/tmp/ff_artifact_report.html"
    dash.save(report_path)
    print(f"Dashboard: {report_path}")
    html = Path(report_path).read_text()
    print(f"HTML size: {len(html):,} bytes")
else:
    print("No bundles for dashboard")

{"mode": "unary", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:30.171642Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.172507Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:30.172769Z", "span": null}
{"round_idx": 0, "agents": ["unary"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:30.173376Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:30.173944Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:30.174478Z", "span": null}
{"agents": ["unary"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.174674Z", "span": null}
{"agent": "unary", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:30.175544Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:30.175808Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:30.409032Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:30.409400Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:30.409730Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.409875Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:30.411298Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 238.9, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.411709Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 239.4, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.411872Z", "span": null}
{"num_selected_features": 0, "latency_ms": 240.5, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.412125Z", "span": null}
{"mode": "unary", "model": "deepseek-chat", "train_shape": [200, 5], "n_rounds": 1, "event": "fit_start", "level": "info", "timestamp": "2026-05-11T13:06:30.423235Z", "span": null}
{"n_rounds": 1, "task": "classification", "strategy": "hybrid", "event": "iterative_pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.423980Z", "span": null}
{"round_idx": 0, "total_rounds": 1, "event": "round_start", "level": "info", "timestamp": "2026-05-11T13:06:30.424192Z", "span": null}
{"round_idx": 0, "agents": ["unary"], "strategy": "hybrid", "event": "agents_selected", "level": "info", "timestamp": "2026-05-11T13:06:30.424455Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "exists": true, "num_keys": 7, "event": "memory_load", "level": "debug", "timestamp": "2026-05-11T13:06:30.424831Z", "span": null}
{"agent": "unary", "path": "memory_files/agent_memories/unary_memory.json", "event": "agent_memory_initialized", "level": "debug", "timestamp": "2026-05-11T13:06:30.425012Z", "span": null}
{"agents": ["unary"], "num_agents": 1, "train_shape": [200, 5], "event": "pipeline_start", "level": "info", "timestamp": "2026-05-11T13:06:30.425196Z", "span": null}
{"agent": "unary", "num_columns": 5, "round_idx": 0, "event": "agent_generate_start", "level": "info", "timestamp": "2026-05-11T13:06:30.425971Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "num_messages": 2, "temperature": 0.2, "max_tokens": 4096, "event": "llm_request", "level": "info", "timestamp": "2026-05-11T13:06:30.426145Z", "span": null}
{"provider": "deepseek", "model": "deepseek-chat", "error": "Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "llm_error", "level": "error", "timestamp": "2026-05-11T13:06:30.596535Z", "span": null}
{"agent": "unary", "error": "OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generate_error", "level": "error", "timestamp": "2026-05-11T13:06:30.596887Z", "span": null}
{"agent": "unary", "error": "unary LLM call failed: OpenAI API error: Error code: 401 - {'error': {'message': 'Authentication Fails, Your api key: ****4973 is invalid', 'type': 'authentication_error', 'param': None, 'code': 'invalid_request_error'}}", "event": "agent_generation_failed", "level": "warning", "timestamp": "2026-05-11T13:06:30.597373Z", "span": null}
{"num_specs": 0, "num_selected": 0, "reason": "no_specs_generated", "event": "pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.597653Z", "span": null}
{"path": "memory_files/agent_memories/unary_memory.json", "num_keys": 7, "event": "memory_save", "level": "debug", "timestamp": "2026-05-11T13:06:30.599292Z", "span": null}
{"round_idx": 0, "features_generated": 0, "features_selected": 0, "baseline_score": 0.0, "latency_ms": 175.6, "event": "round_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.599749Z", "span": null}
{"total_rounds": 1, "total_features": 0, "latency_ms": 176.1, "event": "iterative_pipeline_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.600114Z", "span": null}
{"num_selected_features": 0, "latency_ms": 177.3, "event": "fit_complete", "level": "info", "timestamp": "2026-05-11T13:06:30.600482Z", "span": null}
Dashboard: /tmp/ff_artifact_report.html
HTML size: 2,408 bytes

Storage Backends

Code

from feature_forge.artifacts.storage import DataFrameStorage, LazyDataFrameRef

storage = DataFrameStorage(config=ArtifactConfig(storage_mode="memory"))
ref = storage.store("train_data", X_train)
print(f"Stored as: {type(ref)}")

if isinstance(ref, LazyDataFrameRef):
    print(f"Lazy ref path: {ref.path}")
else:
    print(f"DataFrame shape: {ref.shape}")

Stored as: <class 'pandas.core.frame.DataFrame'>
DataFrame shape: (200, 5)

Per-Bundle DataFrame Export

Code

if bundles:
    for method, bundle in bundles.items():
        feat_df = bundle.to_feature_dataframe()
        prov_df = bundle.to_provenance_dataframe()
        print(f"\n{method}:")
        print(f"  Feature metadata DataFrame: {feat_df.shape}")
        print(f"  Provenance DataFrame: {prov_df.shape}")
        if not feat_df.empty:
            print(f"  Columns: {list(feat_df.columns)}")
        if not prov_df.empty:
            print(f"  Columns: {list(prov_df.columns)}")


a:
  Feature metadata DataFrame: (0, 0)
  Provenance DataFrame: (0, 0)

b:
  Feature metadata DataFrame: (0, 0)
  Provenance DataFrame: (0, 0)

Cross-Method Gain Comparison

Code

if len(bundles) >= 2:
    diff_full = ArtifactDiff(bundles)
    gain_df = diff_full.gain_comparison()
    if not gain_df.empty:
        print("Gain comparison across methods:")
        print(gain_df)

Plot: Artifact Metrics

Code

import matplotlib.pyplot as plt

metrics = {
    "scripts": len(fe.generated_scripts) if hasattr(fe, "generated_scripts") else 0,
    "metadata": len(fe.feature_metadata) if hasattr(fe, "feature_metadata") else 0,
    "provenance": len(fe.provenance_records) if hasattr(fe, "provenance_records") else 0,
}

fig, axes = plt.subplots(1, 2, figsize=(12, 4))

ax = axes[0]
ax.bar(metrics.keys(), metrics.values(), color=["steelblue", "teal", "coral"])
ax.set_title("Artifact Counts from MALMASFeatureEngineer")
ax.set_ylabel("Count")

ax = axes[1]
if not diff.overlap_matrix().empty:
    overlap = diff.overlap_matrix()
    im = ax.imshow(overlap.values, cmap="YlGn", aspect="auto")
    ax.set_xticks(range(len(overlap.columns)))
    ax.set_xticklabels(overlap.columns)
    ax.set_yticks(range(len(overlap.index)))
    ax.set_yticklabels(overlap.index)
    ax.set_title("Feature Overlap Matrix")
    for i in range(len(overlap.index)):
        for j in range(len(overlap.columns)):
            ax.text(j, i, str(int(overlap.values[i, j])), ha="center", va="center")
else:
    ax.text(0.5, 0.5, "No overlap data", ha="center", va="center")
    ax.set_title("Feature Overlap Matrix")

plt.tight_layout()
plt.show()

Plot: Gain Comparison

Code

if len(bundles) >= 2:
    diff_full = ArtifactDiff(bundles)
    gain_df = diff_full.gain_comparison()
    if not gain_df.empty:
        fig, ax = plt.subplots(figsize=(max(8, len(gain_df) * 0.6), 5))
        gain_df.plot(kind="bar", ax=ax, width=0.8)
        ax.set_title("Feature Gain Comparison Across Methods")
        ax.set_ylabel("CV Gain")
        ax.axhline(y=0, color="black", linewidth=0.5)
        ax.tick_params(axis="x", rotation=45)
        ax.legend(title="Method")
        plt.tight_layout()
        plt.show()

Summary

ArtifactExporter provides uniform access to scripts, DataFrames, and metadata
ArtifactConfig controls storage mode: memory, disk, or hybrid
ArtifactDiff computes overlap matrices and shared-feature counts
ArtifactDashboard generates self-contained HTML reports
DataFrameStorage abstracts away parquet/csv/feather backends
Cross-method gain comparison reveals which features matter across different approaches