Experiment tracking, versioning, and reproducible ML workflows
Reproducibility
The Reproducibility Crisis
Imagine baking a cake from a recipe that says “some flour, a bit of sugar, bake until done.” You would never get the same cake twice. Yet this is roughly how many ML experiments are documented: “we used a learning rate around 1e-3 and trained for a while.” The result is a crisis where published results cannot be replicated, team members cannot reproduce each other’s experiments, and yesterday’s best model cannot be retrained after a framework upgrade. Reproducibility is not a luxury — it is the difference between engineering and alchemy.Deep learning experiments are notoriously hard to reproduce, and the sources of randomness are more numerous than most practitioners realize:
Challenge
Impact
Random seeds
Different initializations, data shuffling, dropout masks
Hardware differences
GPU floating-point non-determinism, different GPU architectures
Software versions
Framework updates, CUDA/cuDNN changes, even compiler flags
Data pipelines
Preprocessing order, augmentation randomness, data loading parallelism
Hyperparameters
Undocumented settings, default values that change between versions
Even with identical code and seeds, moving from one GPU architecture to another (say A100 to H100) can produce different results. Floating-point operations are not associative, and different hardware may fuse or reorder operations differently. Perfect bitwise reproducibility across hardware is generally impossible — aim for statistical reproducibility instead (same conclusions, not identical numbers).
Setting random seeds is the most basic reproducibility measure, but it is also the most commonly done incorrectly. The trap: most people set torch.manual_seed(42) and think they are done. In reality, Python, NumPy, PyTorch CPU, PyTorch CUDA, and cuDNN each have their own random number generators, and if you miss even one, your results will vary between runs.
class SeedManager: """ Manage random seeds across ALL sources of randomness. There are at least 5 independent random number generators in a typical PyTorch training pipeline. This class ensures all of them are seeded consistently. """ def __init__(self, seed: int = 42): self.seed = seed self.original_states = {} def set_all_seeds(self, seed: Optional[int] = None): """ Set seeds for all random sources. Note: Even with all seeds set, perfect reproducibility requires additional measures (see below). """ seed = seed or self.seed # Python's random random.seed(seed) # NumPy np.random.seed(seed) # PyTorch torch.manual_seed(seed) # CUDA (if available) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) # Make cuDNN deterministic torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # Environment variable for some operations os.environ['PYTHONHASHSEED'] = str(seed) print(f"All seeds set to {seed}") def save_states(self): """Save random states for later restoration.""" self.original_states = { 'python': random.getstate(), 'numpy': np.random.get_state(), 'torch': torch.get_rng_state(), } if torch.cuda.is_available(): self.original_states['cuda'] = torch.cuda.get_rng_state_all() return self.original_states def restore_states(self, states: Optional[Dict] = None): """Restore random states.""" states = states or self.original_states random.setstate(states['python']) np.random.set_state(states['numpy']) torch.set_rng_state(states['torch']) if torch.cuda.is_available() and 'cuda' in states: torch.cuda.set_rng_state_all(states['cuda']) def worker_init_fn(self, worker_id: int): """ For DataLoader workers - each needs unique but reproducible seed. Usage: DataLoader(..., worker_init_fn=seed_manager.worker_init_fn) """ worker_seed = self.seed + worker_id np.random.seed(worker_seed) random.seed(worker_seed)# Usageseed_manager = SeedManager(42)seed_manager.set_all_seeds()
Deterministic Operations
def enable_deterministic_mode(): """ Enable fully deterministic operations in PyTorch. Warning: May reduce performance significantly! """ # PyTorch 1.8+ if hasattr(torch, 'use_deterministic_algorithms'): torch.use_deterministic_algorithms(True) # Some operations don't have deterministic implementations # Set environment variable to allow them (with warning) os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' # Disable tf32 for full precision torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = Falseclass DeterministicModule(nn.Module): """ Base class that enforces deterministic operations. """ def __init__(self): super().__init__() # Warn about non-deterministic layers self._warned_layers = set() def forward(self, x): raise NotImplementedError def check_determinism(self, x: torch.Tensor, n_runs: int = 5) -> bool: """Test if forward pass is deterministic.""" outputs = [] for _ in range(n_runs): # Reset seeds torch.manual_seed(0) with torch.no_grad(): out = self.forward(x.clone()) outputs.append(out.cpu()) # Check if all outputs are identical reference = outputs[0] for out in outputs[1:]: if not torch.allclose(reference, out, rtol=0, atol=0): return False return True
Experiment Configuration
Configuration Management
@dataclassclass ExperimentConfig: """ Complete experiment configuration. All hyperparameters and settings in one place. """ # Identifiers experiment_name: str = "experiment" run_id: str = field(default_factory=lambda: datetime.now().strftime("%Y%m%d_%H%M%S")) # Seeds seed: int = 42 # Data data_path: str = "./data" train_split: float = 0.8 val_split: float = 0.1 # Model model_type: str = "resnet18" hidden_size: int = 256 num_layers: int = 3 dropout: float = 0.1 # Training batch_size: int = 32 learning_rate: float = 1e-3 weight_decay: float = 1e-4 epochs: int = 100 # Optimizer optimizer: str = "adamw" momentum: float = 0.9 # Scheduler scheduler: str = "cosine" warmup_epochs: int = 5 # Regularization label_smoothing: float = 0.0 mixup_alpha: float = 0.0 # Hardware device: str = "cuda" num_workers: int = 4 mixed_precision: bool = True # Logging log_dir: str = "./logs" checkpoint_dir: str = "./checkpoints" log_interval: int = 100 def to_dict(self) -> Dict[str, Any]: return asdict(self) def save(self, path: str): """Save config to JSON.""" with open(path, 'w') as f: json.dump(self.to_dict(), f, indent=2) @classmethod def load(cls, path: str) -> 'ExperimentConfig': """Load config from JSON.""" with open(path, 'r') as f: data = json.load(f) return cls(**data) def get_hash(self) -> str: """Get unique hash of configuration.""" config_str = json.dumps(self.to_dict(), sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:8] def diff(self, other: 'ExperimentConfig') -> Dict[str, tuple]: """Show differences between two configs.""" self_dict = self.to_dict() other_dict = other.to_dict() diffs = {} all_keys = set(self_dict.keys()) | set(other_dict.keys()) for key in all_keys: v1 = self_dict.get(key) v2 = other_dict.get(key) if v1 != v2: diffs[key] = (v1, v2) return diffs# CLI configuration loadingclass ConfigLoader: """Load configs from multiple sources with overrides.""" @staticmethod def from_cli(default_config: ExperimentConfig) -> ExperimentConfig: """ Load config with CLI overrides. Usage: python train.py --learning_rate 0.001 --batch_size 64 """ import argparse parser = argparse.ArgumentParser() # Add all config fields as arguments for key, value in asdict(default_config).items(): arg_type = type(value) if value is not None else str parser.add_argument(f'--{key}', type=arg_type, default=value) args = parser.parse_args() return ExperimentConfig(**vars(args))
YAML Configuration
class YAMLConfig: """ YAML-based configuration with inheritance. Example YAML: # base.yaml model: type: resnet18 hidden_size: 256 training: batch_size: 32 learning_rate: 0.001 # experiment.yaml defaults: - base training: """ @staticmethod def load(path: str) -> Dict: """Load YAML with inheritance.""" try: import yaml with open(path, 'r') as f: config = yaml.safe_load(f) # Handle defaults/inheritance if 'defaults' in config: base_config = {} for default in config['defaults']: base_path = Path(path).parent / f"{default}.yaml" base_config = YAMLConfig._deep_merge( base_config, YAMLConfig.load(str(base_path)) ) # Remove defaults key and merge del config['defaults'] config = YAMLConfig._deep_merge(base_config, config) return config except ImportError: print("Install PyYAML: pip install pyyaml") return {} @staticmethod def _deep_merge(base: Dict, override: Dict) -> Dict: """Deep merge two dictionaries.""" result = base.copy() for key, value in override.items(): if key in result and isinstance(result[key], dict) and isinstance(value, dict): result[key] = YAMLConfig._deep_merge(result[key], value) else: result[key] = value return result
In practice, most reproducibility failures are not caused by random seeds or hardware differences — they are caused by losing track of which code, data, and hyperparameters produced a given result. Experiment tracking is the disciplined practice of recording everything needed to recreate any past result. Think of it like a lab notebook in chemistry: without it, you might produce gold once but never remember how.
For teams just starting with experiment tracking: begin with the simplest possible approach (a JSON config file + git commit hash saved alongside each run). Only move to heavyweight tools like MLflow or Weights and Biases when you have 3+ people running experiments and need a shared dashboard. Over-engineering your tracking system before you have a stable training pipeline is a common time sink.
Simple Logger
class ExperimentLogger: """ Simple experiment logger for local tracking. The minimum viable experiment tracking system. Saves config, environment, metrics, and checkpoints in a structured directory. Works offline, requires no external services, and is easy to understand and debug. For production teams, consider upgrading to MLflow, Weights & Biases, or Neptune -- but this logger captures the essential pattern. """ def __init__( self, experiment_name: str, log_dir: str = "./experiments", config: Optional[ExperimentConfig] = None ): self.experiment_name = experiment_name self.run_id = datetime.now().strftime("%Y%m%d_%H%M%S") # Create experiment directory self.exp_dir = Path(log_dir) / experiment_name / self.run_id self.exp_dir.mkdir(parents=True, exist_ok=True) # Initialize logs self.metrics = {} self.step = 0 # Save initial info if config: config.save(self.exp_dir / "config.json") EnvironmentSnapshot.save(str(self.exp_dir / "environment.json")) print(f"Experiment logged to: {self.exp_dir}") def log_metric(self, name: str, value: float, step: Optional[int] = None): """Log a metric value.""" step = step or self.step if name not in self.metrics: self.metrics[name] = [] self.metrics[name].append({ 'step': step, 'value': value, 'timestamp': datetime.now().isoformat() }) def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): """Log multiple metrics.""" for name, value in metrics.items(): self.log_metric(name, value, step) def increment_step(self): """Increment global step.""" self.step += 1 def save_checkpoint( self, model: nn.Module, optimizer: optim.Optimizer, epoch: int, metrics: Dict[str, float] ): """Save model checkpoint.""" checkpoint = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'metrics': metrics, 'step': self.step, } path = self.exp_dir / f"checkpoint_epoch_{epoch}.pt" torch.save(checkpoint, path) # Also save as latest torch.save(checkpoint, self.exp_dir / "checkpoint_latest.pt") def load_checkpoint(self, path: str) -> Dict: """Load a checkpoint.""" return torch.load(path) def save_logs(self): """Save all metrics to disk.""" with open(self.exp_dir / "metrics.json", 'w') as f: json.dump(self.metrics, f, indent=2) def finish(self): """Finalize experiment.""" self.save_logs() # Create summary summary = { 'experiment_name': self.experiment_name, 'run_id': self.run_id, 'total_steps': self.step, 'final_metrics': { name: values[-1]['value'] if values else None for name, values in self.metrics.items() } } with open(self.exp_dir / "summary.json", 'w') as f: json.dump(summary, f, indent=2) print(f"Experiment finished. Logs saved to {self.exp_dir}")
Code is version-controlled with git. Models are often tracked with experiment loggers. But data — the most impactful component of any ML system — is frequently the least version-controlled. Data versioning ensures you can always answer the question: “exactly which data did this model train on?” Without it, you will eventually encounter the nightmare scenario where a model regresses, nobody knows what changed, and the answer turns out to be a silent data pipeline change from two weeks ago.
Git is not designed for large binary files. Do not commit datasets to your git repository. Use DVC (Data Version Control), git-lfs, or cloud storage with hash-based versioning. The pattern below computes hashes of your data so you can detect changes, but the actual data should live outside git.
class DataVersioning: """ Track data versions for reproducibility. Core idea: Compute cryptographic hashes of your data files and store the hashes alongside your experiment config. If the hash changes, the data changed. This catches both intentional updates and accidental corruption. """ @staticmethod def compute_hash(path: str, algorithm: str = 'md5') -> str: """ Compute hash of file or directory. Use this to detect data changes. """ hash_fn = hashlib.new(algorithm) path = Path(path) if path.is_file(): with open(path, 'rb') as f: for chunk in iter(lambda: f.read(8192), b''): hash_fn.update(chunk) else: # Directory: hash all files sorted by name for file_path in sorted(path.rglob('*')): if file_path.is_file(): hash_fn.update(str(file_path.relative_to(path)).encode()) with open(file_path, 'rb') as f: for chunk in iter(lambda: f.read(8192), b''): hash_fn.update(chunk) return hash_fn.hexdigest() @staticmethod def create_manifest( data_path: str, output_path: str = "data_manifest.json" ): """ Create manifest of all data files with hashes. """ manifest = { 'created': datetime.now().isoformat(), 'root': data_path, 'files': [] } for file_path in sorted(Path(data_path).rglob('*')): if file_path.is_file(): manifest['files'].append({ 'path': str(file_path.relative_to(data_path)), 'size': file_path.stat().st_size, 'hash': DataVersioning.compute_hash(str(file_path)) }) manifest['total_hash'] = DataVersioning.compute_hash(data_path) with open(output_path, 'w') as f: json.dump(manifest, f, indent=2) return manifest @staticmethod def verify_manifest(manifest_path: str) -> bool: """ Verify data matches manifest. """ with open(manifest_path, 'r') as f: manifest = json.load(f) data_path = manifest['root'] for file_info in manifest['files']: file_path = Path(data_path) / file_info['path'] if not file_path.exists(): print(f"Missing: {file_info['path']}") return False if DataVersioning.compute_hash(str(file_path)) != file_info['hash']: print(f"Changed: {file_info['path']}") return False return True