# Install the dependencies.
$ pip install -r requirements.txt

$ ./scripts/submit.sh

# We used seed from 0 to 29.
seed=0
# We used n_workers in {1,2,4,8}.
n_workers=4
# We have {lc,hpobench,hpolib,jahs,branin,hartmann}.
bench_cond="--bench_name hpolib --dataset_id 0"  # An example.
# tmp_dir is typically necessary for cluster computation to reduce the filesystem burden.
# If you are using your local computer, this variable can be simply omitted.
tmp_dir=$TMPDIR

# Benchmark conditions.
# bench_cond="--bench_name lc --dataset_id 0"  # dataset_id must be in {0, 1, ..., 33}.
# bench_cond="--bench_name hpobench --dataset_id 0"  # dataset_id must be in {0, 1, ..., 7}.
# bench_cond="--bench_name hpolib --dataset_id 0"  # dataset_id must be in {0, 1, 2, 3}.
# bench_cond="--bench_name jahs --dataset_id 0"  # dataset_id must be in {0, 1, 2}.
# bench_cond="--bench_name branin"  # Branin does not have any additional args.
# bench_cond="--bench_name hartmann --dim 3"  # Hartmann has an arg of `--dim` and `--dim` must be either 3 or 6.

# An Experiment for BOHB.
$ python -m src.bohb --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for DEHB.
$ python -m src.dehb --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for SMAC3.
# SMAC3 requires another enviroment.
$ python -m src.smac --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for Random Search.
$ python -m src.random --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for TPE.
$ python -m src.tpe --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for HyperBand.
$ python -m src.hyperband --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# An Experiment for NePS.
$ ./src/neps.sh --n_workers $n_workers --seed $seed --tmp_dir $tmp_dir $bench_cond

# Small Experiments.
# The Experiments for Figures 3 and 4.
$ python -m validation.test_order_match

# The Experiments for Figure 5.
$ python -m validation.test_line_match

# Experiments not in the paper.
$ python -m validation.test_cumtime_match

# Table of Overall Runtime Reduction (Table 1).
$ python -m validation.real_world_runtime_reduction

# Critical Difference Diagrams (Figure 6).
$ python -m validation.rank_test

# Trajectory Match of Performance over Time (Figure 5).
$ python -m validation.viz_runtime_reduction

# Visualization of Average Rank over Cumulative Runtime.
$ python -m validation.viz_avg_rank

# Table of Runtime Reduction for Each Benchmark.
$ python -m validation.create_latex_code_for_perf_over_time

# Visualization of Performance Over Runtime for Each Benchmark.
$ python -m validation.viz_perf_over_time