Commit 5bd0a518 authored by Elvira Clement's avatar Elvira Clement
Browse files

Add/Update xp benchmark

parent 073dd21b
import numpy as np
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions, DualScalingOptions
from src.screening.singletest import GapSphereSingleTest
from src.screening.gap_ptest import GAP_Ptest
from src.screening.kappa_test import Kappa_test
from src.screening.gap_test_p_q import GapTestPequalQ
from src.screening.gap_test_all import GapTestAll
nb_algs = 3
nb_algs = 4
def get_nb_algs(setup):
......@@ -19,36 +19,37 @@ def get_alg_params(setup, vec_gammas, exact):
list_names = []
# --- 1. Xp ---
# 1a. No screening
params1 = SlopeParameters()
# -- 1a. No screening
params1 = SlopeParameters()
params1.eval_gap = False
list_params.append(params1)
list_names.append("no screening")
# -- 1b. Strategy p_q=q
params2 = SlopeParameters()
if exact:
params2.screening2 = GapSphereSingleTest()
params2.screening_it_div = 2.
else:
params2.screening1 = GapSphereSingleTest()
params2.screening1 = GapTestPequalQ()
params2.eval_gap_it = setup.eval_gap_it
list_params.append(params2)
list_names.append("single test")
list_names.append("p_q=q")
# 1c. Single test screening
params3 = SlopeParameters()
params3.screening2 = GAP_Ptest(vec_gammas)
params3.screening_it_div = 2.
# -- 1c. Strategy all
params3 = SlopeParameters()
params3.screening1 = GapTestAll(vec_gammas)
params3.eval_gap_it = setup.eval_gap_it
list_params.append(params3)
list_names.append("test-all")
if not exact:
params3.screening1 = GapSphereSingleTest()
list_params.append(params3)
list_names.append("p-test")
# -- 1d. Bao et al.
params4 = SlopeParameters()
params4.screening1 = GapTestPequalQ()
params4.dual_scaling = DualScalingOptions.BAO_ET_AL
params4.eval_gap_it = setup.eval_gap_it
list_params.append(params4)
list_names.append("Bao-et-al")
# --- 2. Common parameters ---
......
python xp_a_get_state.py --id 1a --precision 8
python xp_b_get_budget.py --id 1a --precision 8
python xp_c_results_time.py --id 1a --precision 8
python xp_d_results_it.py --id 1a --precision 8
python xp_e_fig.py --id 1a --precision 8 --noshow --save
\ No newline at end of file
python xp_a_get_state.py --id SIAM --precision 8
python xp_b_get_budget.py --id SIAM --precision 8
python xp_c_results_time.py --id SIAM --precision 8
python xp_d_viz_paper.py --id SIAM --precision 8
\ No newline at end of file
......@@ -35,8 +35,19 @@ def process(folder, bool_it, setup, log=True):
results_rho[:, :, :, :, i] = np.mean(mat_results <= vec_tau[i], axis=4)
# # Sparsity level
# sparsity_level = 100 * np.mean(mat_results_nbnz, axis=3) / setup.n
# # tim
# results_average_it = np.mean(mat_results_it, axis=4)
# results_std_it = np.std(mat_results_it, axis=4)
# ---- return ----
return {
"vec_tau": vec_tau,
"results_rho": results_rho,
# "sparsity_level": sparsity_level,
# "results_average_it": results_average_it,
# "results_std_it": results_std_it,
}
\ No newline at end of file
# -*- coding: utf-8 -*-
import argparse, sys
from pathlib import Path
import numpy as np
from src import __version__
from src.solver.slope import slope_gp
from experiments.SIAM.setup import Setup
from get_algs_params import get_alg_params, get_nb_algs
# --------------------
# Parse arguments
# --------------------
parser=argparse.ArgumentParser()
parser.add_argument('--id', help='setup id', type=str)
parser.add_argument('--id', help='setup id', type=str, default="SIAM")
parser.add_argument('--erase', help='restart xp', action="store_true")
parser.add_argument('--precision', help='stop when gap reaches 1e-precision')
parser.add_argument('--precision', help='stop when gap reaches 1e-precision', default=8, type=float)
parser.add_argument('--exact', action="store_true")
args=parser.parse_args()
......@@ -28,16 +22,11 @@ dims = (setup.nb_dic, setup.nb_sequence, setup.nb_ratio_lbd, setup.n_rep)
mat_seed = np.random.randint(0, 2**8, size=dims)
folder = f'results/1e-{args.precision}'
folder += 'exact/' if args.exact else 'gersh/'
# check if folder exists, create it otherwise
folder += 'exact' if args.exact else 'gersh'
Path(folder).mkdir(parents=True, exist_ok=True)
time_file_name = f"{folder}setup{args.id}_a_state.npz"
time_file_name = f"{folder}/setup{args.id}_a_state.npz"
# --------------------
# Xp
# --------------------
print(f"Starting \"get seed\" with id {args.id}, precision 1e-{args.precision}")
try:
......
# -*- coding: utf-8 -*-
import argparse, sys
from pathlib import Path
import numpy as np
from src import __version__
from src.solver.slope import slope_gp
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions
from src.utils import get_lambda_max, gamma_sequence_generator
from src.dictionaries import generate_dic
from src.screening.gap_ptest import GAP_Ptest
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions
from src.screening.gap_test_all import GapTestAll
from experiments.SIAM.setup import Setup
parser=argparse.ArgumentParser()
parser.add_argument('--id', help='setup id', type=str, default=1)
parser.add_argument('--id', help='setup id', type=str, default="SIAM")
parser.add_argument('--erase', help='restart xp', action="store_true")
parser.add_argument('--precision', help='stop when gap reaches 1e-precision')
parser.add_argument('--precision', help='stop when gap reaches 1e-precision', default=8)
parser.add_argument('--exact', action="store_true")
args=parser.parse_args()
......@@ -30,8 +30,6 @@ setup = Setup(args.id)
folder = f'results/1e-{args.precision}'
folder += 'exact/' if args.exact else 'gersh/'
# check if folder exists, create it otherwise
Path(folder).mkdir(parents=True, exist_ok=True)
state_file_name = f"{folder}setup{args.id}_a_state.npz"
time_file_name = f"{folder}setup{args.id}_b_times.npz"
......@@ -110,8 +108,7 @@ for i_dic in range(setup.nb_dic):
# --- Solve slope problems ---
params = SlopeParameters()
params.screening2 = GAP_Ptest(vec_gammas)
params.screening_it_div = 2
params.screening1 = GapTestAll(vec_gammas)
params.max_it = np.inf
params.gap_stopping = stopping_gap
......@@ -119,13 +116,14 @@ for i_dic in range(setup.nb_dic):
params.lipchitz_update = update_lip
params.accelerated = True
params.verbose = False
params.eval_gap_it = setup.eval_gap_it
# max_eig = .np.linalg.norm(matA, 2)**2
out_slope = slope_gp(vecy, matA, ratio * lbd_max, vec_gammas, params)
mat_times[i_dic, i_seq, i_ratio, rep] = out_slope["time_run"]
mat_it[i_dic, i_seq, i_ratio, rep] = out_slope["nb_it"]
# --- Saving ---
np.savez(time_file_name,
......
# -*- coding: utf-8 -*-
import argparse, sys
from pathlib import Path
import numpy as np
from src import __version__
from src.solver.slope import slope_gp
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions
from src.screening.gap_ptest import GAP_Ptest
from src.utils import get_lambda_max, gamma_sequence_generator
from src.dictionaries import generate_dic
from experiments.SIAM.setup import Setup
from experiments.SIAM.slopepb import SlopePb
from get_algs_params import get_alg_params, get_nb_algs
parser=argparse.ArgumentParser()
parser.add_argument('--id', help='setup id', type=str, default=1)
parser.add_argument('--id', help='setup id', type=str, default="SIAM")
parser.add_argument('--erase', help='restart xp', action="store_true")
parser.add_argument('--precision', help='stop when gap reaches 1e-precision')
parser.add_argument('--precision', help='stop when gap reaches 1e-precision', default=8)
parser.add_argument('--exact', action="store_true")
args=parser.parse_args()
......@@ -31,8 +30,6 @@ args=parser.parse_args()
setup = Setup(args.id)
folder = f'results/1e-{args.precision}'
folder += 'exact/' if args.exact else 'gersh/'
# check if folder exists, create it otherwise
Path(folder).mkdir(parents=True, exist_ok=True)
state_file_name = f"{folder}setup{args.id}_a_state.npz"
time_file_name = f"{folder}setup{args.id}_b_times.npz"
......@@ -44,6 +41,7 @@ mat_seed = out_state["mat_seed"]
out_times = np.load(time_file_name, allow_pickle=True)
mat_times = out_times["mat_times"]
# mat_it = out_times["mat_it"]
mat_results = np.full(
......@@ -51,6 +49,7 @@ mat_results = np.full(
np.nan
)
# stopping_gap = 10**(-float(args.precision))
update_lip = EnumLipchitzOptions.EXACT if args.exact else EnumLipchitzOptions.GERSHGORIN
# --------------------------
......@@ -76,7 +75,7 @@ for i_dic in range(setup.nb_dic):
for i_seq in range(setup.nb_sequence):
for i_ratio, ratio in enumerate(setup.list_ratio_lbd):
stopping_time = np.mean(mat_times[i_dic, i_seq, i_ratio, :])
stopping_time = np.median(mat_times[i_dic, i_seq, i_ratio, :])
if not np.any(np.isnan(mat_results[:, i_dic, i_seq, i_ratio, :])):
t += setup.n_rep
continue
......@@ -84,6 +83,9 @@ for i_dic in range(setup.nb_dic):
for rep in range(setup.n_rep):
print(f"xp time {t+1} / {nb_xp}")
# print(i_dic, i_seq, i_ratio, rep)
# i_dic, i_seq, i_ratio, rep = 1, 0, 2, 17
# --- set seed ---
np.random.seed(mat_seed[i_dic, i_seq, i_ratio, rep])
......@@ -108,20 +110,29 @@ for i_dic in range(setup.nb_dic):
)
lbd_max = get_lambda_max(vecy, matA, vec_gammas)
slopePb = SlopePb(matA, vecy, vec_gammas, ratio, lbdmax=lbd_max)
# --- Solve slope problems ---
list_params, _ = get_alg_params(setup, vec_gammas, args.exact)
for i_alg, params in enumerate(list_params):
if not np.isnan(mat_results[i_alg, i_dic, i_seq, i_ratio, rep]):
continue
params.lipchitz_update = update_lip
params.time_stopping = stopping_time
out_slope = slope_gp(vecy, matA, ratio * lbd_max, vec_gammas, params)
mat_results[i_alg, i_dic, i_seq, i_ratio, rep] = out_slope["gap"]
best_gap = np.min(out_slope["gap"])
vecx_hat = out_slope["sol"]
vecu_hat = slopePb.make_dual_scaling(vecy - matA @ vecx_hat)
gap = slopePb.eval_gap(vecx_hat, vecu_hat)
mat_results[i_alg, i_dic, i_seq, i_ratio, rep] = min(gap, best_gap)
# --- Saving ---
......
# -*- coding: utf-8 -*-
import argparse, sys
from pathlib import Path
import numpy as np
from src import __version__
from src.solver.slope import slope_gp
from src.solver.parameters import SlopeParameters, EnumLipchitzOptions
from src.screening.gap_ptest import GAP_Ptest
from src.utils import get_lambda_max, gamma_sequence_generator
from src.dictionaries import generate_dic
from experiments.SIAM.setup import Setup
from get_algs_params import get_alg_params, get_nb_algs
parser=argparse.ArgumentParser()
parser.add_argument('--id', help='setup id', type=str, default=1)
parser.add_argument('--erase', help='restart xp', action="store_true")
parser.add_argument('--precision', help='stop when gap reaches 1e-precision')
parser.add_argument('--exact', action="store_true")
args=parser.parse_args()
# -------------------------
# Load Setup
# -------------------------
setup = Setup(args.id)
folder = f'results/1e-{args.precision}'
folder += 'exact/' if args.exact else 'gersh/'
# check if folder exists, create it otherwise
Path(folder).mkdir(parents=True, exist_ok=True)
state_file_name = f"{folder}setup{args.id}_a_state.npz"
time_file_name = f"{folder}setup{args.id}_b_times.npz"
results_it_file_name = f"{folder}setup{args.id}_d_results_it.npz"
out_state = np.load(state_file_name, allow_pickle=True)
mat_seed = out_state["mat_seed"]
out_times = np.load(time_file_name, allow_pickle=True)
mat_it = out_times["mat_it"]
mat_results_it = np.full(
(get_nb_algs(setup), setup.nb_dic, setup.nb_sequence, setup.nb_ratio_lbd, setup.n_rep),
np.nan
)
update_lip = EnumLipchitzOptions.EXACT if args.exact else EnumLipchitzOptions.GERSHGORIN
# --------------------------
# Load existing results
# --------------------------
try:
if args.erase:
raise FileNotFoundError
load_results = np.load(results_it_file_name, allow_pickle=True)
mat_results_it = load_results['mat_results_it']
except FileNotFoundError:
# Everything goes well
pass
nb_xp = setup.nb_dic * setup.nb_sequence * setup.nb_ratio_lbd * setup.n_rep
t = 0
print(f"Starting \"xp time\" with id {args.id}, precision 1e-{args.precision}")
for i_dic in range(setup.nb_dic):
for i_seq in range(setup.nb_sequence):
for i_ratio, ratio in enumerate(setup.list_ratio_lbd):
stopping_it = np.mean(mat_it[i_dic, i_seq, i_ratio, :])
if not np.any(np.isnan(mat_results_it[:, i_dic, i_seq, i_ratio, :])):
t += setup.n_rep
continue
for rep in range(setup.n_rep):
print(f"xp it {t+1} / {nb_xp}")
# --- set seed ---
np.random.seed(mat_seed[i_dic, i_seq, i_ratio, rep])
# --- Data and parameters ---
matA = generate_dic(
setup.list_dic[i_dic],
setup.m,
setup.n,
setup.normalize
)
vecy = np.random.randn(setup.m)
vecy /= np.linalg.norm(vecy)
vec_gammas = gamma_sequence_generator(
setup.m,
setup.n,
setup.list_sequence[i_seq][0],
setup.list_sequence[i_seq][1:]
)
lbd_max = get_lambda_max(vecy, matA, vec_gammas)
# --- Solve slope problems ---
list_params, _ = get_alg_params(setup, vec_gammas, args.exact)
for i_alg, params in enumerate(list_params):
if not np.isnan(mat_results_it[i_alg, i_dic, i_seq, i_ratio, rep]):
continue
params.lipchitz_update = update_lip
params.max_it = stopping_it
out_slope = slope_gp(vecy, matA, ratio * lbd_max, vec_gammas, params)
mat_results_it[i_alg, i_dic, i_seq, i_ratio, rep] = out_slope["gap"]
# --- Saving ---
np.savez(results_it_file_name,
mat_results_it = mat_results_it,
version = __version__,
allow_pickle = True
)
t += 1
......@@ -4,10 +4,13 @@ from pathlib import Path
import numpy as np
parser=argparse.ArgumentParser()
parser.add_argument('--id', help='setup id', type=str)
parser.add_argument('--id', help='setup id', type=str, default="SIAM")
parser.add_argument('--precision', help='stop when gap reaches 1e-precision', default=8)
parser.add_argument('--exact', action="store_true")
# parser.add_argument('--it', help='show it results', action="store_true")
parser.add_argument('--ilbd', help='save figure', type=int, default=1)
parser.add_argument('--noshow', help='show plots', action="store_true")
parser.add_argument('--save', help='save figure', action="store_true")
args=parser.parse_args()
......@@ -18,12 +21,12 @@ if args.noshow:
else:
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
from experiments.SIAM.setup import Setup
from process_data import process
from get_algs_params import get_alg_params, get_nb_algs
from src.utils import gamma_sequence_generator
import matplotlib.font_manager as font_manager
from experiments.SIAM.setup import Setup
fs = 20
......@@ -58,15 +61,16 @@ font_ttt = font_manager.FontProperties(
folder = f'results/1e-{args.precision}'
folder += 'exact/' if args.exact else 'gersh/'
# check if folder exists, create it otherwise
Path(folder).mkdir(parents=True, exist_ok=True)
setup = Setup(args.id)
_, list_names = get_alg_params(setup, np.zeros(5), args.exact)
dic_process = process(folder, False, setup)
vec_tau = dic_process["vec_tau"]
results_rho = dic_process["results_rho"]
vec_tau = dic_process["vec_tau"]
results_rho = dic_process["results_rho"]
# sparsity_level = dic_process["sparsity_level"]
# results_average_it = dic_process["results_average_it"]
# results_std_it = dic_process["results_std_it"]
# -------------------------
......@@ -79,22 +83,22 @@ f, ax = plt.subplots(setup.nb_sequence, setup.nb_dic, figsize=(16, 13.),
)
# f.suptitle(f"{setup.list_dic[i_dic]} dictionary", fontsize=14)
list_algnames = ["PG-no", "PGp=q", "PG-all"]
list_algnames = ["PG-no", "PG-p=q", "PG-all", 'PG-Bao']
list_dicname = ["Gaussian", "Uniform", "Toeplitz"]
i_lbd = 1
# args.ilbd = 1
print("ploting time fig for")
print(f" - {list_dicname} dictionaries")
print(f" - lbd / lbd_max = {setup.list_ratio_lbd[i_lbd]}")
print(f" - lbd / lbd_max = {setup.list_ratio_lbd[args.ilbd]}")
for i_dic in range(setup.nb_dic):
for i_seq in range(setup.nb_sequence):
list_colors = ["tab:blue", "tab:orange", "tab:green"]
for i_alg in range(len(list_names)):
list_colors = ["tab:blue", "tab:orange", "tab:green", "tab:purple"]
for i_alg in [0, 1, 3, 2]:#range(len(list_names)):
ax[i_seq, i_dic].plot(
vec_tau,
100. * results_rho[i_alg, i_dic, i_seq, i_lbd],
100. * results_rho[i_alg, i_dic, i_seq, args.ilbd],
linewidth=3.,
label=list_algnames[i_alg],
color=list_colors[i_alg]
......@@ -142,7 +146,8 @@ for i_dic in range(setup.nb_dic):
if args.save:
fig_name = f"figs/setup{args.id}_time"
Path("figs").mkdir(parents=True, exist_ok=True)
fig_name = f"figs/setup{args.id}_precision{args.precision}_exact{args.exact}_time"
plt.savefig(fig_name + ".eps", bbox_inches='tight')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment