From 99d614746201c6eecf4d9e9ace62c3be025aac47 Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Thu, 20 Oct 2022 03:07:39 +0800 Subject: [PATCH 1/6] fix: typo `Standford` to `Stanford` --- benchmark/dataset_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/dataset_utils.py b/benchmark/dataset_utils.py index 86030cc..7f65c46 100644 --- a/benchmark/dataset_utils.py +++ b/benchmark/dataset_utils.py @@ -134,9 +134,9 @@ def load_dataset(dataset_name, dataset_path): return load_cub(dataset_path) elif dataset_name == 'OxfordFlowers': return load_oxford_flowers(dataset_path) - elif dataset_name == 'StandfordDogs': + elif dataset_name == 'StanfordDogs': return load_stanford_dogs(dataset_path) - elif dataset_name == 'StandfordCars': + elif dataset_name == 'StanfordCars': return load_stanford_cars(dataset_path) else: - raise ValueError("Unknown dataset name: %s" % dataset_name) \ No newline at end of file + raise ValueError("Unknown dataset name: %s" % dataset_name) From c7bde0ca2fd762ccce1b9da2862ce56043404ac1 Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Fri, 21 Oct 2022 00:57:28 +0800 Subject: [PATCH 2/6] chore: remove tensorflow experiments --- benchmark/configs.py | 89 -------------------------------------------- 1 file changed, 89 deletions(-) diff --git a/benchmark/configs.py b/benchmark/configs.py index 6dd3ae1..c68206b 100644 --- a/benchmark/configs.py +++ b/benchmark/configs.py @@ -49,50 +49,6 @@ "train_objective" : SUPERVISED, "pretrained_weights" : None }, - { - "name" : "imagenet_supervised_tf", - "display_name" : "ImageNet Supervised (tensorflow)", - "color" : "grey", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/supervised/resnet50_no_top.h5', - "training_dataset" : IMAGENET, - "train_objective" : SUPERVISED, - "pretrained_weights" : None - }, - { - "name" : "imagenet_simclr", - "display_name" : "ImageNet SimCLR", - "color" : "C0", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v1/resnet50_simclr_v1_imagenet_no_top.h5', - "training_dataset" : IMAGENET, - "train_objective" : SIMCLR, - "pretrained_weights" : None - }, - { - "name" : "imagenet_simclr_x4", - "display_name" : "ImageNet SimCLR x4", - "color" : "C1", - "format" : TENSORFLOW, - "backbone" : RESNET50_X4, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v1_4x/225206_resnet50_simclr_v1_4x_imagenet_no_top.h5', - "training_dataset" : IMAGENET, - "train_objective" : SIMCLR, - "pretrained_weights" : None - }, - { - "name" : "imagenet_simclr_v2", - "display_name" : "ImageNet SimCLR v2", - "color" : "C2", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'imagenet/simclr_v2/250228_resnet50_simclr_v2_1x_imagenet_no_top.h5', - "training_dataset" : IMAGENET, - "train_objective" : SIMCLR_V2, - "pretrained_weights" : None - }, { "name" : "imagenet_swav", "display_name" : "ImageNet SwAV", @@ -137,17 +93,6 @@ "train_objective" : SUPERVISED, "pretrained_weights" : None }, - { - "name" : "inat2021_simclr", - "display_name" : "iNat2021 SimCLR", - "color" : "C5", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021/simclr_v1/2623871_resnet50_simclr_v1_inat20_no_top.h5', - "training_dataset" : INAT2021, - "train_objective" : SIMCLR, - "pretrained_weights" : None - }, { "name" : "inat2021_mini_supervised", "display_name" : "iNat2021 Mini Supervised", @@ -170,39 +115,6 @@ "train_objective" : SUPERVISED, "pretrained_weights" : None }, - { - "name" : "inat2021_mini_simclr", - "display_name" : "iNat2021 Mini SimCLR", - "color" : "tab:blue", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v1/resnet50_simclr_v1_inat20_mini_no_top.h5', - "training_dataset" : INAT2021_MINI, - "train_objective" : SIMCLR, - "pretrained_weights" : None - }, - { - "name" : "inat2021_mini_simclr_x4", - "display_name" : "iNat2021 Mini SimCLR x4", - "color" : "tab:orange", - "format" : TENSORFLOW, - "backbone" : RESNET50_X4, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v1_4x/488282_resnet50_simclr_4x_inat_mini_no_top.h5', - "training_dataset" : INAT2021_MINI, - "train_objective" : SIMCLR, - "pretrained_weights" : None - }, - { - "name" : "inat2021_mini_simclr_v2", - "display_name" : "iNat2021 Mini SimCLR v2", - "color" : "tab:green", - "format" : TENSORFLOW, - "backbone" : RESNET50, - "weights" : TENSORFLOW_PRETRAINED_MODELS_DIR + 'inat2021_mini/simclr_v2/488282_resnet50_simclr_v2_inat20_mini_no_top.h5', - "training_dataset" : INAT2021_MINI, - "train_objective" : SIMCLR_V2, - "pretrained_weights" : None - }, { "name" : "inat2021_mini_swav", "display_name" : "iNat2021 Mini SwAV", @@ -247,5 +159,4 @@ "train_objective" : SUPERVISED, "pretrained_weights" : IMAGENET }, - ] \ No newline at end of file From 40e4d802b9e45d810507c0e1344426abec2c59db Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Fri, 21 Oct 2022 00:57:45 +0800 Subject: [PATCH 3/6] chore: reformat --- benchmark/make_fg_plots.py | 57 +++++++++++++++----------------------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/benchmark/make_fg_plots.py b/benchmark/make_fg_plots.py index 1b668f3..0817558 100644 --- a/benchmark/make_fg_plots.py +++ b/benchmark/make_fg_plots.py @@ -13,17 +13,16 @@ SMALL_SIZE = 12 MEDIUM_SIZE = 14 BIGGER_SIZE = 18 -plt.rc('font', size=SMALL_SIZE) # controls default text sizes -plt.rc('axes', titlesize=SMALL_SIZE) # fontsize of the axes title -plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels -plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels -plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize +plt.rc('font', size=SMALL_SIZE) # controls default text sizes +plt.rc('axes', titlesize=SMALL_SIZE) # fontsize of the axes title +plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels +plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels +plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title def create_fg_plots(fg_results_dir, output_dir): - fg_results_df = pd.DataFrame(None, columns=['model_name', 'name', 'acc']) for model_spec in configs.model_specs: @@ -42,16 +41,19 @@ def create_fg_plots(fg_results_dir, output_dir): # models are the rows fg_model_results_df = fg_results_df.pivot(index='model_name', columns='name', values='acc') fg_model_results_df = fg_model_results_df.loc[[model_spec['name'] for model_spec in configs.model_specs]] - fg_model_results_df = fg_model_results_df[['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars']] + fg_model_results_df = fg_model_results_df[ + ['OxfordFlowers', 'CUB', 'NABirds', 'StandfordDogs']] # datasets are the rows fg_dataset_results_df = fg_results_df.pivot(index='name', columns='model_name', values='acc') - fg_dataset_results_df = fg_dataset_results_df.loc[['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars']] + fg_dataset_results_df = fg_dataset_results_df.loc[ + ['OxfordFlowers', 'CUB', 'NABirds', 'StandfordDogs']] fg_dataset_results_df = fg_dataset_results_df[[model_spec['name'] for model_spec in configs.model_specs]] ##################### # Stem plot for FG tasks - datasets = ['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StandfordDogs', 'StandfordCars'] + # datasets = ['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StanfordDogs'] + datasets = ['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs'] result_names=[ 'imagenet_simclr', @@ -73,19 +75,17 @@ def create_fg_plots(fg_results_dir, output_dir): baseline_scores = fg_model_results_df.loc['imagenet_supervised'][datasets].values exp_results = [] for model_name in result_names: - model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name) if model_spec['name'] == 'imagenet_supervised': continue - task_scores = fg_model_results_df.loc[ model_spec['name']][datasets].values - baseline_scores - + task_scores = fg_model_results_df.loc[model_spec['name']][datasets].values - baseline_scores r = { - 'name' : model_spec['name'], - 'scores' : task_scores, - 'color' : model_spec['color'], - 'display_name' : model_spec['display_name'], + 'name': model_spec['name'], + 'scores': task_scores, + 'color': model_spec['color'], + 'display_name': model_spec['display_name'], } if model_spec['name'] == 'random': @@ -112,10 +112,9 @@ def create_fg_plots(fg_results_dir, output_dir): exp_results.append(r) - result_df = pd.DataFrame(exp_results) - task_labels = ['Flowers102', 'CUB', 'CUBExpert', 'NABirds', 'StanfordDogs', 'StanfordCars'] + task_labels = ['Flowers102', 'CUB', 'NABirds', 'StanfordDogs'] plot_utils.task_stem_plot( result_df, @@ -138,7 +137,7 @@ def create_fg_plots(fg_results_dir, output_dir): ############## # Latex Table of results - result_names=[ + result_names = [ 'imagenet_supervised', 'imagenet_simclr', 'imagenet_simclr_x4', @@ -158,7 +157,6 @@ def create_fg_plots(fg_results_dir, output_dir): 'inat2021_mini_moco_v2' ] - num_cols = len(datasets) num_rows = len(result_names) @@ -177,22 +175,18 @@ def create_fg_plots(fg_results_dir, output_dir): print("\hline\hline") for model_name in result_names: - model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name) - model_scores = fg_model_results_df.loc[model_spec['name']] ys = [] ry = [] for i, label in enumerate(datasets): - v = model_scores[label] ys.append( "%0.3f" % v ) ry.append(v) - td = model_spec['training_dataset'] if model_spec['training_dataset'] is not None else "" to = model_spec['train_objective'] if model_spec['backbone'] == configs.RESNET50_X4: @@ -211,26 +205,21 @@ def create_fg_plots(fg_results_dir, output_dir): print("\\end{table*}") - def parse_args(): - - parser = argparse.ArgumentParser(description='Create the stem plot figure and latex table of results for the FG datasets.') - + parser = argparse.ArgumentParser( + description='Create the stem plot figure and latex table of results for the FG datasets.') parser.add_argument('--result_dir', dest='result_dir', help='Path to the directory containing the FG results.', type=str, required=True) - parser.add_argument('--output_dir', dest='output_dir', help='Path to the directory to save figures and tables.', type=str, required=True) - parsed_args = parser.parse_args() return parsed_args -if __name__ == '__main__': - +if __name__ == '__main__': args = parse_args() if not os.path.exists(args.result_dir): @@ -240,4 +229,4 @@ def parse_args(): print("Creating %s to store plots and tables" % args.output_dir) os.makedirs(args.output_dir) - create_fg_plots(args.result_dir, args.output_dir) \ No newline at end of file + create_fg_plots(args.result_dir, args.output_dir) From e8563d2daaf163bb6d705cd303130666762527bb Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Fri, 21 Oct 2022 00:59:34 +0800 Subject: [PATCH 4/6] fix: typo `Standford` to `Stanford` --- benchmark/make_fg_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/make_fg_plots.py b/benchmark/make_fg_plots.py index 0817558..c8cc8ad 100644 --- a/benchmark/make_fg_plots.py +++ b/benchmark/make_fg_plots.py @@ -42,12 +42,12 @@ def create_fg_plots(fg_results_dir, output_dir): fg_model_results_df = fg_results_df.pivot(index='model_name', columns='name', values='acc') fg_model_results_df = fg_model_results_df.loc[[model_spec['name'] for model_spec in configs.model_specs]] fg_model_results_df = fg_model_results_df[ - ['OxfordFlowers', 'CUB', 'NABirds', 'StandfordDogs']] + ['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs']] # datasets are the rows fg_dataset_results_df = fg_results_df.pivot(index='name', columns='model_name', values='acc') fg_dataset_results_df = fg_dataset_results_df.loc[ - ['OxfordFlowers', 'CUB', 'NABirds', 'StandfordDogs']] + ['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs']] fg_dataset_results_df = fg_dataset_results_df[[model_spec['name'] for model_spec in configs.model_specs]] ##################### From 5e6905b2815c0692b8cba30df589a30ea31ec41c Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Fri, 21 Oct 2022 01:02:31 +0800 Subject: [PATCH 5/6] fix: StopIteration error. --- benchmark/make_fg_plots.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmark/make_fg_plots.py b/benchmark/make_fg_plots.py index c8cc8ad..04371e3 100644 --- a/benchmark/make_fg_plots.py +++ b/benchmark/make_fg_plots.py @@ -74,9 +74,12 @@ def create_fg_plots(fg_results_dir, output_dir): baseline_scores = fg_model_results_df.loc['imagenet_supervised'][datasets].values exp_results = [] - for model_name in result_names: - model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name) - + model_specs = [ + model_spec + for model_name in result_names + for model_spec in configs.model_specs if model_spec['name'] == model_name + ] + for model_spec in model_specs: if model_spec['name'] == 'imagenet_supervised': continue From 73fa4a0d4bf00874ced91f07ea7a5607142cc650 Mon Sep 17 00:00:00 2001 From: Hao-Ting Li Date: Fri, 21 Oct 2022 01:03:59 +0800 Subject: [PATCH 6/6] fix: StopIteration error. --- benchmark/make_fg_plots.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/benchmark/make_fg_plots.py b/benchmark/make_fg_plots.py index 04371e3..52c0fae 100644 --- a/benchmark/make_fg_plots.py +++ b/benchmark/make_fg_plots.py @@ -55,7 +55,7 @@ def create_fg_plots(fg_results_dir, output_dir): # datasets = ['OxfordFlowers', 'CUB', 'CUBExpert', 'NABirds', 'StanfordDogs'] datasets = ['OxfordFlowers', 'CUB', 'NABirds', 'StanfordDogs'] - result_names=[ + result_names = [ 'imagenet_simclr', 'imagenet_simclr_x4', 'imagenet_simclr_v2', @@ -159,6 +159,11 @@ def create_fg_plots(fg_results_dir, output_dir): 'inat2021_mini_swav', 'inat2021_mini_moco_v2' ] + model_specs = [ + model_spec + for model_name in result_names + for model_spec in configs.model_specs if model_spec['name'] == model_name + ] num_cols = len(datasets) num_rows = len(result_names) @@ -177,8 +182,7 @@ def create_fg_plots(fg_results_dir, output_dir): print(" & \t".join(header) + "\\\\") print("\hline\hline") - for model_name in result_names: - model_spec = next(model_spec for model_spec in configs.model_specs if model_spec['name'] == model_name) + for model_spec in model_specs: model_scores = fg_model_results_df.loc[model_spec['name']] ys = []