Allow visualizer to be specified about powerful-benchmarker

[dorcus01]

Can you please show an example of how to specify the visualizer and visualizer-hook?

[KevinMusgrave]

@dorcus01 This hasn't been implemented yet, so here's a hard-coded way of getting it done. Replace the last 2 lines of run.py with this:

from powerful_benchmarker.factories import TesterFactory
from powerful_benchmarker.api_parsers import BaseAPIParser
from easy_module_attribute_getter import utils as emag_utils
import copy
import umap
import os
import numpy as np
import matplotlib.pyplot as plt
from cycler import cycler

class TesterFactoryWithUMAP(TesterFactory):
    def _create_general(self, tester_type, plots_folder, **kwargs):
        tester, tester_params = self.getter.get("tester", yaml_dict=tester_type, return_uninitialized=True)
        tester_params = copy.deepcopy(tester_params)
        tester_params["accuracy_calculator"] = self.getter.get("accuracy_calculator", yaml_dict=tester_params["accuracy_calculator"])
        tester_params["visualizer"] = umap.UMAP()
        tester_params["visualizer_hook"] = self.visualizer_hook(plots_folder)
        tester_params = emag_utils.merge_two_dicts(tester_params, kwargs)
        return tester(**tester_params)

    def visualizer_hook(self, plots_folder):
        def actual_visualizer_hook(visualizer, embeddings, labels, split_name, keyname, epoch):
            logging.info("UMAP plot for the {} split and label set {}".format(split_name, keyname))
            label_set = np.unique(labels)
            num_classes = len(label_set)
            fig = plt.figure(figsize=(20,15))
            plt.gca().set_prop_cycle(cycler("color", [plt.cm.nipy_spectral(i) for i in np.linspace(0, 0.9, num_classes)]))
            for i in range(num_classes):
                idx = labels == label_set[i]
                plt.plot(embeddings[idx, 0], embeddings[idx, 1], ".", markersize=1)   
            plt.savefig(os.path.join(plots_folder, "{}_epoch{}.png".format(keyname, epoch)))
        return actual_visualizer_hook


class APIMetricLossOnly(BaseAPIParser):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.sub_experiment_dirs["plots"] = os.path.join("%s", "%s", "plots")

    def set_curr_folders(self):
        folders = self.get_sub_experiment_dir_paths()[self.split_manager.curr_split_scheme_name]
        self.model_folder, self.csv_folder, self.tensorboard_folder, self.plots_folder = folders["models"], folders["csvs"], folders["tensorboard"], folders["plots"]

    def default_kwargs_tester(self):
        kwargs_dict = super().default_kwargs_tester()
        kwargs_dict["plots_folder"] = lambda: self.plots_folder
        return kwargs_dict

               
r = runner(**(args.__dict__))
r.register("factory", TesterFactoryWithUMAP)
r.register("api_parser", APIMetricLossOnly)
r.run()

(You can replace UMAP with another visualizer if you want, like sklearn.manifold.TSNE)

Then on the command line append this flag:

--factories {tester~OVERRIDE~: {TesterFactoryWithUMAP: {}}}

Plots will be saved in png form inside a "plots" folder

and the plots look like this

[dorcus01]

Thank you for your reply. I've tried the method you showed below. In my test, having APIMetricLossOnly registered here will return TypeError: _init_ hot multiple values for argument 'args'. Is IT due to that there is already an APIMetricLossOnly in the library?

…

On Wed, Aug 19, 2020, 13:30 Kevin Musgrave ***@***.***> wrote: This hasn't been implemented yet, so here's a hard-coded way of getting it done. Replace the last 2 lines of run.py with this: from powerful_benchmarker.factories import TesterFactoryfrom powerful_benchmarker.api_parsers import BaseAPIParserfrom easy_module_attribute_getter import utils as emag_utilsimport copyimport umapimport osimport numpy as npimport matplotlib.pyplot as pltfrom cycler import cycler class TesterFactoryWithUMAP(TesterFactory): def _create_general(self, tester_type, plots_folder, **kwargs): tester, tester_params = self.getter.get("tester", yaml_dict=tester_type, return_uninitialized=True) tester_params = copy.deepcopy(tester_params) tester_params["accuracy_calculator"] = self.getter.get("accuracy_calculator", yaml_dict=tester_params["accuracy_calculator"]) tester_params["visualizer"] = umap.UMAP() tester_params["visualizer_hook"] = self.visualizer_hook(plots_folder) tester_params = emag_utils.merge_two_dicts(tester_params, kwargs) return tester(**tester_params) def visualizer_hook(self, plots_folder): def actual_visualizer_hook(visualizer, embeddings, labels, split_name, keyname, epoch): logging.info("UMAP plot for the {} split and label set {}".format(split_name, keyname)) label_set = np.unique(labels) num_classes = len(label_set) fig = plt.figure(figsize=(20,15)) plt.gca().set_prop_cycle(cycler("color", [plt.cm.nipy_spectral(i) for i in np.linspace(0, 0.9, num_classes)])) for i in range(num_classes): idx = labels == label_set[i] plt.plot(embeddings[idx, 0], embeddings[idx, 1], ".", markersize=1) plt.savefig(os.path.join(plots_folder, "{}_epoch{}.png".format(keyname, epoch))) return actual_visualizer_hook class APIMetricLossOnly(BaseAPIParser): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.sub_experiment_dirs["plots"] = os.path.join("%s", "%s", "plots") def set_curr_folders(self): folders = self.get_sub_experiment_dir_paths()[self.split_manager.curr_split_scheme_name] self.model_folder, self.csv_folder, self.tensorboard_folder, self.plots_folder = folders["models"], folders["csvs"], folders["tensorboard"], folders["plots"] def default_kwargs_tester(self): kwargs_dict = super().default_kwargs_tester() kwargs_dict["plots_folder"] = lambda: self.plots_folder return kwargs_dict r = runner(**(args.__dict__))r.register("factory", TesterFactoryWithUMAP)r.register("api_parser", APIMetricLossOnly)r.run() Then on the command line append this flag: --factories {tester~OVERRIDE~: {TesterFactoryWithUMAP: {}}} Plots will be saved in png form inside a "plots" folder [image: image] <https://user-images.githubusercontent.com/2314891/90634598-f14fd700-e1f5-11ea-981f-3811bbe7543d.png> and the plots look like this [image: image] <https://user-images.githubusercontent.com/2314891/90634650-062c6a80-e1f6-11ea-8e57-bd624b013366.png> — You are receiving this because you commented. Reply to this email directly, view it on GitHub <#30 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AOSSKBIIESRUXEMPV3CPW7TSBPAWBANCNFSM4NN6EISA> .

[KevinMusgrave]

I don't think that should matter, and the code worked for me on google colab. Can you paste in the complete error message, and the command line argument you used?

[dorcus01]

Thank you. The problem has been resolved.