From aab4ba3a1d60070da6bc39c955dce1f729194ee2 Mon Sep 17 00:00:00 2001 From: Lluis Date: Sat, 1 May 2021 17:33:35 +0200 Subject: [PATCH] Fix to cuda functions on new Pytorch --- fastai_amalgam/_nbdev.py | 135 ++++++++------- fastai_amalgam/interpret/gradcam.py | 251 +++++++++++++++++++--------- nbs/interpret_gradcam.ipynb | 15 +- 3 files changed, 247 insertions(+), 154 deletions(-) diff --git a/fastai_amalgam/_nbdev.py b/fastai_amalgam/_nbdev.py index d836c00..a8df764 100644 --- a/fastai_amalgam/_nbdev.py +++ b/fastai_amalgam/_nbdev.py @@ -2,74 +2,79 @@ __all__ = ["index", "modules", "custom_doc_links", "git_url"] -index = {"is_3dlut_row": "augment_PIL-img_filters.ipynb", - "read_lut": "augment_PIL-img_filters.ipynb", - "ApplyPILFilter": "augment_PIL-img_filters.ipynb", - "albu_augment": "augment_albumentations.ipynb", - "AlbumentationsWrapper": "augment_albumentations.ipynb", - "BlurringTfms": "augment_albumentations.ipynb", - "StyleTfms": "augment_albumentations.ipynb", - "WeatherTfms": "augment_albumentations.ipynb", - "NoiseTfms": "augment_albumentations.ipynb", - "ColorTonesTfms": "augment_albumentations.ipynb", - "ColorChannelTfms": "augment_albumentations.ipynb", - "LightingTfms": "augment_albumentations.ipynb", - "OtherTfms": "augment_albumentations.ipynb", - "Tfms": "augment_albumentations.ipynb", - "KorniaBase": "augment_kornia.ipynb", - "MotionBlur": "augment_kornia.ipynb", - "ColorJitter": "augment_kornia.ipynb", - "RandomRotation": "augment_kornia.ipynb", - "MedianBlur": "augment_kornia.ipynb", - "HFlip": "augment_kornia.ipynb", - "VFlip": "augment_kornia.ipynb", - "RandomGrayscale": "augment_kornia.ipynb", - "RandomPerspective": "augment_kornia.ipynb", - "torch_to_onnx": "export_onnx.ipynb", - "ClassificationInterpretationEx": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.plot_confusion_matrix": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.plot_accuracy": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.plot_label_confidence": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.plot_top_losses_grid": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.plot_lowest_losses_grid": "interpret_classification-interpretation.ipynb", - "ClassificationInterpretationEx.print_classification_report": "interpret_classification-interpretation.ipynb", - "intersection": "interpret_compare-models.ipynb", - "compare_venn": "interpret_compare-models.ipynb", - "Hook": "interpret_gradcam.ipynb", - "HookBwd": "interpret_gradcam.ipynb", - "to_cuda": "interpret_gradcam.ipynb", - "get_label_idx": "interpret_gradcam.ipynb", - "get_target_layer": "interpret_gradcam.ipynb", - "compute_gcam_items": "interpret_gradcam.ipynb", - "compute_gcam_map": "interpret_gradcam.ipynb", - "plt2pil": "interpret_gradcam.ipynb", - "plt_decoded": "interpret_gradcam.ipynb", - "plot_gcam": "interpret_gradcam.ipynb", - "Learner.gradcam": "interpret_gradcam.ipynb", - "FocalLoss": "loss_functions.ipynb", - "FocalLossFlat": "loss_functions.ipynb", - "TfmdDL.set_font_path": "show_data.ipynb", - "draw_label": "show_data.ipynb", - "TfmdDL.show_batch_grid": "show_data.ipynb", - "open_image": "utils.ipynb", - "img2arraylike": "utils.ipynb", - "make_img_grid": "utils.ipynb", - "PIL.Image.Image.draw_labels": "utils.ipynb", - "color": "utils.ipynb"} +index = { + "is_3dlut_row": "augment_PIL-img_filters.ipynb", + "read_lut": "augment_PIL-img_filters.ipynb", + "ApplyPILFilter": "augment_PIL-img_filters.ipynb", + "albu_augment": "augment_albumentations.ipynb", + "AlbumentationsWrapper": "augment_albumentations.ipynb", + "BlurringTfms": "augment_albumentations.ipynb", + "StyleTfms": "augment_albumentations.ipynb", + "WeatherTfms": "augment_albumentations.ipynb", + "NoiseTfms": "augment_albumentations.ipynb", + "ColorTonesTfms": "augment_albumentations.ipynb", + "ColorChannelTfms": "augment_albumentations.ipynb", + "LightingTfms": "augment_albumentations.ipynb", + "OtherTfms": "augment_albumentations.ipynb", + "Tfms": "augment_albumentations.ipynb", + "KorniaBase": "augment_kornia.ipynb", + "MotionBlur": "augment_kornia.ipynb", + "ColorJitter": "augment_kornia.ipynb", + "RandomRotation": "augment_kornia.ipynb", + "MedianBlur": "augment_kornia.ipynb", + "HFlip": "augment_kornia.ipynb", + "VFlip": "augment_kornia.ipynb", + "RandomGrayscale": "augment_kornia.ipynb", + "RandomPerspective": "augment_kornia.ipynb", + "torch_to_onnx": "export_onnx.ipynb", + "ClassificationInterpretationEx": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.plot_confusion_matrix": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.plot_accuracy": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.plot_label_confidence": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.plot_top_losses_grid": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.plot_lowest_losses_grid": "interpret_classification-interpretation.ipynb", + "ClassificationInterpretationEx.print_classification_report": "interpret_classification-interpretation.ipynb", + "intersection": "interpret_compare-models.ipynb", + "compare_venn": "interpret_compare-models.ipynb", + "Hook": "interpret_gradcam.ipynb", + "HookBwd": "interpret_gradcam.ipynb", + "get_label_idx": "interpret_gradcam.ipynb", + "get_target_layer": "interpret_gradcam.ipynb", + "compute_gcam_items": "interpret_gradcam.ipynb", + "compute_gcam_map": "interpret_gradcam.ipynb", + "plt2pil": "interpret_gradcam.ipynb", + "plt_decoded": "interpret_gradcam.ipynb", + "plot_gcam": "interpret_gradcam.ipynb", + "Learner.gradcam": "interpret_gradcam.ipynb", + "FocalLoss": "loss_functions.ipynb", + "FocalLossFlat": "loss_functions.ipynb", + "TfmdDL.set_font_path": "show_data.ipynb", + "draw_label": "show_data.ipynb", + "TfmdDL.show_batch_grid": "show_data.ipynb", + "open_image": "utils.ipynb", + "img2arraylike": "utils.ipynb", + "make_img_grid": "utils.ipynb", + "PIL.Image.Image.draw_labels": "utils.ipynb", + "color": "utils.ipynb", +} -modules = ["augment/pil_filters.py", - "augment/albumentations.py", - "augment/kornia.py", - "export/onnx.py", - "interpret/interpret.py", - "interpret/compare.py", - "interpret/gradcam.py", - "loss_funcs.py", - "show_data.py", - "utils.py"] +modules = [ + "augment/pil_filters.py", + "augment/albumentations.py", + "augment/kornia.py", + "export/onnx.py", + "interpret/interpret.py", + "interpret/compare.py", + "interpret/gradcam.py", + "loss_funcs.py", + "show_data.py", + "utils.py", +] doc_url = "https://Synopsis.github.io/fastai_amalgam/" git_url = "https://github.com/Synopsis/fastai_amalgam/tree/master/" -def custom_doc_links(name): return None + +def custom_doc_links(name): + return None diff --git a/fastai_amalgam/interpret/gradcam.py b/fastai_amalgam/interpret/gradcam.py index 295c362..e8fdfa1 100644 --- a/fastai_amalgam/interpret/gradcam.py +++ b/fastai_amalgam/interpret/gradcam.py @@ -1,33 +1,54 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/interpret_gradcam.ipynb (unless otherwise specified). -__all__ = ['Hook', 'HookBwd', 'to_cuda', 'get_label_idx', 'get_target_layer', 'compute_gcam_items', 'compute_gcam_map', - 'plt2pil', 'plt_decoded', 'plot_gcam'] +__all__ = [ + "Hook", + "HookBwd", + "get_label_idx", + "get_target_layer", + "compute_gcam_items", + "compute_gcam_map", + "plt2pil", + "plt_decoded", + "plot_gcam", +] # Cell from fastai.vision.all import * from typing import List, Tuple, Callable, Union, Optional, Any, Dict # Cell -class Hook(): +class Hook: def __init__(self, m): self.hook = m.register_forward_hook(self.hook_func) - def hook_func(self, m, inp, out): self.stored = out.detach().clone() - def __enter__(self, *args): return self - def __exit__ (self, *args): self.hook.remove() -class HookBwd(): - def __init__(self,m): + def hook_func(self, m, inp, out): + self.stored = out.detach().clone() + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + self.hook.remove() + + +class HookBwd: + def __init__(self, m): self.hook = m.register_backward_hook(self.hook_func) - def hook_func(self, model, grad_in, grad_out): self.stored = grad_out[0].detach().clone() - def __enter__(self, *args): return self - def __exit__ (self, *args): self.hook.remove() -# Cell -def to_cuda(*args): [o.cuda() for o in args] + def hook_func(self, model, grad_in, grad_out): + self.stored = grad_out[0].detach().clone() + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + self.hook.remove() + # Cell -def get_label_idx(learn:Learner, preds:torch.Tensor, - label:Union[str,int,None]) -> Tuple[int,str]: +def get_label_idx( + learn: Learner, preds: torch.Tensor, label: Union[str, int, None] +) -> Tuple[int, str]: """Either: * Get the label idx of a specific `label` * Get the max pred using `learn.loss_func.decode` and `learn.loss_func.activation` @@ -37,69 +58,91 @@ def get_label_idx(learn:Learner, preds:torch.Tensor, if label is not None: # if `label` is a string, check that it exists in the vocab # and return the label's index - if isinstance(label,str): - if not label in learn.dls.vocab: raise ValueError(f"'{label}' is not part of the Learner's vocab: {learn.dls.vocab}") + if isinstance(label, str): + if not label in learn.dls.vocab: + raise ValueError( + f"'{label}' is not part of the Learner's vocab: {learn.dls.vocab}" + ) return learn.dls.vocab.o2i[label], label # if `label` is an index, return itself - elif isinstance(label,int): return label, learn.dls.vocab[label] - else: raise TypeError(f"Expected `str`, `int` or `None`, got {type(label)} instead") + elif isinstance(label, int): + return label, learn.dls.vocab[label] + else: + raise TypeError( + f"Expected `str`, `int` or `None`, got {type(label)} instead" + ) else: # if no `label` is specified, check that `learn.loss_func` has `decodes` # and `activation` implemented, run the predictions through them, # then check that the output length is 1. If not, the activation must be # sigmoid, which is incompatible - if not hasattr(learn.loss_func, 'activation') or\ - not hasattr(learn.loss_func, 'decodes'): - raise NotImplementedError(f"learn.loss_func does not have `.activation` or `.decodes` methods implemented") + if not hasattr(learn.loss_func, "activation") or not hasattr( + learn.loss_func, "decodes" + ): + raise NotImplementedError( + f"learn.loss_func does not have `.activation` or `.decodes` methods implemented" + ) decode_pred = compose(learn.loss_func.activation, learn.loss_func.decodes) - label_idx = decode_pred(preds) + label_idx = decode_pred(preds) if len(label_idx) > 1: - raise RuntimeError(f"Output label idx must be of length==1. If your loss func has a sigmoid activation, please specify `label`") + raise RuntimeError( + f"Output label idx must be of length==1. If your loss func has a sigmoid activation, please specify `label`" + ) return label_idx, learn.dls.vocab[label_idx][0] + # Cell -def get_target_layer(learn: Learner, - target_layer:Union[nn.Module, Callable, None]) -> nn.Module: +def get_target_layer( + learn: Learner, target_layer: Union[nn.Module, Callable, None] +) -> nn.Module: if target_layer is None: if has_pool_type(learn.model[0]): - warnings.warn(f"Detected a pooling layer in the model body. Unless this is intentional, ensure that the feature map is not flattened") + warnings.warn( + f"Detected a pooling layer in the model body. Unless this is intentional, ensure that the feature map is not flattened" + ) return learn.model[0] elif isinstance(target_layer, nn.Module): return target_layer elif callable(target_layer): return target_layer(learn.model) + # Cell -def compute_gcam_items(learn: Learner, - x: TensorImage, - label: Union[str,int,None] = None, - target_layer: Union[nn.Module, Callable, None] = None - ) -> Tuple[torch.Tensor]: +def compute_gcam_items( + learn: Learner, + x: TensorImage, + label: Union[str, int, None] = None, + target_layer: Union[nn.Module, Callable, None] = None, +) -> Tuple[torch.Tensor]: """Compute gradient and activations of `target_layer` of `learn.model` for `x` with respect to `label`. If `target_layer` is None, then it is set to `learn.model[:-1]` """ - to_cuda(learn.model, x) + if torch.cuda.is_available: + x = x.cuda() + learn.model = learn.model.cuda() + target_layer = get_target_layer(learn, target_layer) with HookBwd(target_layer) as hook_g: with Hook(target_layer) as hook: - preds = learn.model.eval()(x) + preds = learn.model.eval()(x) activations = hook.stored - label_idx, label = get_label_idx(learn,preds,label) - #print(preds.shape, label, label_idx) - #print(preds) + label_idx, label = get_label_idx(learn, preds, label) + # print(preds.shape, label, label_idx) + # print(preds) preds[0, label_idx].backward() gradients = hook_g.stored - preds = getattr(learn.loss_func, 'activation', noop)(preds) + preds = getattr(learn.loss_func, "activation", noop)(preds) # remove leading batch_size axis - gradients = gradients [0] + gradients = gradients[0] activations = activations[0] - preds = preds.detach().cpu().numpy().flatten() + preds = preds.detach().cpu().numpy().flatten() return gradients, activations, preds, label + # Cell def compute_gcam_map(gradients, activations) -> torch.Tensor: """Take the mean of `gradients`, multiply by `activations`, @@ -107,61 +150,91 @@ def compute_gcam_map(gradients, activations) -> torch.Tensor: """ # Mean over the feature maps. If you don't use `keepdim`, it returns # a value of shape (1280) which isn't amenable to `*` with the activations - gcam_weights = gradients.mean(dim=[1,2], keepdim=True) # (1280,7,7) --> (1280,1,1) - gcam_map = (gcam_weights * activations) # (1280,1,1) * (1280,7,7) --> (1280,7,7) - gcam_map = gcam_map.sum(0) # (1280,7,7) --> (7,7) + gcam_weights = gradients.mean( + dim=[1, 2], keepdim=True + ) # (1280,7,7) --> (1280,1,1) + gcam_map = gcam_weights * activations # (1280,1,1) * (1280,7,7) --> (1280,7,7) + gcam_map = gcam_map.sum(0) # (1280,7,7) --> (7,7) return gcam_map + # Cell import PIL + + def plt2pil(fig) -> PIL.Image.Image: """Convert a matplotlib `figure` to a PILImage""" buf = io.BytesIO() - fig.savefig(buf, bbox_inches='tight', pad_inches=0) + fig.savefig(buf, bbox_inches="tight", pad_inches=0) buf.seek(0) - pil_img = PIL.Image.open(buf).convert('RGB') - plt.close('all') + pil_img = PIL.Image.open(buf).convert("RGB") + plt.close("all") return pil_img + def plt_decoded(learn, x, ctx, cmap=None): - 'Processed tensor --> plottable image, return `extent`' + "Processed tensor --> plottable image, return `extent`" x_decoded = TensorImage(learn.dls.train.decode((x,))[0][0]) extent = (0, x_decoded.shape[1], x_decoded.shape[2], 0) x_decoded.show(ctx=ctx, cmap=cmap) return extent -def plot_gcam(learn, img:PILImage, x:tensor, gcam_map:tensor, - full_size=True, alpha=0.6, dpi=100, - interpolation='bilinear', cmap=None, gcam_cmap='magma', **kwargs): - 'Plot the `gcam_map` over `img`' - fig,ax = plt.subplots(dpi=dpi, **kwargs) + +def plot_gcam( + learn, + img: PILImage, + x: tensor, + gcam_map: tensor, + full_size=True, + alpha=0.6, + dpi=100, + interpolation="bilinear", + cmap=None, + gcam_cmap="magma", + **kwargs, +): + "Plot the `gcam_map` over `img`" + fig, ax = plt.subplots(dpi=dpi, **kwargs) if full_size: - extent = (0, img.width,img.height, 0) + extent = (0, img.width, img.height, 0) show_image(img, ctx=ax, cmap=cmap) else: extent = plt_decoded(learn, x, ax, cmap=cmap) - show_image(gcam_map.detach().cpu(), ctx=ax, - alpha=alpha, extent=extent, - interpolation=interpolation, cmap=gcam_cmap) + show_image( + gcam_map.detach().cpu(), + ctx=ax, + alpha=alpha, + extent=extent, + interpolation=interpolation, + cmap=gcam_cmap, + ) return plt2pil(fig) + # Cell from palettable.colorbrewer.diverging import RdYlBu_10_r from ..utils import * + @patch -def gradcam(self: Learner, - item: Union[PILImage, os.PathLike], - target_layer: Union[nn.Module, Callable, None] = None, - labels: Union[str,List[str], int,List[int], None] = None, - full_size=False, show_original=False, img_size=None, alpha=0.5, - cmap = None, - gcam_cmap = RdYlBu_10_r.mpl_colormap, - font_path=None, font_size=None, grid_ncol=4, - **kwargs - ): +def gradcam( + self: Learner, + item: Union[PILImage, os.PathLike], + target_layer: Union[nn.Module, Callable, None] = None, + labels: Union[str, List[str], int, List[int], None] = None, + full_size=False, + show_original=False, + img_size=None, + alpha=0.5, + cmap=None, + gcam_cmap=RdYlBu_10_r.mpl_colormap, + font_path=None, + font_size=None, + grid_ncol=4, + **kwargs, +): """Plot Grad-CAMs of all specified `labels` with respect to `target_layer` Key Args: * `item`: a `PILImage` or path to a file. Use like you would `Learner.predict` @@ -180,12 +253,16 @@ def gradcam(self: Learner, dl = self.dls.test_dl([item]) x = detuplify(first(dl)) - - if not isinstance(labels, list): labels=[labels] - if isinstance(item, PILImage): img = item - else: img = PILImage.create(item) - if img_size is not None: img=img.resize(img_size) - if grid_ncol is None: grid_ncol = 1+len(labels) if show_original else len(labels) + if not isinstance(labels, list): + labels = [labels] + if isinstance(item, PILImage): + img = item + else: + img = PILImage.create(item) + if img_size is not None: + img = img.resize(img_size) + if grid_ncol is None: + grid_ncol = 1 + len(labels) if show_original else len(labels) gcams = defaultdict() @@ -193,12 +270,30 @@ def gradcam(self: Learner, for label in labels: grads, acts, preds, _label = compute_gcam_items(self, x, label, target_layer) gcams[label] = compute_gcam_map(grads, acts) - preds_dict = {l:pred for pred,l in zip(preds, self.dls.vocab)} - pred_img = plot_gcam(self, img, x, gcams[label], full_size=full_size, alpha=alpha, cmap=cmap, gcam_cmap=gcam_cmap) - pred_img.draw_labels(f"{_label}: {preds_dict[_label]* 100:.02f}%", - font_path=font_path, font_size=font_size, location="top") + preds_dict = {l: pred for pred, l in zip(preds, self.dls.vocab)} + pred_img = plot_gcam( + self, + img, + x, + gcams[label], + full_size=full_size, + alpha=alpha, + cmap=cmap, + gcam_cmap=gcam_cmap, + ) + pred_img.draw_labels( + f"{_label}: {preds_dict[_label]* 100:.02f}%", + font_path=font_path, + font_size=font_size, + location="top", + ) results.append(pred_img) if show_original: img = img.resize(results[0].size) - results.insert(0, img.draw_labels("Original", font_path=font_path, font_size=font_size, location="top")) - return make_img_grid(results, img_size=None, ncol=grid_ncol) \ No newline at end of file + results.insert( + 0, + img.draw_labels( + "Original", font_path=font_path, font_size=font_size, location="top" + ), + ) + return make_img_grid(results, img_size=None, ncol=grid_ncol) diff --git a/nbs/interpret_gradcam.ipynb b/nbs/interpret_gradcam.ipynb index d0285c0..0210e97 100644 --- a/nbs/interpret_gradcam.ipynb +++ b/nbs/interpret_gradcam.ipynb @@ -213,16 +213,6 @@ "#### 2. Compute activations (forward pass) and gradients (backward pass)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#export\n", - "def to_cuda(*args): [o.cuda() for o in args]" - ] - }, { "cell_type": "code", "execution_count": null, @@ -308,7 +298,10 @@ "\n", " If `target_layer` is None, then it is set to `learn.model[:-1]`\n", " \"\"\"\n", - " to_cuda(learn.model, x)\n", + " if torch.cuda.is_available:\n", + " x = x.cuda()\n", + " learn.model = learn.model.cuda()\n", + "\n", " target_layer = get_target_layer(learn, target_layer)\n", " with HookBwd(target_layer) as hook_g:\n", " with Hook(target_layer) as hook:\n",