{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "67024584-d139-4450-b6c6-e35473ebeada",
   "metadata": {},
   "source": [
    "# DETR - Detection Transformer\n",
    "Reference: https://github.com/NielsRogge/Transformers-Tutorials/blob/master/DETR/Fine_tuning_DetrForObjectDetection_on_custom_dataset_(balloon).ipynb\n",
    "\n",
    "\n",
    "Original DETR paper: https://arxiv.org/abs/2005.12872\n",
    "\r\n",
    "Original DETR repo: https://github.com/facebookresearch/detr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "0facd98c-44d0-41e4-83ea-2b00a63434ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.9.10\n"
     ]
    }
   ],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "5ac8d340-a033-4eb0-bcea-62da2986ea69",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m pip install --upgrade pip\n",
    "\n",
    "# !pip install supervision==0.3.0\n",
    "\n",
    "# !pip install transformers\n",
    "\n",
    "# !pip install pytorch-lightning\n",
    "\n",
    "# !pip install timm\n",
    "\n",
    "# !pip install cython\n",
    "\n",
    "# !pip install pycocotools\n",
    "\n",
    "# !pip install scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "9972369d-f326-42c8-9acd-2386912af26a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pip 23.2 from D:\\transformer_detr_env\\myvenv\\lib\\site-packages\\pip (python 3.9)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!pip --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "50de65dc-2100-4abd-b20b-0eaaa1007d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.13.1+cu117'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd82a917-f1a8-473b-a4b4-f49019191aea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0.3.0', '4.31.0')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import supervision\n",
    "import transformers\n",
    "\n",
    "supervision.__version__ , transformers.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d0c1b81-66c5-4076-a664-e154b523e396",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.5\n"
     ]
    }
   ],
   "source": [
    "import pytorch_lightning\n",
    "print(pytorch_lightning.__version__)"
   ]
  },
  {
   "attachments": {
    "3c3cf0f7-faa8-477c-b8df-40ff4415b187.png": {
     "image/png": "[encoded data, 434080 characters]"
    }
   },
   "cell_type": "markdown",
   "id": "65977a3b-682b-41a5-8fd2-3f203a0f91b2",
   "metadata": {},
   "source": [
    "## Create COCO data loaders\n",
    "\n",
    "### About dataset:\n",
    "\n",
    "#### number of classes - 4  ['angle', 'fracture', 'line', 'messed_up_angle'\n",
    "\n",
    "\n",
    "![image.png](attachment:3c3cf0f7-faa8-477c-b8df-40ff4415b187.png)]"
   ]
  },
  {
   "attachments": {
    "d4d14b07-49fc-4d16-b0c0-702e6620cd17.png": {
     "image/png": "[encoded data, 569428 characters]"
    }
   },
   "cell_type": "markdown",
   "id": "f8706435-98d4-4de3-b874-7a61f042767e",
   "metadata": {},
   "source": [
    "![image.png](attachment:d4d14b07-49fc-4d16-b0c0-702e6620cd17.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "96bb6f09-6e5b-484d-aa13-d405bdedd212",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "loading annotations into memory...\n",
      "Done (t=0.08s)\n",
      "creating index...\n",
      "index created!\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Number of training examples: 326\n",
      "Number of validation examples: 88\n",
      "Number of test examples: 44\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import torchvision\n",
    "\n",
    "dataset = 'D:/transformer_detr_env/bone_fracture_coco'\n",
    "\n",
    "ANNOTATION_FILE_NAME = \"annotations.json\"\n",
    "TRAIN_DIRECTORY = os.path.join(dataset, \"train\")\n",
    "VAL_DIRECTORY = os.path.join(dataset, \"val\")\n",
    "TEST_DIRECTORY = os.path.join(dataset, \"test\")\n",
    "\n",
    "\n",
    "class CocoDetection(torchvision.datasets.CocoDetection):\n",
    "    def __init__(\n",
    "        self, \n",
    "        image_directory_path: str, \n",
    "        image_processor, \n",
    "        train: bool = True\n",
    "    ):\n",
    "        annotation_file_path = os.path.join(image_directory_path, ANNOTATION_FILE_NAME)\n",
    "        super(CocoDetection, self).__init__(image_directory_path, annotation_file_path)\n",
    "        self.image_processor = image_processor\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        images, annotations = super(CocoDetection, self).__getitem__(idx)        \n",
    "        image_id = self.ids[idx]\n",
    "        annotations = {'image_id': image_id, 'annotations': annotations}\n",
    "        encoding = self.image_processor(images=images, annotations=annotations, return_tensors=\"pt\")\n",
    "        pixel_values = encoding[\"pixel_values\"].squeeze()\n",
    "        target = encoding[\"labels\"][0]\n",
    "\n",
    "        return pixel_values, target\n",
    "\n",
    "\n",
    "TRAIN_DATASET = CocoDetection(image_directory_path=TRAIN_DIRECTORY, image_processor=image_processor, train=True)\n",
    "VAL_DATASET = CocoDetection(image_directory_path=VAL_DIRECTORY, image_processor=image_processor, train=False)\n",
    "TEST_DATASET = CocoDetection(image_directory_path=TEST_DIRECTORY, image_processor=image_processor, train=False)\n",
    "\n",
    "print(\"Number of training examples:\", len(TRAIN_DATASET))\n",
    "print(\"Number of validation examples:\", len(VAL_DATASET))\n",
    "print(\"Number of test examples:\", len(TEST_DATASET))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "db4132d2-0782-411d-842e-be5c53a04903",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image #235\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 318336 characters]",
      "text/plain": [
       "<Figure size 800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize if dataset is loaded properly\n",
    "\n",
    "import random\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "# select random image\n",
    "image_ids = TRAIN_DATASET.coco.getImgIds()\n",
    "image_id = random.choice(image_ids)\n",
    "print('Image #{}'.format(image_id))\n",
    "\n",
    "# load image and annotatons \n",
    "image = TRAIN_DATASET.coco.loadImgs(image_id)[0]\n",
    "annotations = TRAIN_DATASET.coco.imgToAnns[image_id]\n",
    "image_path = os.path.join(TRAIN_DATASET.root, image['file_name'])\n",
    "image = cv2.imread(image_path)\n",
    "\n",
    "# annotate\n",
    "detections = sv.Detections.from_coco_annotations(coco_annotation=annotations)\n",
    "\n",
    "# we will use id2label function for training\n",
    "categories = TRAIN_DATASET.coco.cats\n",
    "id2label = {k: v['name'] for k,v in categories.items()}\n",
    "\n",
    "labels = [\n",
    "    f\"{id2label[class_id]}\" \n",
    "    for _, _, class_id, _ \n",
    "    in detections\n",
    "]\n",
    "\n",
    "box_annotator = sv.BoxAnnotator()\n",
    "frame = box_annotator.annotate(scene=image, detections=detections, labels=labels)\n",
    "\n",
    "%matplotlib inline  \n",
    "sv.show_frame_in_notebook(image, (8, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e2a4cb2-1ce1-40ad-8465-65a9e0faf8e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "def collate_fn(batch):\n",
    "    pixel_values = [item[0] for item in batch]\n",
    "    encoding = image_processor.pad(pixel_values, return_tensors=\"pt\")\n",
    "    labels = [item[1] for item in batch]\n",
    "    return {\n",
    "        'pixel_values': encoding['pixel_values'],\n",
    "        'pixel_mask': encoding['pixel_mask'],\n",
    "        'labels': labels\n",
    "    }\n",
    "\n",
    "TRAIN_DATALOADER = DataLoader(dataset=TRAIN_DATASET, collate_fn=collate_fn, batch_size=4, shuffle=True)\n",
    "VAL_DATALOADER = DataLoader(dataset=VAL_DATASET, collate_fn=collate_fn, batch_size=4)\n",
    "TEST_DATALOADER = DataLoader(dataset=TEST_DATASET, collate_fn=collate_fn, batch_size=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72fcb245-4d1c-42cf-916b-f1541da14b40",
   "metadata": {},
   "source": [
    "# Train model with PyTorch Lightning\n",
    "\n",
    "The DETR model is loaded using the Hugging Face Transformers library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "24f05df1-7768-4561-9e2b-803dd58cb209",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pytorch_lightning as pl\n",
    "from transformers import DetrForObjectDetection\n",
    "import torch\n",
    "\n",
    "\n",
    "class Detr(pl.LightningModule):\n",
    "\n",
    "    def __init__(self, lr, lr_backbone, weight_decay):\n",
    "        super().__init__()\n",
    "        self.model = DetrForObjectDetection.from_pretrained(\n",
    "            pretrained_model_name_or_path=CHECKPOINT, \n",
    "            num_labels=len(id2label),\n",
    "            ignore_mismatched_sizes=True\n",
    "        )\n",
    "        \n",
    "        self.lr = lr\n",
    "        self.lr_backbone = lr_backbone\n",
    "        self.weight_decay = weight_decay\n",
    "\n",
    "    def forward(self, pixel_values, pixel_mask):\n",
    "        return self.model(pixel_values=pixel_values, pixel_mask=pixel_mask)\n",
    "\n",
    "    def common_step(self, batch, batch_idx):\n",
    "        pixel_values = batch[\"pixel_values\"]\n",
    "        pixel_mask = batch[\"pixel_mask\"]\n",
    "        labels = [{k: v.to(self.device) for k, v in t.items()} for t in batch[\"labels\"]]\n",
    "\n",
    "        outputs = self.model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)\n",
    "\n",
    "        loss = outputs.loss\n",
    "        loss_dict = outputs.loss_dict\n",
    "\n",
    "        return loss, loss_dict\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        loss, loss_dict = self.common_step(batch, batch_idx)     \n",
    "        # logs metrics for each training_step, and the average across the epoch\n",
    "        self.log(\"training_loss\", loss)\n",
    "        for k,v in loss_dict.items():\n",
    "            self.log(\"train_\" + k, v.item())\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        loss, loss_dict = self.common_step(batch, batch_idx)     \n",
    "        self.log(\"validation/loss\", loss)\n",
    "        for k, v in loss_dict.items():\n",
    "            self.log(\"validation_\" + k, v.item())\n",
    "            \n",
    "        return loss\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        # DETR authors decided to use different learning rate for backbone\n",
    "        # you can learn more about it here: \n",
    "        # - https://github.com/facebookresearch/detr/blob/3af9fa878e73b6894ce3596450a8d9b89d918ca9/main.py#L22-L23\n",
    "        # - https://github.com/facebookresearch/detr/blob/3af9fa878e73b6894ce3596450a8d9b89d918ca9/main.py#L131-L139\n",
    "        param_dicts = [\n",
    "            {\n",
    "                \"params\": [p for n, p in self.named_parameters() if \"backbone\" not in n and p.requires_grad]},\n",
    "            {\n",
    "                \"params\": [p for n, p in self.named_parameters() if \"backbone\" in n and p.requires_grad],\n",
    "                \"lr\": self.lr_backbone,\n",
    "            },\n",
    "        ]\n",
    "        return torch.optim.AdamW(param_dicts, lr=self.lr, weight_decay=self.weight_decay)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return TRAIN_DATALOADER\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return VAL_DATALOADER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64cf1368-1f4a-4c08-aefe-3d90d2fda638",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of DetrForObjectDetection were not initialized from the model checkpoint at facebook/detr-resnet-50 and are newly initialized because the shapes did not match:\n",
      "- class_labels_classifier.weight: found shape torch.Size([92, 256]) in the checkpoint and torch.Size([6, 256]) in the model instantiated\n",
      "- class_labels_classifier.bias: found shape torch.Size([92]) in the checkpoint and torch.Size([6]) in the model instantiated\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = Detr(lr=1e-4, lr_backbone=1e-5, weight_decay=1e-4)\n",
    "\n",
    "batch = next(iter(TRAIN_DATALOADER))\n",
    "outputs = model(pixel_values=batch['pixel_values'], pixel_mask=batch['pixel_mask'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "16ec6225-5a31-4cb7-93f6-13584215d75b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "D:\\transformer_detr_env\\myvenv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\logger_connector\\logger_connector.py:67: UserWarning: Starting from v1.9.0, `tensorboardX` has been removed as a dependency of the `pytorch_lightning` package, due to potential conflicts with other packages in the ML ecosystem. For this reason, `logger=True` will use `CSVLogger` as the default logger, unless the `tensorboard` or `tensorboardX` packages are found. Please `pip install lightning[extra]` or one of them to enable TensorBoard support by default\n",
      "  warning_cache.warn(\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type                   | Params\n",
      "-------------------------------------------------\n",
      "0 | model | DetrForObjectDetection | 41.5 M\n",
      "-------------------------------------------------\n",
      "41.3 M    Trainable params\n",
      "222 K     Non-trainable params\n",
      "41.5 M    Total params\n",
      "166.041   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\transformer_detr_env\\myvenv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:432: PossibleUserWarning: The dataloader, val_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 24 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "D:\\transformer_detr_env\\myvenv\\lib\\site-packages\\pytorch_lightning\\utilities\\data.py:76: UserWarning: Trying to infer the `batch_size` from an ambiguous collection. The batch size we found is 4. To avoid any miscalculations, use `self.log(..., batch_size=batch_size)`.\n",
      "  warning_cache.warn(\n",
      "D:\\transformer_detr_env\\myvenv\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:432: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 24 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66f1e46045374fa2b5fb8d4ad8013a7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=200` reached.\n"
     ]
    }
   ],
   "source": [
    "from pytorch_lightning import Trainer\n",
    "\n",
    "# settings\n",
    "MAX_EPOCHS = 200\n",
    "\n",
    "trainer = Trainer(devices=1, accelerator=\"gpu\", max_epochs=MAX_EPOCHS, gradient_clip_val=0.1, accumulate_grad_batches=8, log_every_n_steps=5)\n",
    "\n",
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0138f3cf-5dd2-4ae6-b200-3db7c4674b21",
   "metadata": {},
   "source": [
    "# Save and load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "609c44c2-7164-435d-a618-6b66a62d5a3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DetrForObjectDetection(\n",
       "  (model): DetrModel(\n",
       "    (backbone): DetrConvModel(\n",
       "      (conv_encoder): DetrConvEncoder(\n",
       "        (model): FeatureListNet(\n",
       "          (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "          (bn1): DetrFrozenBatchNorm2d()\n",
       "          (act1): ReLU(inplace=True)\n",
       "          (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "          (layer1): Sequential(\n",
       "            (0): Bottleneck(\n",
       "              (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "              (downsample): Sequential(\n",
       "                (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "                (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "              )\n",
       "            )\n",
       "            (1): Bottleneck(\n",
       "              (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (2): Bottleneck(\n",
       "              (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "          )\n",
       "          (layer2): Sequential(\n",
       "            (0): Bottleneck(\n",
       "              (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "              (downsample): Sequential(\n",
       "                (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "                (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "              )\n",
       "            )\n",
       "            (1): Bottleneck(\n",
       "              (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (2): Bottleneck(\n",
       "              (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (3): Bottleneck(\n",
       "              (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "          )\n",
       "          (layer3): Sequential(\n",
       "            (0): Bottleneck(\n",
       "              (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "              (downsample): Sequential(\n",
       "                (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "                (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "              )\n",
       "            )\n",
       "            (1): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (2): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (3): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (4): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (5): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "          )\n",
       "          (layer4): Sequential(\n",
       "            (0): Bottleneck(\n",
       "              (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "              (downsample): Sequential(\n",
       "                (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "                (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "              )\n",
       "            )\n",
       "            (1): Bottleneck(\n",
       "              (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "            (2): Bottleneck(\n",
       "              (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn1): DetrFrozenBatchNorm2d()\n",
       "              (act1): ReLU(inplace=True)\n",
       "              (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "              (bn2): DetrFrozenBatchNorm2d()\n",
       "              (drop_block): Identity()\n",
       "              (act2): ReLU(inplace=True)\n",
       "              (aa): Identity()\n",
       "              (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "              (bn3): DetrFrozenBatchNorm2d()\n",
       "              (act3): ReLU(inplace=True)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (position_embedding): DetrSinePositionEmbedding()\n",
       "    )\n",
       "    (input_projection): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))\n",
       "    (query_position_embeddings): Embedding(100, 256)\n",
       "    (encoder): DetrEncoder(\n",
       "      (layers): ModuleList(\n",
       "        (0): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): DetrEncoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (activation_fn): ReLU()\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (decoder): DetrDecoder(\n",
       "      (layers): ModuleList(\n",
       "        (0): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): DetrDecoderLayer(\n",
       "          (self_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (activation_fn): ReLU()\n",
       "          (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): DetrAttention(\n",
       "            (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "            (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
       "          (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (layernorm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (class_labels_classifier): Linear(in_features=256, out_features=6, bias=True)\n",
       "  (bbox_predictor): DetrMLPPredictionHead(\n",
       "    (layers): ModuleList(\n",
       "      (0): Linear(in_features=256, out_features=256, bias=True)\n",
       "      (1): Linear(in_features=256, out_features=256, bias=True)\n",
       "      (2): Linear(in_features=256, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "MODEL_PATH = 'custom-model'\n",
    "model.model.save_pretrained(MODEL_PATH)\n",
    "\n",
    "# loading model\n",
    "model = DetrForObjectDetection.from_pretrained(MODEL_PATH)\n",
    "model.to(DEVICE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2b200b9-87c7-4993-bddf-18f57969714b",
   "metadata": {},
   "source": [
    "# Inference on test dataset\n",
    "\n",
    "Let's visualize the predictions of DETR on the first image of the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "af859be2-e1b8-4da4-ad4e-91e54dcc1c8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image #23\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 354412 characters]",
      "text/plain": [
       "<Figure size 2000x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# utils\n",
    "categories = TEST_DATASET.coco.cats\n",
    "id2label = {k: v['name'] for k,v in categories.items()}\n",
    "box_annotator = sv.BoxAnnotator()\n",
    "\n",
    "# select random image\n",
    "image_ids = TEST_DATASET.coco.getImgIds()\n",
    "image_id = random.choice(image_ids)\n",
    "print('Image #{}'.format(image_id))\n",
    "\n",
    "# load image and annotatons \n",
    "image = TEST_DATASET.coco.loadImgs(image_id)[0]\n",
    "annotations = TEST_DATASET.coco.imgToAnns[image_id]\n",
    "image_path = os.path.join(TEST_DATASET.root, image['file_name'])\n",
    "image = cv2.imread(image_path)\n",
    "\n",
    "# Annotate ground truth\n",
    "detections = sv.Detections.from_coco_annotations(coco_annotation=annotations)\n",
    "labels = [f\"{id2label[class_id]}\" for _, _, class_id, _ in detections]\n",
    "frame_ground_truth = box_annotator.annotate(scene=image.copy(), detections=detections, labels=labels)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Annotate detections\n",
    "with torch.no_grad():\n",
    "\n",
    "    # load image and predict\n",
    "    inputs = image_processor(images=image, return_tensors='pt').to(DEVICE)\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "    # post-process\n",
    "    target_sizes = torch.tensor([image.shape[:2]]).to(DEVICE)\n",
    "    results = image_processor.post_process_object_detection(\n",
    "        outputs=outputs, \n",
    "        threshold=CONFIDENCE_TRESHOLD, \n",
    "        target_sizes=target_sizes\n",
    "    )[0]\n",
    "\n",
    "\n",
    "    detections = sv.Detections.from_transformers(transformers_results=results)\n",
    "    labels = [f\"{id2label[class_id]} {confidence:.2f}\" for _, confidence, class_id, _ in detections]\n",
    "    frame_detections = box_annotator.annotate(scene=image.copy(), detections=detections, labels=labels)\n",
    "\n",
    "\n",
    "# %matplotlib inline # Remove this line, we won't use it anymore\n",
    "\n",
    "# Combine both images side by side and display\n",
    "fig, axs = plt.subplots(1, 2, figsize=(20, 10))\n",
    "axs[0].imshow(cv2.cvtColor(frame_ground_truth, cv2.COLOR_BGR2RGB))\n",
    "axs[0].axis('off')\n",
    "axs[0].set_title('Ground Truth')\n",
    "\n",
    "axs[1].imshow(cv2.cvtColor(frame_detections, cv2.COLOR_BGR2RGB))\n",
    "axs[1].axis('off')\n",
    "axs[1].set_title('Detections')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dee5914-184f-4f99-978e-25ea725c644b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62889f15-9400-4894-a02a-f8e5d741d5bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
