seam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "history_visible": true,
      "gpuType": "T4",
      "authorship_tag": "ABX9TyO8hA66GRfMqqYuQ5oA82aB",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/haarsh09/Brain-Tumor-Classification/blob/master/seam.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import cv2\n",
        "\n",
        "\n",
        "class SeamCarver:\n",
        "    def __init__(self, filename, out_height, out_width, protect_mask='', object_mask=''):\n",
        "        # initialize parameter\n",
        "        self.filename = filename\n",
        "        self.out_height = out_height\n",
        "        self.out_width = out_width\n",
        "\n",
        "        # read in image and store as np.float64 format\n",
        "        self.in_image = cv2.imread(filename).astype(np.float64)\n",
        "        self.in_height, self.in_width = self.in_image.shape[: 2]\n",
        "\n",
        "        # keep tracking resulting image\n",
        "        self.out_image = np.copy(self.in_image)\n",
        "\n",
        "        # object removal --> self.object = True\n",
        "        self.object = (object_mask != '')\n",
        "        if self.object:\n",
        "            # read in object mask image file as np.float64 format in gray scale\n",
        "            self.mask = cv2.imread(object_mask, 0).astype(np.float64)\n",
        "            self.protect = False\n",
        "        # image re-sizing with or without protect mask\n",
        "        else:\n",
        "            self.protect = (protect_mask != '')\n",
        "            if self.protect:\n",
        "                # if protect_mask filename is provided, read in protect mask image file as np.float64 format in gray scale\n",
        "                self.mask = cv2.imread(protect_mask, 0).astype(np.float64)\n",
        "\n",
        "        # kernel for forward energy map calculation\n",
        "        self.kernel_x = np.array([[0., 0., 0.], [-1., 0., 1.], [0., 0., 0.]], dtype=np.float64)\n",
        "        self.kernel_y_left = np.array([[0., 0., 0.], [0., 0., 1.], [0., -1., 0.]], dtype=np.float64)\n",
        "        self.kernel_y_right = np.array([[0., 0., 0.], [1., 0., 0.], [0., -1., 0.]], dtype=np.float64)\n",
        "\n",
        "        # constant for covered area by protect mask or object mask\n",
        "        self.constant = 1000\n",
        "\n",
        "        # starting program\n",
        "        self.start()\n",
        "\n",
        "\n",
        "    def start(self):\n",
        "        \"\"\"\n",
        "        :return:\n",
        "\n",
        "        If object mask is provided --> object removal function will be executed\n",
        "        else --> seam carving function (image retargeting) will be process\n",
        "        \"\"\"\n",
        "        if self.object:\n",
        "            self.object_removal()\n",
        "        else:\n",
        "            self.seams_carving()\n",
        "\n",
        "\n",
        "    def seams_carving(self):\n",
        "        \"\"\"\n",
        "        :return:\n",
        "\n",
        "        We first process seam insertion or removal in vertical direction then followed by horizontal direction.\n",
        "\n",
        "        If targeting height or width is greater than original ones --> seam insertion,\n",
        "        else --> seam removal\n",
        "\n",
        "        The algorithm is written for seam processing in vertical direction (column), so image is rotated 90 degree\n",
        "        counter-clockwise for seam processing in horizontal direction (row)\n",
        "        \"\"\"\n",
        "\n",
        "        # calculate number of rows and columns needed to be inserted or removed\n",
        "        delta_row, delta_col = int(self.out_height - self.in_height), int(self.out_width - self.in_width)\n",
        "\n",
        "        # remove column\n",
        "        if delta_col < 0:\n",
        "            self.seams_removal(delta_col * -1)\n",
        "        # insert column\n",
        "        elif delta_col > 0:\n",
        "            self.seams_insertion(delta_col)\n",
        "\n",
        "        # remove row\n",
        "        if delta_row < 0:\n",
        "            self.out_image = self.rotate_image(self.out_image, 1)\n",
        "            if self.protect:\n",
        "                self.mask = self.rotate_mask(self.mask, 1)\n",
        "            self.seams_removal(delta_row * -1)\n",
        "            self.out_image = self.rotate_image(self.out_image, 0)\n",
        "        # insert row\n",
        "        elif delta_row > 0:\n",
        "            self.out_image = self.rotate_image(self.out_image, 1)\n",
        "            if self.protect:\n",
        "                self.mask = self.rotate_mask(self.mask, 1)\n",
        "            self.seams_insertion(delta_row)\n",
        "            self.out_image = self.rotate_image(self.out_image, 0)\n",
        "\n",
        "\n",
        "    def object_removal(self):\n",
        "        \"\"\"\n",
        "        :return:\n",
        "\n",
        "        Object covered by mask will be removed first and seam will be inserted to return to original image dimension\n",
        "        \"\"\"\n",
        "        rotate = False\n",
        "        object_height, object_width = self.get_object_dimension()\n",
        "        if object_height < object_width:\n",
        "            self.out_image = self.rotate_image(self.out_image, 1)\n",
        "            self.mask = self.rotate_mask(self.mask, 1)\n",
        "            rotate = True\n",
        "\n",
        "        while len(np.where(self.mask[:, :] > 0)[0]) > 0:\n",
        "            energy_map = self.calc_energy_map()\n",
        "            energy_map[np.where(self.mask[:, :] > 0)] *= -self.constant\n",
        "            cumulative_map = self.cumulative_map_forward(energy_map)\n",
        "            seam_idx = self.find_seam(cumulative_map)\n",
        "            self.delete_seam(seam_idx)\n",
        "            self.delete_seam_on_mask(seam_idx)\n",
        "\n",
        "        if not rotate:\n",
        "            num_pixels = self.in_width - self.out_image.shape[1]\n",
        "        else:\n",
        "            num_pixels = self.in_height - self.out_image.shape[1]\n",
        "\n",
        "        self.seams_insertion(num_pixels)\n",
        "        if rotate:\n",
        "            self.out_image = self.rotate_image(self.out_image, 0)\n",
        "\n",
        "\n",
        "    def seams_removal(self, num_pixel):\n",
        "        if self.protect:\n",
        "            for dummy in range(num_pixel):\n",
        "                energy_map = self.calc_energy_map()\n",
        "                energy_map[np.where(self.mask > 0)] *= self.constant\n",
        "                cumulative_map = self.cumulative_map_forward(energy_map)\n",
        "                seam_idx = self.find_seam(cumulative_map)\n",
        "                self.delete_seam(seam_idx)\n",
        "                self.delete_seam_on_mask(seam_idx)\n",
        "        else:\n",
        "            for dummy in range(num_pixel):\n",
        "                energy_map = self.calc_energy_map()\n",
        "                cumulative_map = self.cumulative_map_forward(energy_map)\n",
        "                seam_idx = self.find_seam(cumulative_map)\n",
        "                self.delete_seam(seam_idx)\n",
        "\n",
        "\n",
        "    def seams_insertion(self, num_pixel):\n",
        "        if self.protect:\n",
        "            temp_image = np.copy(self.out_image)\n",
        "            temp_mask = np.copy(self.mask)\n",
        "            seams_record = []\n",
        "\n",
        "            for dummy in range(num_pixel):\n",
        "                energy_map = self.calc_energy_map()\n",
        "                energy_map[np.where(self.mask[:, :] > 0)] *= self.constant\n",
        "                cumulative_map = self.cumulative_map_backward(energy_map)\n",
        "                seam_idx = self.find_seam(cumulative_map)\n",
        "                seams_record.append(seam_idx)\n",
        "                self.delete_seam(seam_idx)\n",
        "                self.delete_seam_on_mask(seam_idx)\n",
        "\n",
        "            self.out_image = np.copy(temp_image)\n",
        "            self.mask = np.copy(temp_mask)\n",
        "            n = len(seams_record)\n",
        "            for dummy in range(n):\n",
        "                seam = seams_record.pop(0)\n",
        "                self.add_seam(seam)\n",
        "                self.add_seam_on_mask(seam)\n",
        "                seams_record = self.update_seams(seams_record, seam)\n",
        "        else:\n",
        "            temp_image = np.copy(self.out_image)\n",
        "            seams_record = []\n",
        "\n",
        "            for dummy in range(num_pixel):\n",
        "                energy_map = self.calc_energy_map()\n",
        "                cumulative_map = self.cumulative_map_backward(energy_map)\n",
        "                seam_idx = self.find_seam(cumulative_map)\n",
        "                seams_record.append(seam_idx)\n",
        "                self.delete_seam(seam_idx)\n",
        "\n",
        "            self.out_image = np.copy(temp_image)\n",
        "            n = len(seams_record)\n",
        "            for dummy in range(n):\n",
        "                seam = seams_record.pop(0)\n",
        "                self.add_seam(seam)\n",
        "                seams_record = self.update_seams(seams_record, seam)\n",
        "\n",
        "\n",
        "    def calc_energy_map(self):\n",
        "        b, g, r = cv2.split(self.out_image)\n",
        "        b_energy = np.absolute(cv2.Scharr(b, -1, 1, 0)) + np.absolute(cv2.Scharr(b, -1, 0, 1))\n",
        "        g_energy = np.absolute(cv2.Scharr(g, -1, 1, 0)) + np.absolute(cv2.Scharr(g, -1, 0, 1))\n",
        "        r_energy = np.absolute(cv2.Scharr(r, -1, 1, 0)) + np.absolute(cv2.Scharr(r, -1, 0, 1))\n",
        "        return b_energy + g_energy + r_energy\n",
        "\n",
        "\n",
        "    def cumulative_map_backward(self, energy_map):\n",
        "        m, n = energy_map.shape\n",
        "        output = np.copy(energy_map)\n",
        "        for row in range(1, m):\n",
        "            for col in range(n):\n",
        "                output[row, col] = \\\n",
        "                    energy_map[row, col] + np.amin(output[row - 1, max(col - 1, 0): min(col + 2, n - 1)])\n",
        "        return output\n",
        "\n",
        "\n",
        "    def cumulative_map_forward(self, energy_map):\n",
        "        matrix_x = self.calc_neighbor_matrix(self.kernel_x)\n",
        "        matrix_y_left = self.calc_neighbor_matrix(self.kernel_y_left)\n",
        "        matrix_y_right = self.calc_neighbor_matrix(self.kernel_y_right)\n",
        "\n",
        "        m, n = energy_map.shape\n",
        "        output = np.copy(energy_map)\n",
        "        for row in range(1, m):\n",
        "            for col in range(n):\n",
        "                if col == 0:\n",
        "                    e_right = output[row - 1, col + 1] + matrix_x[row - 1, col + 1] + matrix_y_right[row - 1, col + 1]\n",
        "                    e_up = output[row - 1, col] + matrix_x[row - 1, col]\n",
        "                    output[row, col] = energy_map[row, col] + min(e_right, e_up)\n",
        "                elif col == n - 1:\n",
        "                    e_left = output[row - 1, col - 1] + matrix_x[row - 1, col - 1] + matrix_y_left[row - 1, col - 1]\n",
        "                    e_up = output[row - 1, col] + matrix_x[row - 1, col]\n",
        "                    output[row, col] = energy_map[row, col] + min(e_left, e_up)\n",
        "                else:\n",
        "                    e_left = output[row - 1, col - 1] + matrix_x[row - 1, col - 1] + matrix_y_left[row - 1, col - 1]\n",
        "                    e_right = output[row - 1, col + 1] + matrix_x[row - 1, col + 1] + matrix_y_right[row - 1, col + 1]\n",
        "                    e_up = output[row - 1, col] + matrix_x[row - 1, col]\n",
        "                    output[row, col] = energy_map[row, col] + min(e_left, e_right, e_up)\n",
        "        return output\n",
        "\n",
        "\n",
        "    def calc_neighbor_matrix(self, kernel):\n",
        "        b, g, r = cv2.split(self.out_image)\n",
        "        output = np.absolute(cv2.filter2D(b, -1, kernel=kernel)) + \\\n",
        "                 np.absolute(cv2.filter2D(g, -1, kernel=kernel)) + \\\n",
        "                 np.absolute(cv2.filter2D(r, -1, kernel=kernel))\n",
        "        return output\n",
        "\n",
        "\n",
        "    def find_seam(self, cumulative_map):\n",
        "        m, n = cumulative_map.shape\n",
        "        output = np.zeros((m,), dtype=np.uint32)\n",
        "        output[-1] = np.argmin(cumulative_map[-1])\n",
        "        for row in range(m - 2, -1, -1):\n",
        "            prv_x = output[row + 1]\n",
        "            if prv_x == 0:\n",
        "                output[row] = np.argmin(cumulative_map[row, : 2])\n",
        "            else:\n",
        "                output[row] = np.argmin(cumulative_map[row, prv_x - 1: min(prv_x + 2, n - 1)]) + prv_x - 1\n",
        "        return output\n",
        "\n",
        "\n",
        "    def delete_seam(self, seam_idx):\n",
        "        m, n = self.out_image.shape[: 2]\n",
        "        output = np.zeros((m, n - 1, 3))\n",
        "        for row in range(m):\n",
        "            col = seam_idx[row]\n",
        "            output[row, :, 0] = np.delete(self.out_image[row, :, 0], [col])\n",
        "            output[row, :, 1] = np.delete(self.out_image[row, :, 1], [col])\n",
        "            output[row, :, 2] = np.delete(self.out_image[row, :, 2], [col])\n",
        "        self.out_image = np.copy(output)\n",
        "\n",
        "\n",
        "    def add_seam(self, seam_idx):\n",
        "        m, n = self.out_image.shape[: 2]\n",
        "        output = np.zeros((m, n + 1, 3))\n",
        "        for row in range(m):\n",
        "            col = seam_idx[row]\n",
        "            for ch in range(3):\n",
        "                if col == 0:\n",
        "                    p = np.average(self.out_image[row, col: col + 2, ch])\n",
        "                    output[row, col, ch] = self.out_image[row, col, ch]\n",
        "                    output[row, col + 1, ch] = p\n",
        "                    output[row, col + 1:, ch] = self.out_image[row, col:, ch]\n",
        "                else:\n",
        "                    p = np.average(self.out_image[row, col - 1: col + 1, ch])\n",
        "                    output[row, : col, ch] = self.out_image[row, : col, ch]\n",
        "                    output[row, col, ch] = p\n",
        "                    output[row, col + 1:, ch] = self.out_image[row, col:, ch]\n",
        "        self.out_image = np.copy(output)\n",
        "\n",
        "\n",
        "    def update_seams(self, remaining_seams, current_seam):\n",
        "        output = []\n",
        "        for seam in remaining_seams:\n",
        "            seam[np.where(seam >= current_seam)] += 2\n",
        "            output.append(seam)\n",
        "        return output\n",
        "\n",
        "\n",
        "    def rotate_image(self, image, ccw):\n",
        "        m, n, ch = image.shape\n",
        "        output = np.zeros((n, m, ch))\n",
        "        if ccw:\n",
        "            image_flip = np.fliplr(image)\n",
        "            for c in range(ch):\n",
        "                for row in range(m):\n",
        "                    output[:, row, c] = image_flip[row, :, c]\n",
        "        else:\n",
        "            for c in range(ch):\n",
        "                for row in range(m):\n",
        "                    output[:, m - 1 - row, c] = image[row, :, c]\n",
        "        return output\n",
        "\n",
        "\n",
        "    def rotate_mask(self, mask, ccw):\n",
        "        m, n = mask.shape\n",
        "        output = np.zeros((n, m))\n",
        "        if ccw > 0:\n",
        "            image_flip = np.fliplr(mask)\n",
        "            for row in range(m):\n",
        "                output[:, row] = image_flip[row, : ]\n",
        "        else:\n",
        "            for row in range(m):\n",
        "                output[:, m - 1 - row] = mask[row, : ]\n",
        "        return output\n",
        "\n",
        "\n",
        "    def delete_seam_on_mask(self, seam_idx):\n",
        "        m, n = self.mask.shape\n",
        "        output = np.zeros((m, n - 1))\n",
        "        for row in range(m):\n",
        "            col = seam_idx[row]\n",
        "            output[row, : ] = np.delete(self.mask[row, : ], [col])\n",
        "        self.mask = np.copy(output)\n",
        "\n",
        "\n",
        "    def add_seam_on_mask(self, seam_idx):\n",
        "        m, n = self.mask.shape\n",
        "        output = np.zeros((m, n + 1))\n",
        "        for row in range(m):\n",
        "            col = seam_idx[row]\n",
        "            if col == 0:\n",
        "                p = np.average(self.mask[row, col: col + 2])\n",
        "                output[row, col] = self.mask[row, col]\n",
        "                output[row, col + 1] = p\n",
        "                output[row, col + 1: ] = self.mask[row, col: ]\n",
        "            else:\n",
        "                p = np.average(self.mask[row, col - 1: col + 1])\n",
        "                output[row, : col] = self.mask[row, : col]\n",
        "                output[row, col] = p\n",
        "                output[row, col + 1: ] = self.mask[row, col: ]\n",
        "        self.mask = np.copy(output)\n",
        "\n",
        "\n",
        "    def get_object_dimension(self):\n",
        "        rows, cols = np.where(self.mask > 0)\n",
        "        height = np.amax(rows) - np.amin(rows) + 1\n",
        "        width = np.amax(cols) - np.amin(cols) + 1\n",
        "        return height, width\n",
        "\n",
        "\n",
        "    def save_result(self, filename):\n",
        "        cv2.imwrite(filename, self.out_image.astype(np.uint8))\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "mCLcT_rRmNJM"
      },
      "execution_count": 2,
      "outputs": []
    }
  ]
}