Spot the Crop Hackathon/ZINDI_SPOT_THE_CROP_CHALLENGE.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "colab": {
      "name": "ZINDI_SPOT_THE_CROP_CHALLENGE",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "af56dcbcf0db48229a3178fd3d744889": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_22b675c486fc46fcbb5e13ad46c53151",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_73fa6c498a8243e6ac8e73efd1312dcb",
              "IPY_MODEL_7cc1a59b82364896bbda699df8d7bdaf",
              "IPY_MODEL_f2bffcd0fb6d463b97ad047dc5635633"
            ]
          }
        },
        "22b675c486fc46fcbb5e13ad46c53151": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "73fa6c498a8243e6ac8e73efd1312dcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3a4da3082f1549b3b10283c2da8290ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a245510fdf0b4043a4e0f829a129b633"
          }
        },
        "7cc1a59b82364896bbda699df8d7bdaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bb26b090397044279c8fd5b2e98fa3d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 31.4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 31.4,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5753d8c21c5d48adbb742ae0b2f96438"
          }
        },
        "f2bffcd0fb6d463b97ad047dc5635633": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b50833839daa43a4a3146dc868c707be",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 35.0/? [00:01&lt;00:00, 14.81M/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c323843f0b0f482597661877872c838d"
          }
        },
        "3a4da3082f1549b3b10283c2da8290ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a245510fdf0b4043a4e0f829a129b633": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb26b090397044279c8fd5b2e98fa3d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5753d8c21c5d48adbb742ae0b2f96438": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b50833839daa43a4a3146dc868c707be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c323843f0b0f482597661877872c838d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83184e1c441f47a5a5c33eeb198091b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0f2be89c15b54921b09ffc7e62b344c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0ca8cef0d014ed8953a94d55163ed4f",
              "IPY_MODEL_5468ef2377fd4039852f8e0c14bc2ae6",
              "IPY_MODEL_14badefc521440a9b0e29354c388e312"
            ]
          }
        },
        "0f2be89c15b54921b09ffc7e62b344c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0ca8cef0d014ed8953a94d55163ed4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b41a28560c8b4553ae9af308816798cb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d6bd25a92d5f4541a11a661d239170ee"
          }
        },
        "5468ef2377fd4039852f8e0c14bc2ae6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2d294cc0843c4e3abf3ebc8f71a86cdc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5987.8,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5987.8,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5503d07e26b445d480ef41b5e856a411"
          }
        },
        "14badefc521440a9b0e29354c388e312": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5ae761afb3a14da787857644753f8b02",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5990.0/? [02:02&lt;00:00, 58.33M/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_54663a0b9c3041119bb35343fcb755e1"
          }
        },
        "b41a28560c8b4553ae9af308816798cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d6bd25a92d5f4541a11a661d239170ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d294cc0843c4e3abf3ebc8f71a86cdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5503d07e26b445d480ef41b5e856a411": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5ae761afb3a14da787857644753f8b02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "54663a0b9c3041119bb35343fcb755e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "610b3468c08e4c5d96ff6f6c60a2a0db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0f5a942dd75f4d2088a9557cbb1f8511",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e81937aead884923afeda4f989fe72ed",
              "IPY_MODEL_6b76e4063040488d9413749bac0caff8",
              "IPY_MODEL_408cd4882c8e4f27aa85953c0cbf9561"
            ]
          }
        },
        "0f5a942dd75f4d2088a9557cbb1f8511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e81937aead884923afeda4f989fe72ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a89bdffe91d747d4b40ed37cc5b7175f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f502f4790a2940bca54e9b58142a34d0"
          }
        },
        "6b76e4063040488d9413749bac0caff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_90ed9eadb66b4438939bf4c11b1895a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10.9,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10.9,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c0a21ad23842499791a9c6dbfc6e229c"
          }
        },
        "408cd4882c8e4f27aa85953c0cbf9561": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_42e33b1bc7c04ba180c01560b896c8c6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 15.0/? [00:01&lt;00:00,  8.86M/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_40b4c24456e24979a6f99fd232ae622d"
          }
        },
        "a89bdffe91d747d4b40ed37cc5b7175f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f502f4790a2940bca54e9b58142a34d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "90ed9eadb66b4438939bf4c11b1895a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c0a21ad23842499791a9c6dbfc6e229c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "42e33b1bc7c04ba180c01560b896c8c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "40b4c24456e24979a6f99fd232ae622d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b27a137511374e33bbb1b0f08fb24df2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e74a3198148c419ca0f1edb1c062a162",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aa61139af1ac4c8a83f234b01c37b968",
              "IPY_MODEL_79cccb4939564c44831165db0ce38f05",
              "IPY_MODEL_176f0ac29e554a968fef35cbdd1b276e"
            ]
          }
        },
        "e74a3198148c419ca0f1edb1c062a162": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa61139af1ac4c8a83f234b01c37b968": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5383dd86fb9f44b4b845a371c86a4ac3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c2c2d62a76ee4d57b2646ae7ac3e79dd"
          }
        },
        "79cccb4939564c44831165db0ce38f05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_03b4d6518d8a4adf8c47184aeb8ee898",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2566.1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2566.1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dddbafb673214164a5645dd69ef8a3be"
          }
        },
        "176f0ac29e554a968fef35cbdd1b276e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_515df42ea0d249fe85b7eac6bfcbb0d7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2570.0/? [00:54&lt;00:00, 40.55M/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7fb13e02c8c4a9b98735274f0ac90b7"
          }
        },
        "5383dd86fb9f44b4b845a371c86a4ac3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c2c2d62a76ee4d57b2646ae7ac3e79dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03b4d6518d8a4adf8c47184aeb8ee898": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dddbafb673214164a5645dd69ef8a3be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "515df42ea0d249fe85b7eac6bfcbb0d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7fb13e02c8c4a9b98735274f0ac90b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amanbagrecha/datascience_project/blob/master/ZINDI_SPOT_THE_CROP_CHALLENGE.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1a2ae7b9-4da2-472b-860d-5acb063242b0"
      },
      "source": [
        "<img src='https://radiant-assets.s3-us-west-2.amazonaws.com/PrimaryRadiantMLHubLogo.png' alt='Radiant MLHub Logo' width='200'/>\n",
        "\n",
        "# Top 5 solution for the Radiant Earth Spot the Crop Challenge\n",
        "\n",
        "This notebook walks you through the steps to load the data and build a Top 5 solution model using Random Forests and Xgboost for `Radiant Earth Spot the Crop Challenge`. \n",
        "\n",
        "The notebook builds on top of baseline notebook provided by Radiant Earth"
      ],
      "id": "1a2ae7b9-4da2-472b-860d-5acb063242b0"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k0MQB1jR6U1R",
        "outputId": "5e055241-90d0-48df-c887-f11c52cb99ae"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "id": "k0MQB1jR6U1R",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5894a1e0-23f6-4f8d-940b-1317fba1e762"
      },
      "source": [
        "## Radiant MLHub API\n",
        "\n",
        "\n",
        "The Radiant MLHub API gives access to open Earth imagery training data for machine learning applications. You can learn more about the repository at the [Radiant MLHub site](https://mlhub.earth) and about the organization behind it at the [Radiant Earth Foundation site](https://radiant.earth).\n",
        "\n",
        "Full documentation for the API is available at [docs.mlhub.earth](docs.mlhub.earth).\n",
        "\n",
        "Each item in our collection is explained in json format compliant with [STAC](https://stacspec.org/) [label extension](https://github.com/radiantearth/stac-spec/tree/master/extensions/label) definition."
      ],
      "id": "5894a1e0-23f6-4f8d-940b-1317fba1e762"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "973f1d8d-12cd-4661-9cf5-c50ecb9b2ab6"
      },
      "source": [
        "## Dependencies\n",
        "\n"
      ],
      "id": "973f1d8d-12cd-4661-9cf5-c50ecb9b2ab6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "210d52cd-d81e-446b-8a87-2b0a6cac3bae"
      },
      "source": [
        "**You must replace the `YOUR_API_KEY_HERE` text with your API key which you can obtain by creating a free account on the [MLHub Dashboard](https://dashboard.mlhub.earth/) within the `API Keys` tab at the top of the page.**"
      ],
      "id": "210d52cd-d81e-446b-8a87-2b0a6cac3bae"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPjQonkZSCTO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8115fbbb-36a2-476c-81a5-7f010a2b0420"
      },
      "source": [
        "!pip install radiant_mlhub\n",
        "!pip install rasterio"
      ],
      "id": "kPjQonkZSCTO",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting radiant_mlhub\n",
            "  Downloading radiant_mlhub-0.2.2-py3-none-any.whl (26 kB)\n",
            "Collecting pystac==0.5.4\n",
            "  Downloading pystac-0.5.4-py3-none-any.whl (133 kB)\n",
            "\u001b[K     |████████████████████████████████| 133 kB 8.0 MB/s \n",
            "\u001b[?25hCollecting tqdm~=4.56.0\n",
            "  Downloading tqdm-4.56.2-py2.py3-none-any.whl (72 kB)\n",
            "\u001b[K     |████████████████████████████████| 72 kB 769 kB/s \n",
            "\u001b[?25hRequirement already satisfied: click~=7.1.2 in /usr/local/lib/python3.7/dist-packages (from radiant_mlhub) (7.1.2)\n",
            "Collecting requests~=2.25.1\n",
            "  Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 7.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.0 in /usr/local/lib/python3.7/dist-packages (from pystac==0.5.4->radiant_mlhub) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.0->pystac==0.5.4->radiant_mlhub) (1.15.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->radiant_mlhub) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->radiant_mlhub) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->radiant_mlhub) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests~=2.25.1->radiant_mlhub) (2021.5.30)\n",
            "Installing collected packages: tqdm, requests, pystac, radiant-mlhub\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.62.0\n",
            "    Uninstalling tqdm-4.62.0:\n",
            "      Successfully uninstalled tqdm-4.62.0\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.25.1 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed pystac-0.5.4 radiant-mlhub-0.2.2 requests-2.25.1 tqdm-4.56.2\n",
            "Collecting rasterio\n",
            "  Downloading rasterio-1.2.6-cp37-cp37m-manylinux1_x86_64.whl (19.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 19.3 MB 1.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click>=4.0 in /usr/local/lib/python3.7/dist-packages (from rasterio) (7.1.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from rasterio) (1.19.5)\n",
            "Collecting snuggs>=1.4.1\n",
            "  Downloading snuggs-1.4.7-py3-none-any.whl (5.4 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from rasterio) (57.4.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from rasterio) (2021.5.30)\n",
            "Collecting click-plugins\n",
            "  Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)\n",
            "Collecting affine\n",
            "  Downloading affine-2.3.0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.7/dist-packages (from rasterio) (21.2.0)\n",
            "Collecting cligj>=0.5\n",
            "  Downloading cligj-0.7.2-py3-none-any.whl (7.1 kB)\n",
            "Requirement already satisfied: pyparsing>=2.1.6 in /usr/local/lib/python3.7/dist-packages (from snuggs>=1.4.1->rasterio) (2.4.7)\n",
            "Installing collected packages: snuggs, cligj, click-plugins, affine, rasterio\n",
            "Successfully installed affine-2.3.0 click-plugins-1.1.1 cligj-0.7.2 rasterio-1.2.6 snuggs-1.4.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a5ec7942-db89-4039-9065-555c6a5775aa"
      },
      "source": [
        "from radiant_mlhub import Collection\n",
        "import tarfile\n",
        "import os\n",
        "from pathlib import Path\n",
        "import json\n",
        "\n",
        "import datetime\n",
        "import rasterio\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import log_loss\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from sklearn.metrics import accuracy_score\n",
        "import xgboost as xgb\n",
        "import random\n",
        "import gc"
      ],
      "id": "a5ec7942-db89-4039-9065-555c6a5775aa",
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDMCCUK-n0Go"
      },
      "source": [
        "random.seed(10)"
      ],
      "id": "qDMCCUK-n0Go",
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ae21a662-b755-4456-b722-d242c154143c"
      },
      "source": [
        "## Downloading and Loading the Data\n",
        "\n",
        "In this part, we will download the data from Radiant MLHub and load the properties of each item in the dataset into a DataFrame\n"
      ],
      "id": "ae21a662-b755-4456-b722-d242c154143c"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "466b7593-e197-4e00-a392-4ef34f0f4e28",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219,
          "referenced_widgets": [
            "af56dcbcf0db48229a3178fd3d744889",
            "22b675c486fc46fcbb5e13ad46c53151",
            "73fa6c498a8243e6ac8e73efd1312dcb",
            "7cc1a59b82364896bbda699df8d7bdaf",
            "f2bffcd0fb6d463b97ad047dc5635633",
            "3a4da3082f1549b3b10283c2da8290ae",
            "a245510fdf0b4043a4e0f829a129b633",
            "bb26b090397044279c8fd5b2e98fa3d1",
            "5753d8c21c5d48adbb742ae0b2f96438",
            "b50833839daa43a4a3146dc868c707be",
            "c323843f0b0f482597661877872c838d",
            "83184e1c441f47a5a5c33eeb198091b6",
            "0f2be89c15b54921b09ffc7e62b344c2",
            "f0ca8cef0d014ed8953a94d55163ed4f",
            "5468ef2377fd4039852f8e0c14bc2ae6",
            "14badefc521440a9b0e29354c388e312",
            "b41a28560c8b4553ae9af308816798cb",
            "d6bd25a92d5f4541a11a661d239170ee",
            "2d294cc0843c4e3abf3ebc8f71a86cdc",
            "5503d07e26b445d480ef41b5e856a411",
            "5ae761afb3a14da787857644753f8b02",
            "54663a0b9c3041119bb35343fcb755e1",
            "610b3468c08e4c5d96ff6f6c60a2a0db",
            "0f5a942dd75f4d2088a9557cbb1f8511",
            "e81937aead884923afeda4f989fe72ed",
            "6b76e4063040488d9413749bac0caff8",
            "408cd4882c8e4f27aa85953c0cbf9561",
            "a89bdffe91d747d4b40ed37cc5b7175f",
            "f502f4790a2940bca54e9b58142a34d0",
            "90ed9eadb66b4438939bf4c11b1895a2",
            "c0a21ad23842499791a9c6dbfc6e229c",
            "42e33b1bc7c04ba180c01560b896c8c6",
            "40b4c24456e24979a6f99fd232ae622d",
            "b27a137511374e33bbb1b0f08fb24df2",
            "e74a3198148c419ca0f1edb1c062a162",
            "aa61139af1ac4c8a83f234b01c37b968",
            "79cccb4939564c44831165db0ce38f05",
            "176f0ac29e554a968fef35cbdd1b276e",
            "5383dd86fb9f44b4b845a371c86a4ac3",
            "c2c2d62a76ee4d57b2646ae7ac3e79dd",
            "03b4d6518d8a4adf8c47184aeb8ee898",
            "dddbafb673214164a5645dd69ef8a3be",
            "515df42ea0d249fe85b7eac6bfcbb0d7",
            "b7fb13e02c8c4a9b98735274f0ac90b7"
          ]
        },
        "outputId": "884c09f0-8dcd-4f84-ac93-8c1ebed2d675"
      },
      "source": [
        "os.environ['MLHUB_API_KEY'] = 'YOUR_API_KEY_HERE'\n",
        "\n",
        "collections = [\n",
        "    'ref_south_africa_crops_competition_v1_train_labels',\n",
        "    'ref_south_africa_crops_competition_v1_train_source_s1', \n",
        "    'ref_south_africa_crops_competition_v1_test_labels',\n",
        "    'ref_south_africa_crops_competition_v1_test_source_s1', \n",
        "#     'ref_south_africa_crops_competition_v1_test_source_s2' # Uncomment this out if you want to download the Sentinel-2 Data (not needed for the Hackathon)\n",
        "#     'ref_south_africa_crops_competition_v1_train_source_s2', # Uncomment this out if you want to download the Sentinel-2 Data (not needed for the Hackathon)\n",
        "\n",
        "]\n",
        "\n",
        "def download(collection_id):\n",
        "    print(f'Downloading {collection_id}...')\n",
        "    collection = Collection.fetch(collection_id)\n",
        "    path = collection.download('.')\n",
        "    path = collection_id + '.tar.gz'\n",
        "    tar = tarfile.open(path, \"r:gz\")\n",
        "    tar.extractall()\n",
        "    tar.close()\n",
        "    os.remove(path)\n",
        "    \n",
        "def resolve_path(base, path):\n",
        "    return Path(os.path.join(base, path)).resolve()\n",
        "    \n",
        "def load_df(collection_id):\n",
        "    collection = json.load(open(f'{collection_id}/collection.json', 'r'))\n",
        "    rows = []\n",
        "    item_links = []\n",
        "    for link in collection['links']:\n",
        "        if link['rel'] != 'item':\n",
        "            continue\n",
        "        item_links.append(link['href'])\n",
        "        \n",
        "    for item_link in item_links:\n",
        "        item_path = f'{collection_id}/{item_link}'\n",
        "        current_path = os.path.dirname(item_path)\n",
        "        item = json.load(open(item_path, 'r'))\n",
        "        tile_id = item['id'].split('_')[-1]\n",
        "        for asset_key, asset in item['assets'].items():\n",
        "            rows.append([\n",
        "                tile_id,\n",
        "                None,\n",
        "                None,\n",
        "                asset_key,\n",
        "                str(resolve_path(current_path, asset['href']))\n",
        "            ])\n",
        "            \n",
        "        for link in item['links']:\n",
        "            if link['rel'] != 'source':\n",
        "                continue\n",
        "            link_path = resolve_path(current_path, link['href'])\n",
        "            source_path = os.path.dirname(link_path)\n",
        "            try:\n",
        "                source_item = json.load(open(link_path, 'r'))\n",
        "            except FileNotFoundError:\n",
        "                continue\n",
        "            datetime = source_item['properties']['datetime']\n",
        "            satellite_platform = source_item['collection'].split('_')[-1]\n",
        "            for asset_key, asset in source_item['assets'].items():\n",
        "                rows.append([\n",
        "                    tile_id,\n",
        "                    datetime,\n",
        "                    satellite_platform,\n",
        "                    asset_key,\n",
        "                    str(resolve_path(source_path, asset['href']))\n",
        "                ])\n",
        "    return pd.DataFrame(rows, columns=['tile_id', 'datetime', 'satellite_platform', 'asset', 'file_path'])\n",
        "\n",
        "for c in collections:\n",
        "    download(c)\n",
        "\n"
      ],
      "id": "466b7593-e197-4e00-a392-4ef34f0f4e28",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading ref_south_africa_crops_competition_v1_train_labels...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "af56dcbcf0db48229a3178fd3d744889",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/31.4 [00:00<?, ?M/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading ref_south_africa_crops_competition_v1_train_source_s1...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "83184e1c441f47a5a5c33eeb198091b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/5987.8 [00:00<?, ?M/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading ref_south_africa_crops_competition_v1_test_labels...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "610b3468c08e4c5d96ff6f6c60a2a0db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/10.9 [00:00<?, ?M/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading ref_south_africa_crops_competition_v1_test_source_s1...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b27a137511374e33bbb1b0f08fb24df2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/2566.1 [00:00<?, ?M/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iP5RjMGHO60k"
      },
      "source": [
        "competition_train_df = load_df('ref_south_africa_crops_competition_v1_train_labels')\n",
        "competition_test_df = load_df('ref_south_africa_crops_competition_v1_test_labels')"
      ],
      "id": "iP5RjMGHO60k",
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6bd68551-758a-482f-b022-f6ce59bc27e0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33a87f81-436d-4b51-dd61-6cecd3b4a574"
      },
      "source": [
        "# This DataFrame lists all types of assets including documentation of the data. \n",
        "# In the following, we will use the Sentinel-1 bands (VV and VH) as well as labels. \n",
        "competition_train_df['asset'].unique()"
      ],
      "id": "6bd68551-758a-482f-b022-f6ce59bc27e0",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['documentation', 'field_ids', 'field_info_train', 'labels',\n",
              "       'raster_values', 'VH', 'VV'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "asntPQz40Gsw",
        "outputId": "18a8648b-d85d-4428-f5ca-f4a02afcce76"
      },
      "source": [
        "competition_train_df.head()"
      ],
      "id": "asntPQz40Gsw",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tile_id</th>\n",
              "      <th>datetime</th>\n",
              "      <th>satellite_platform</th>\n",
              "      <th>asset</th>\n",
              "      <th>file_path</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2587</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>documentation</td>\n",
              "      <td>/content/ref_south_africa_crops_competition_v1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2587</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>field_ids</td>\n",
              "      <td>/content/ref_south_africa_crops_competition_v1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2587</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>field_info_train</td>\n",
              "      <td>/content/ref_south_africa_crops_competition_v1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2587</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>labels</td>\n",
              "      <td>/content/ref_south_africa_crops_competition_v1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2587</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>raster_values</td>\n",
              "      <td>/content/ref_south_africa_crops_competition_v1...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  tile_id  ...                                          file_path\n",
              "0    2587  ...  /content/ref_south_africa_crops_competition_v1...\n",
              "1    2587  ...  /content/ref_south_africa_crops_competition_v1...\n",
              "2    2587  ...  /content/ref_south_africa_crops_competition_v1...\n",
              "3    2587  ...  /content/ref_south_africa_crops_competition_v1...\n",
              "4    2587  ...  /content/ref_south_africa_crops_competition_v1...\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fdddc229-eb18-424e-8bce-0b1a0ac7cbc4"
      },
      "source": [
        "# find unique tiles id\n",
        "tile_ids_train = competition_train_df['tile_id'].unique()"
      ],
      "id": "fdddc229-eb18-424e-8bce-0b1a0ac7cbc4",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rx4401SeqNwY"
      },
      "source": [
        "# calculate the distribution of tiles time-stamp  \n",
        "cdf={}\n",
        "for tile_id in tile_ids_train:\n",
        "    if tile_id != '1951': # avoid using this specific tile for the Hackathon as it might have a missing file\n",
        "        \n",
        "        tile_df = competition_train_df[competition_train_df['tile_id']==tile_id]\n",
        "        tile_date_times = tile_df[tile_df['satellite_platform']=='s1']['datetime'].unique()\n",
        "        cdf[len(tile_date_times)] = cdf.get(len(tile_date_times),0)+1 "
      ],
      "id": "Rx4401SeqNwY",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4oPKQ11qos9",
        "outputId": "1c24b150-fbae-4088-fa8a-789a973cf824"
      },
      "source": [
        "cdf"
      ],
      "id": "R4oPKQ11qos9",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{20: 5, 21: 1713, 23: 2, 33: 1, 37: 2, 40: 1, 41: 925}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ba16fdd-0d81-4661-aab9-e95d6156f329"
      },
      "source": [
        "# loop over first 1000 tiles\n",
        "X = np.empty((0, 14), np.uint8)\n",
        "y = np.empty((0, 1), np.uint8)\n",
        "field_ids = np.empty((0, 1), np.uint32)\n",
        "\n",
        "for tile_id in tile_ids_train[:1000]:\n",
        "    if tile_id != '1951': # avoid using this specific tile for the Hackathon as it might have a missing file\n",
        "        \n",
        "        tile_df = competition_train_df[competition_train_df['tile_id']==tile_id]\n",
        "\n",
        "        field_id_src = rasterio.open(tile_df[tile_df['asset']=='field_ids']['file_path'].values[0])\n",
        "        field_id_array = field_id_src.read(1).flatten()\n",
        "        my_idx = np.where(field_id_array!=0)[0]\n",
        "        field_ids = np.append(field_ids, field_id_array[my_idx])\n",
        "        \n",
        "        label_src = rasterio.open(tile_df[tile_df['asset']=='labels']['file_path'].values[0])\n",
        "        label_array = label_src.read(1).flatten()\n",
        "        y = np.append(y, label_array[my_idx])\n",
        "\n",
        "\n",
        "        tile_date_times = tile_df[tile_df['satellite_platform']=='s1']['datetime'].unique()\n",
        "        X_tile = np.empty((len(my_idx), 0), np.uint8)\n",
        "        if(len(tile_date_times))>36:\n",
        "          for date_time in tile_date_times[ :40:6]:\n",
        "              vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "              vv_array = np.expand_dims(vv_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "              vh_array = np.expand_dims(vh_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "              X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "        else:\n",
        "\n",
        "          for date_time in tile_date_times[ :20:3]:\n",
        "              vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "              vv_array = np.expand_dims(vv_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "              vh_array = np.expand_dims(vh_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "              X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "\n",
        "        X = np.append(X, X_tile, axis=0)"
      ],
      "id": "3ba16fdd-0d81-4661-aab9-e95d6156f329",
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IDwsgNR_QsV2"
      },
      "source": [
        "# loop over last ~1600 tiles and append to prev X array\n",
        "for tile_id in tile_ids_train[1000:]:\n",
        "    if tile_id != '1951': # avoid using this specific tile for the Hackathon as it might have a missing file\n",
        "        \n",
        "        tile_df = competition_train_df[competition_train_df['tile_id']==tile_id]\n",
        "\n",
        "        field_id_src = rasterio.open(tile_df[tile_df['asset']=='field_ids']['file_path'].values[0])\n",
        "        field_id_array = field_id_src.read(1).flatten()\n",
        "        my_idx = np.where(field_id_array!=0)[0]\n",
        "        field_ids = np.append(field_ids, field_id_array[my_idx])\n",
        "        \n",
        "        label_src = rasterio.open(tile_df[tile_df['asset']=='labels']['file_path'].values[0])\n",
        "        label_array = label_src.read(1).flatten()\n",
        "        y = np.append(y, label_array[my_idx])\n",
        "\n",
        "\n",
        "        tile_date_times = tile_df[tile_df['satellite_platform']=='s1']['datetime'].unique()\n",
        "        X_tile = np.empty((len(my_idx), 0), np.uint8)\n",
        "        if(len(tile_date_times))>36:\n",
        "          for date_time in tile_date_times[ :40:6]:\n",
        "              vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "              vv_array = np.expand_dims(vv_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "              vh_array = np.expand_dims(vh_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "              X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "        else:\n",
        "\n",
        "          for date_time in tile_date_times[ :20:3]:\n",
        "              vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "              vv_array = np.expand_dims(vv_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "              vh_array = np.expand_dims(vh_src.read(1).flatten()[my_idx], axis=1)\n",
        "\n",
        "              X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "              X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "\n",
        "        X = np.append(X, X_tile, axis=0)"
      ],
      "id": "IDwsgNR_QsV2",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d6b5--rMS2wL",
        "outputId": "17b8265e-4ac9-4f41-fcfc-45c40ca71c59"
      },
      "source": [
        "X.shape"
      ],
      "id": "d6b5--rMS2wL",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(65780218, 14)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9b2543fe-4450-4515-bdab-160e7d206069",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "3926192d-6363-4293-e966-4d96f2d92721"
      },
      "source": [
        "data = pd.DataFrame(X)\n",
        "data['label'] = y\n",
        "data['field_id'] = field_ids\n",
        "data = data[data.field_id != 0]\n",
        "data.head()"
      ],
      "id": "9b2543fe-4450-4515-bdab-160e7d206069",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>label</th>\n",
              "      <th>field_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13</td>\n",
              "      <td>3</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>37</td>\n",
              "      <td>3</td>\n",
              "      <td>33</td>\n",
              "      <td>3</td>\n",
              "      <td>29</td>\n",
              "      <td>4</td>\n",
              "      <td>10</td>\n",
              "      <td>5</td>\n",
              "      <td>18</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>3020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>21</td>\n",
              "      <td>1</td>\n",
              "      <td>16</td>\n",
              "      <td>1</td>\n",
              "      <td>29</td>\n",
              "      <td>4</td>\n",
              "      <td>18</td>\n",
              "      <td>4</td>\n",
              "      <td>31</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>3020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>3</td>\n",
              "      <td>16</td>\n",
              "      <td>2</td>\n",
              "      <td>21</td>\n",
              "      <td>3</td>\n",
              "      <td>25</td>\n",
              "      <td>3</td>\n",
              "      <td>33</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>3020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>32</td>\n",
              "      <td>3</td>\n",
              "      <td>30</td>\n",
              "      <td>3</td>\n",
              "      <td>25</td>\n",
              "      <td>8</td>\n",
              "      <td>35</td>\n",
              "      <td>4</td>\n",
              "      <td>48</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>3020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>3</td>\n",
              "      <td>16</td>\n",
              "      <td>2</td>\n",
              "      <td>22</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>5</td>\n",
              "      <td>29</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>3020</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    0  1  2  3   4  5   6  7   8  9  10  11  12  13  label  field_id\n",
              "0  13  3  9  1  37  3  33  3  29  4  10   5  18   5      2      3020\n",
              "1  16  1  5  2  21  1  16  1  29  4  18   4  31   3      2      3020\n",
              "2  11  2  6  0  20  3  16  2  21  3  25   3  33   5      2      3020\n",
              "3   6  2  6  1  32  3  30  3  25  8  35   4  48   4      2      3020\n",
              "4  11  2  7  0  20  3  16  2  22  1  22   5  29   7      2      3020"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QKfCl1doTBzO",
        "outputId": "27171a37-3891-4683-9e0b-5728425f13ed"
      },
      "source": [
        "del X\n",
        "gc.collect()"
      ],
      "id": "QKfCl1doTBzO",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "209"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b72aeb93-f95c-4e40-9f73-ef55982da982"
      },
      "source": [
        "## Building the Model"
      ],
      "id": "b72aeb93-f95c-4e40-9f73-ef55982da982"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yWcBT-lxqhD-"
      },
      "source": [
        "class group_by_attribute(object):\n",
        "    \"\"\"groub by field id and calculate mean on incoming data\"\"\"\n",
        "    def __init__(self):\n",
        "        super(group_by_attribute, self).__init__()\n",
        "        self.gb = np.empty((0,1), np.uint8)\n",
        "\n",
        "    def compute(self,df, idx):\n",
        "        self.gb = df.loc[:,['field_id', idx]].groupby('field_id').mean()\n",
        "        if(idx=='label'):\n",
        "            self.gb = self.gb.round(0).astype(int)"
      ],
      "id": "yWcBT-lxqhD-",
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CYIV6kF9WIJF"
      },
      "source": [
        "# create objects for each cloumn to perform group-by \n",
        "gb_objs = [group_by_attribute() for i in range(len(data.columns)-1) ]"
      ],
      "id": "CYIV6kF9WIJF",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ml-9VSfMyh1d"
      },
      "source": [
        "# perform group-by on individual column by passing column name and dataframe\n",
        "for i,j in enumerate(data.columns[:-1]):\n",
        "  gb_objs[i].compute(data,j)"
      ],
      "id": "Ml-9VSfMyh1d",
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYWbTMyXs6lE"
      },
      "source": [
        "# perform concatination on field_id\n",
        "from functools import reduce \n",
        "df_final = [i.gb for i in gb_objs]\n",
        "\n",
        "df_merged = reduce(lambda left,right: pd.merge(left,right,on='field_id'), df_final)"
      ],
      "id": "nYWbTMyXs6lE",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5zq_5C-fteuh"
      },
      "source": [
        "df_merged = df_merged.reset_index()"
      ],
      "id": "5zq_5C-fteuh",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2_mJHEDmHFPZ",
        "outputId": "01bf821d-0b9f-430e-8f5f-369a6434f9a2"
      },
      "source": [
        "df_merged.head()"
      ],
      "id": "2_mJHEDmHFPZ",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>field_id</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>14.143087</td>\n",
              "      <td>2.979100</td>\n",
              "      <td>11.432476</td>\n",
              "      <td>2.872990</td>\n",
              "      <td>27.040193</td>\n",
              "      <td>7.993569</td>\n",
              "      <td>19.217042</td>\n",
              "      <td>4.901929</td>\n",
              "      <td>22.573955</td>\n",
              "      <td>6.652733</td>\n",
              "      <td>22.879421</td>\n",
              "      <td>7.479100</td>\n",
              "      <td>21.461415</td>\n",
              "      <td>5.821543</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>10.521739</td>\n",
              "      <td>2.913043</td>\n",
              "      <td>7.478261</td>\n",
              "      <td>2.956522</td>\n",
              "      <td>36.217391</td>\n",
              "      <td>2.913043</td>\n",
              "      <td>27.695652</td>\n",
              "      <td>5.347826</td>\n",
              "      <td>19.956522</td>\n",
              "      <td>5.695652</td>\n",
              "      <td>15.565217</td>\n",
              "      <td>4.782609</td>\n",
              "      <td>15.304348</td>\n",
              "      <td>2.043478</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>4.383033</td>\n",
              "      <td>0.877892</td>\n",
              "      <td>4.885604</td>\n",
              "      <td>0.898458</td>\n",
              "      <td>51.457584</td>\n",
              "      <td>5.526992</td>\n",
              "      <td>29.917738</td>\n",
              "      <td>5.484576</td>\n",
              "      <td>13.113111</td>\n",
              "      <td>4.377892</td>\n",
              "      <td>8.542416</td>\n",
              "      <td>3.605398</td>\n",
              "      <td>17.742931</td>\n",
              "      <td>2.984576</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>11.125828</td>\n",
              "      <td>1.576159</td>\n",
              "      <td>14.807947</td>\n",
              "      <td>1.900662</td>\n",
              "      <td>36.132450</td>\n",
              "      <td>3.172185</td>\n",
              "      <td>34.317881</td>\n",
              "      <td>7.000000</td>\n",
              "      <td>32.933775</td>\n",
              "      <td>8.662252</td>\n",
              "      <td>25.576159</td>\n",
              "      <td>11.761589</td>\n",
              "      <td>25.920530</td>\n",
              "      <td>6.880795</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>14.630682</td>\n",
              "      <td>4.767045</td>\n",
              "      <td>16.625000</td>\n",
              "      <td>5.090909</td>\n",
              "      <td>29.204545</td>\n",
              "      <td>8.113636</td>\n",
              "      <td>25.170455</td>\n",
              "      <td>8.482955</td>\n",
              "      <td>30.636364</td>\n",
              "      <td>11.301136</td>\n",
              "      <td>25.176136</td>\n",
              "      <td>6.528409</td>\n",
              "      <td>30.284091</td>\n",
              "      <td>9.778409</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   field_id          0         1  ...         12        13  label\n",
              "0         1  14.143087  2.979100  ...  21.461415  5.821543      4\n",
              "1         2  10.521739  2.913043  ...  15.304348  2.043478      7\n",
              "2         3   4.383033  0.877892  ...  17.742931  2.984576      6\n",
              "3         4  11.125828  1.576159  ...  25.920530  6.880795      8\n",
              "4         6  14.630682  4.767045  ...  30.284091  9.778409      4\n",
              "\n",
              "[5 rows x 16 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6383d15a-5e27-467d-9ea6-2c7af202a8c7"
      },
      "source": [
        "# Split train and test\n",
        "# We use field_ids to split the data to train and test. Note that the test portion for training is different than the test \n",
        "# portion provided as part of the competition. \n",
        "train_per = 0.8\n",
        "\n",
        "n_fields = len(df_merged['field_id'])\n",
        "np.random.seed(10)\n",
        "train_fields = np.random.choice(df_merged['field_id'], int(n_fields * train_per), replace=False)\n",
        "test_fields = df_merged['field_id'][~np.in1d(df_merged['field_id'], train_fields)]"
      ],
      "id": "6383d15a-5e27-467d-9ea6-2c7af202a8c7",
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf3dc6ed-d908-43f3-b854-3d273b45d593"
      },
      "source": [
        "X_train, X_test = df_merged[df_merged['field_id'].isin(train_fields)], df_merged[df_merged['field_id'].isin(test_fields)]\n",
        "X_train = X_train.drop(columns=['label', 'field_id'])\n",
        "X_test = X_test.drop(columns=['label', 'field_id'])\n",
        "y_train, y_test = df_merged[df_merged['field_id'].isin(train_fields)]['label'], df_merged[df_merged['field_id'].isin(test_fields)]['label']"
      ],
      "id": "bf3dc6ed-d908-43f3-b854-3d273b45d593",
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeXc8ns602my"
      },
      "source": [
        "xgb_params = {\n",
        "    'n_estimators': 500,\n",
        "    'reg_alpha': .7,\n",
        "    # 'tree_method':'gpu_hist', # gpu_hist to be used when GPU turned on\n",
        "    'max_depth': 8,\n",
        "    'colsample_bytree':0.8\n",
        "}\n",
        "xgb_ = xgb.XGBClassifier(**xgb_params,verbosity=1 )\n",
        "xgb_.fit(X_train, y_train)"
      ],
      "id": "ZeXc8ns602my",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lTV5SYNANi_P"
      },
      "source": [
        "pred_proba = xgb_.predict_proba(X_test)"
      ],
      "id": "lTV5SYNANi_P",
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SeTLm-PYMoex",
        "outputId": "6e155e16-b9c8-4fcf-c0ae-899044d15987"
      },
      "source": [
        "# validation loss\n",
        "log_loss(y_test, pred_proba)"
      ],
      "id": "SeTLm-PYMoex",
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.182687285624778"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6eaf48aa-431f-4b05-a978-6e3729c1ef1d"
      },
      "source": [
        "# After running grid search hyperparameter tuning, and concluded to use:\n",
        "params = {'bootstrap': True,\n",
        " 'max_depth': 120,\n",
        " 'max_features': 4,\n",
        " 'min_samples_leaf': 5,\n",
        " 'min_samples_split': 10,\n",
        " 'n_estimators': 300}"
      ],
      "id": "6eaf48aa-431f-4b05-a978-6e3729c1ef1d",
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fced7dae-01e5-4677-9e41-e94790f057e0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aef35e58-e28c-4961-fc7a-ce4cdc9a6bb4"
      },
      "source": [
        "# Fitting the RF model\n",
        "rf = RandomForestClassifier(**params, random_state = 0, n_jobs = -1)\n",
        "rf.fit(X_train, y_train)"
      ],
      "id": "fced7dae-01e5-4677-9e41-e94790f057e0",
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=120, max_features=4,\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=5, min_samples_split=10,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=300,\n",
              "                       n_jobs=-1, oob_score=False, random_state=0, verbose=0,\n",
              "                       warm_start=False)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EnrNcArDO3Br"
      },
      "source": [
        "pred_proba_rf = rf.predict_proba(X_test)"
      ],
      "id": "EnrNcArDO3Br",
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r4tAEt0uO_pw",
        "outputId": "6f62ce6b-c420-483c-c938-c47b72a609e7"
      },
      "source": [
        "# validation loss\n",
        "log_loss(y_test, pred_proba_rf)"
      ],
      "id": "r4tAEt0uO_pw",
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.197472087051298"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "548b703c-575d-4ecc-99bd-64d6e6ceeba9"
      },
      "source": [
        "## Competition Test Data\n",
        "\n",
        "In this part we will load the competition test data (which does not have labels) and predict the crop class for each field"
      ],
      "id": "548b703c-575d-4ecc-99bd-64d6e6ceeba9"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5f67db15-3866-4641-92cf-c44af29b1396"
      },
      "source": [
        "tile_ids_test = competition_test_df['tile_id'].unique()"
      ],
      "id": "5f67db15-3866-4641-92cf-c44af29b1396",
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "565e5451-982a-4cda-9de9-4c939203cdd6"
      },
      "source": [
        "X_competition_test = np.empty((0, 14), np.uint8)\n",
        "field_ids_test = np.empty((0, 1), np.uint32)\n",
        "\n",
        "\n",
        "for tile_id in tile_ids_test:\n",
        "    tile_df = competition_test_df[competition_test_df['tile_id']==tile_id]\n",
        "    \n",
        "    field_id_src = rasterio.open(tile_df[tile_df['asset']=='field_ids']['file_path'].values[0])\n",
        "    field_id_array = field_id_src.read(1).flatten()\n",
        "    my_test_idx = np.where(field_id_array!=0)[0]\n",
        "    field_ids_test = np.append(field_ids_test, field_id_array[my_test_idx])\n",
        "    \n",
        "    tile_date_times = tile_df[tile_df['satellite_platform']=='s1']['datetime'].unique()\n",
        "    \n",
        "    X_tile = np.empty((len(my_test_idx), 0), np.uint8)\n",
        "              \n",
        "    if(len(tile_date_times))>36:\n",
        "      for date_time in tile_date_times[ :40: 6]:\n",
        "          vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "          vv_array = np.expand_dims(vv_src.read(1).flatten()[my_test_idx], axis=1)\n",
        "          \n",
        "          vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "          vh_array = np.expand_dims(vh_src.read(1).flatten()[my_test_idx], axis=1)\n",
        "          \n",
        "          X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "          X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "    else:\n",
        "      for date_time in tile_date_times[ :20: 3]:\n",
        "          vv_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VV')]['file_path'].values[0])\n",
        "          vv_array = np.expand_dims(vv_src.read(1).flatten()[my_test_idx], axis=1)\n",
        "          \n",
        "          vh_src = rasterio.open(tile_df[(tile_df['datetime']==date_time) & (tile_df['asset']=='VH')]['file_path'].values[0])\n",
        "          vh_array = np.expand_dims(vh_src.read(1).flatten()[my_test_idx], axis=1)\n",
        "          \n",
        "          X_tile = np.append(X_tile, vv_array, axis = 1)\n",
        "          X_tile = np.append(X_tile, vh_array, axis = 1)\n",
        "        \n",
        "    X_competition_test = np.append(X_competition_test, X_tile, axis=0)"
      ],
      "id": "565e5451-982a-4cda-9de9-4c939203cdd6",
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g333NsTKw04N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c26b48a6-db6e-4f20-c209-f54a3d00ee24"
      },
      "source": [
        "X_competition_test.shape"
      ],
      "id": "g333NsTKw04N",
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(27572137, 14)"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5a3f26cd-a299-44aa-99b3-7ae562d7564d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "3886f634-5b27-4776-adb1-5e5a54f4c4d5"
      },
      "source": [
        "data_test = pd.DataFrame(X_competition_test)\n",
        "data_test['field_id'] = field_ids_test\n",
        "data_test.head()"
      ],
      "id": "5a3f26cd-a299-44aa-99b3-7ae562d7564d",
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>field_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>27</td>\n",
              "      <td>3</td>\n",
              "      <td>102896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>15</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>2</td>\n",
              "      <td>15</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>27</td>\n",
              "      <td>3</td>\n",
              "      <td>102896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>15</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>27</td>\n",
              "      <td>2</td>\n",
              "      <td>102896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>15</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>2</td>\n",
              "      <td>102896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "      <td>25</td>\n",
              "      <td>1</td>\n",
              "      <td>24</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>35</td>\n",
              "      <td>4</td>\n",
              "      <td>102896</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   0  1  2  3   4  5   6  7   8  9  10  11  12  13  field_id\n",
              "0  8  1  4  1  11  2  11  2   7  5   3   4  27   3    102896\n",
              "1  3  0  3  1  15  0  13  2  15  2   3   4  27   3    102896\n",
              "2  3  0  4  1   9  0   7  1  15  2   3   0  27   2    102896\n",
              "3  6  0  7  2   8  0   5  2  15  2   5   0  33   2    102896\n",
              "4  5  1  2  1  12  1  25  1  24  4   5   1  35   4    102896"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sYLPsxJYwMSf"
      },
      "source": [
        "del X_competition_test"
      ],
      "id": "sYLPsxJYwMSf",
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Dhu5L5UKU5V"
      },
      "source": [
        "#Memory optimised way: perform group-by on individual column \n",
        "GP_DATA = data_test\n",
        "\n",
        "# create instances of the group_by_atrribute object for each cloumn  \n",
        "gb_objs_test = [group_by_attribute() for i in range(len(GP_DATA.columns)-1) ]\n",
        "\n",
        "# perform group-by on individual column by passing column name and dataframe\n",
        "for i,j in enumerate(GP_DATA.columns[:-1]):\n",
        "  gb_objs_test[i].compute(GP_DATA,j)\n",
        "\n",
        "# perform concatination on field_id\n",
        "from functools import reduce \n",
        "df_final = [i.gb for i in gb_objs_test] \n",
        "\n",
        "df_merged_test = reduce(lambda left,right: pd.merge(left,right,on='field_id'), df_final)"
      ],
      "id": "6Dhu5L5UKU5V",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wsiFfHrkxBqk"
      },
      "source": [
        "df_merged_test = df_merged_test.reset_index()"
      ],
      "id": "wsiFfHrkxBqk",
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97ded63f-3f5e-497b-a10a-05882d466bc6"
      },
      "source": [
        "y_competition_prob = rf.predict_proba(df_merged_test.drop(columns=['field_id']))"
      ],
      "id": "97ded63f-3f5e-497b-a10a-05882d466bc6",
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1cf3be09-3bbd-440d-bce8-94a665e60602",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "8199e2a0-9772-4e3e-fba5-6f8ea5a2d0fe"
      },
      "source": [
        "# In this part we format the DataFrame to have column names and order similar to the sample submission file. \n",
        "pred_df = pd.DataFrame(y_competition_prob)\n",
        "pred_df = pred_df.rename(columns={\n",
        "  7:'Crop_ID_1',\n",
        "  2:'Crop_ID_2',\n",
        "  0:'Crop_ID_3',\n",
        "  1:'Crop_ID_4',\n",
        "  8:'Crop_ID_5',\n",
        "  5:'Crop_ID_6',\n",
        "  4:'Crop_ID_7',\n",
        "  6:'Crop_ID_8',\n",
        "  3:'Crop_ID_9'\n",
        "})\n",
        "\n",
        "pred_df['field_id']=df_merged_test['field_id']\n",
        "pred_df = pred_df[['field_id', 'Crop_ID_1', 'Crop_ID_2', 'Crop_ID_3', 'Crop_ID_4', 'Crop_ID_5', 'Crop_ID_6', 'Crop_ID_7', 'Crop_ID_8', 'Crop_ID_9']]\n",
        "pred_df"
      ],
      "id": "1cf3be09-3bbd-440d-bce8-94a665e60602",
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>field_id</th>\n",
              "      <th>Crop_ID_1</th>\n",
              "      <th>Crop_ID_2</th>\n",
              "      <th>Crop_ID_3</th>\n",
              "      <th>Crop_ID_4</th>\n",
              "      <th>Crop_ID_5</th>\n",
              "      <th>Crop_ID_6</th>\n",
              "      <th>Crop_ID_7</th>\n",
              "      <th>Crop_ID_8</th>\n",
              "      <th>Crop_ID_9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5</td>\n",
              "      <td>0.001048</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000516</td>\n",
              "      <td>0.009696</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.042772</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.943997</td>\n",
              "      <td>0.001972</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10</td>\n",
              "      <td>0.002167</td>\n",
              "      <td>0.149491</td>\n",
              "      <td>0.097280</td>\n",
              "      <td>0.194443</td>\n",
              "      <td>0.137910</td>\n",
              "      <td>0.123892</td>\n",
              "      <td>0.222862</td>\n",
              "      <td>0.020788</td>\n",
              "      <td>0.051167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>11</td>\n",
              "      <td>0.004871</td>\n",
              "      <td>0.085225</td>\n",
              "      <td>0.093828</td>\n",
              "      <td>0.439713</td>\n",
              "      <td>0.037440</td>\n",
              "      <td>0.169821</td>\n",
              "      <td>0.109271</td>\n",
              "      <td>0.045760</td>\n",
              "      <td>0.014071</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>17</td>\n",
              "      <td>0.012396</td>\n",
              "      <td>0.121238</td>\n",
              "      <td>0.261770</td>\n",
              "      <td>0.177297</td>\n",
              "      <td>0.052882</td>\n",
              "      <td>0.137716</td>\n",
              "      <td>0.130472</td>\n",
              "      <td>0.099076</td>\n",
              "      <td>0.007153</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>18</td>\n",
              "      <td>0.002639</td>\n",
              "      <td>0.084294</td>\n",
              "      <td>0.070540</td>\n",
              "      <td>0.413510</td>\n",
              "      <td>0.090041</td>\n",
              "      <td>0.098374</td>\n",
              "      <td>0.165399</td>\n",
              "      <td>0.047877</td>\n",
              "      <td>0.027325</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35290</th>\n",
              "      <td>122722</td>\n",
              "      <td>0.003238</td>\n",
              "      <td>0.682277</td>\n",
              "      <td>0.019072</td>\n",
              "      <td>0.035698</td>\n",
              "      <td>0.104559</td>\n",
              "      <td>0.054050</td>\n",
              "      <td>0.095638</td>\n",
              "      <td>0.004601</td>\n",
              "      <td>0.000868</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35291</th>\n",
              "      <td>122724</td>\n",
              "      <td>0.005655</td>\n",
              "      <td>0.147426</td>\n",
              "      <td>0.132404</td>\n",
              "      <td>0.252480</td>\n",
              "      <td>0.086987</td>\n",
              "      <td>0.145288</td>\n",
              "      <td>0.117645</td>\n",
              "      <td>0.026703</td>\n",
              "      <td>0.085412</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35292</th>\n",
              "      <td>122726</td>\n",
              "      <td>0.010275</td>\n",
              "      <td>0.009046</td>\n",
              "      <td>0.020239</td>\n",
              "      <td>0.010620</td>\n",
              "      <td>0.000303</td>\n",
              "      <td>0.111707</td>\n",
              "      <td>0.001866</td>\n",
              "      <td>0.834406</td>\n",
              "      <td>0.001538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35293</th>\n",
              "      <td>122730</td>\n",
              "      <td>0.002362</td>\n",
              "      <td>0.066975</td>\n",
              "      <td>0.104546</td>\n",
              "      <td>0.462595</td>\n",
              "      <td>0.036267</td>\n",
              "      <td>0.110957</td>\n",
              "      <td>0.100462</td>\n",
              "      <td>0.044770</td>\n",
              "      <td>0.071064</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35294</th>\n",
              "      <td>122734</td>\n",
              "      <td>0.010836</td>\n",
              "      <td>0.002050</td>\n",
              "      <td>0.003987</td>\n",
              "      <td>0.011894</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.201459</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.734661</td>\n",
              "      <td>0.035113</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35295 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       field_id  Crop_ID_1  Crop_ID_2  ...  Crop_ID_7  Crop_ID_8  Crop_ID_9\n",
              "0             5   0.001048   0.000000  ...   0.000000   0.943997   0.001972\n",
              "1            10   0.002167   0.149491  ...   0.222862   0.020788   0.051167\n",
              "2            11   0.004871   0.085225  ...   0.109271   0.045760   0.014071\n",
              "3            17   0.012396   0.121238  ...   0.130472   0.099076   0.007153\n",
              "4            18   0.002639   0.084294  ...   0.165399   0.047877   0.027325\n",
              "...         ...        ...        ...  ...        ...        ...        ...\n",
              "35290    122722   0.003238   0.682277  ...   0.095638   0.004601   0.000868\n",
              "35291    122724   0.005655   0.147426  ...   0.117645   0.026703   0.085412\n",
              "35292    122726   0.010275   0.009046  ...   0.001866   0.834406   0.001538\n",
              "35293    122730   0.002362   0.066975  ...   0.100462   0.044770   0.071064\n",
              "35294    122734   0.010836   0.002050  ...   0.000000   0.734661   0.035113\n",
              "\n",
              "[35295 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4541f43b-520a-4f12-b797-3b64630a6527"
      },
      "source": [
        "# Write the predicted probabilites to a csv for submission\n",
        "pred_df.to_csv('submission.csv', index=False)"
      ],
      "id": "4541f43b-520a-4f12-b797-3b64630a6527",
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "awWJJXuYRT0e"
      },
      "source": [
        "# Public leaderboard 5th position \n",
        "![]()\n",
        "<img src='https://i.imgur.com/1AXyWPz.png' alt='Public leaderboard 5th position ' width='1500'/>"
      ],
      "id": "awWJJXuYRT0e"
    }
  ]
}