worker-Water.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Debug notebook\n",
    "This notebook is only used to troubleshoot the Water Classification workflow processor."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Water Classification workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from utils.genprepWater import per_scene_wofs\n",
    "\n",
    "def process_scene(json_data):\n",
    "    loaded_json = json.loads(json_data)\n",
    "    #for x in loaded_json:\n",
    "    #    print(\"%s: %s\" % (x, loaded_json[x]))\n",
    "    per_scene_wofs(**loaded_json)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Water Classification job queue processor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Worker with sessionID: b18c0f7c-f4a4-40ed-a261-8227138fa36e\n",
      "Initial queue state: empty=False\n"
     ]
    }
   ],
   "source": [
    "import rediswq\n",
    "\n",
    "import os\n",
    "host = os.getenv(\"REDIS_SERVICE_HOST\", \"redis-master\")\n",
    "\n",
    "q = rediswq.RedisWQ(name=\"jobWater\", host=host)\n",
    "print(\"Worker with sessionID: \" +  q.sessionID())\n",
    "print(\"Initial queue state: empty=\" + str(q.empty()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on {\"optical_yaml_path\": \"common_sensing/fiji/landsat_8/LC08_L1TP_072069_20130522/datacube-metadata.yaml\", \"s3_bucket\": \"public-eo-data\", \"s3_dir\": \"luigi/landsat_8_wofs_testing/\", \"inter_dir\": \"/tmp/data/intermediate/\"}\n",
      "LANDSAT_8 ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'pixel_qa']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/notebooks/utils/dc_water_classifier.py:119: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  return (a - b) / (a + b)\n",
      "/opt/notebooks/utils/dc_water_classifier.py:139: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r1 = ndi_52 <= -0.01\n",
      "/opt/notebooks/utils/dc_water_classifier.py:141: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r2 = band1 <= 2083.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:144: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r3 = band7 <= 323.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:149: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r4 = ndi_43 <= 0.61\n",
      "/opt/notebooks/utils/dc_water_classifier.py:153: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r5 = band1 <= 1400.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:156: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r6 = ndi_43 <= -0.01\n",
      "/opt/notebooks/utils/dc_water_classifier.py:162: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r7 = ndi_72 <= -0.23\n",
      "/opt/notebooks/utils/dc_water_classifier.py:165: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r8 = band1 <= 379\n",
      "/opt/notebooks/utils/dc_water_classifier.py:171: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r9 = ndi_43 <= 0.22\n",
      "/opt/notebooks/utils/dc_water_classifier.py:175: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r10 = band1 <= 473\n",
      "/opt/notebooks/utils/dc_water_classifier.py:186: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r11 = ndi_52 <= 0.23\n",
      "/opt/notebooks/utils/dc_water_classifier.py:189: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r12 = band1 <= 334.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:195: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r13 = ndi_43 <= 0.54\n",
      "/opt/notebooks/utils/dc_water_classifier.py:201: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r14 = ndi_52 <= 0.12\n",
      "/opt/notebooks/utils/dc_water_classifier.py:207: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r15 = band3 <= 364.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:210: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r16 = band1 <= 129.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:216: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r17 = band1 <= 300.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:224: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r18 = ndi_52 <= 0.34\n",
      "/opt/notebooks/utils/dc_water_classifier.py:228: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r19 = band1 <= 249.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:232: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r20 = ndi_43 <= 0.45\n",
      "/opt/notebooks/utils/dc_water_classifier.py:236: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r21 = band3 <= 364.5\n",
      "/opt/notebooks/utils/dc_water_classifier.py:240: RuntimeWarning: invalid value encountered in less_equal\n",
      "  r22 = band1 <= 129.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1GT_072069_20130522_20180202_01_T2_waternc.tif /tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1TP_072069_20130522/LC08_L1GT_072069_20130522_20180202_01_T2_water.tif\n",
      "Preparing scene LC08_L1TP_072069_20130522\n",
      "Scene path /tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1TP_072069_20130522/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:root:Endpoint URL: http://s3-uk-1.sa-catapult.co.uk\n",
      "DEBUG:botocore.loaders:Loading JSON file: /opt/conda/lib/python3.7/site-packages/boto3/data/s3/2006-03-01/resources-1.json\n",
      "DEBUG:botocore.loaders:Loading JSON file: /opt/conda/lib/python3.7/site-packages/botocore/data/endpoints.json\n",
      "DEBUG:botocore.hooks:Event choose-service-name: calling handler <function handle_service_name_alias at 0x7fdd05a670d0>\n",
      "DEBUG:botocore.loaders:Loading JSON file: /opt/conda/lib/python3.7/site-packages/botocore/data/s3/2006-03-01/service-2.json\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function add_generate_presigned_post at 0x7fdd05a50730>\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function lazy_call.<locals>._handler at 0x7fdd00c659d8>\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function add_generate_presigned_url at 0x7fdd05a50510>\n",
      "DEBUG:botocore.endpoint:Setting s3 timeout as (60, 60)\n",
      "DEBUG:botocore.loaders:Loading JSON file: /opt/conda/lib/python3.7/site-packages/botocore/data/_retry.json\n",
      "DEBUG:botocore.client:Registering retry handlers for service: s3\n",
      "DEBUG:boto3.resources.factory:Loading s3:s3\n",
      "DEBUG:boto3.resources.factory:Loading s3:Bucket\n",
      "DEBUG:boto3.resources.model:Renaming Bucket attribute name\n",
      "DEBUG:botocore.hooks:Event creating-resource-class.s3.Bucket: calling handler <function lazy_call.<locals>._handler at 0x7fdd00fe1f28>\n",
      "DEBUG:botocore.hooks:Event choose-service-name: calling handler <function handle_service_name_alias at 0x7fdd05a670d0>\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function add_generate_presigned_post at 0x7fdd05a50730>\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function lazy_call.<locals>._handler at 0x7fdd01d32ae8>\n",
      "DEBUG:botocore.hooks:Event creating-client-class.s3: calling handler <function add_generate_presigned_url at 0x7fdd05a50510>\n",
      "DEBUG:botocore.endpoint:Setting s3 timeout as (60, 60)\n",
      "DEBUG:botocore.client:Registering retry handlers for service: s3\n",
      "INFO:root:Local source file: /tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1TP_072069_20130522/log_file.txt\n",
      "INFO:root:S3 target file: luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt\n",
      "INFO:root:Start: /tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1TP_072069_20130522/log_file.txt 2020-01-30 13:47:15\n",
      "DEBUG:s3transfer.utils:Acquiring 0\n",
      "DEBUG:s3transfer.tasks:UploadSubmissionTask(transfer_id=0, {'transfer_future': <s3transfer.futures.TransferFuture object at 0x7fdd01752ac8>}) about to wait for the following futures []\n",
      "DEBUG:s3transfer.tasks:UploadSubmissionTask(transfer_id=0, {'transfer_future': <s3transfer.futures.TransferFuture object at 0x7fdd01752ac8>}) done waiting for dependent futures\n",
      "DEBUG:s3transfer.tasks:Executing task UploadSubmissionTask(transfer_id=0, {'transfer_future': <s3transfer.futures.TransferFuture object at 0x7fdd01752ac8>}) with kwargs {'client': <botocore.client.S3 object at 0x7fdd0178cac8>, 'config': <boto3.s3.transfer.TransferConfig object at 0x7fdd01efad68>, 'osutil': <s3transfer.utils.OSUtils object at 0x7fdd0178cfd0>, 'request_executor': <s3transfer.futures.BoundedExecutor object at 0x7fdd01752630>, 'transfer_future': <s3transfer.futures.TransferFuture object at 0x7fdd01752ac8>}\n",
      "DEBUG:s3transfer.futures:Submitting task PutObjectTask(transfer_id=0, {'bucket': 'public-eo-data', 'key': 'luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'extra_args': {}}) to executor <s3transfer.futures.BoundedExecutor object at 0x7fdd01752630> for transfer request: 0.\n",
      "DEBUG:s3transfer.utils:Acquiring 0\n",
      "DEBUG:s3transfer.tasks:PutObjectTask(transfer_id=0, {'bucket': 'public-eo-data', 'key': 'luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'extra_args': {}}) about to wait for the following futures []\n",
      "DEBUG:s3transfer.utils:Releasing acquire 0/None\n",
      "DEBUG:s3transfer.tasks:PutObjectTask(transfer_id=0, {'bucket': 'public-eo-data', 'key': 'luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'extra_args': {}}) done waiting for dependent futures\n",
      "DEBUG:s3transfer.tasks:Executing task PutObjectTask(transfer_id=0, {'bucket': 'public-eo-data', 'key': 'luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'extra_args': {}}) with kwargs {'client': <botocore.client.S3 object at 0x7fdd0178cac8>, 'fileobj': <s3transfer.utils.ReadFileChunk object at 0x7fdd01752cf8>, 'bucket': 'public-eo-data', 'key': 'luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'extra_args': {}}\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <function validate_ascii_metadata at 0x7fdd05a05598>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <function sse_md5 at 0x7fdd05a029d8>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <function convert_body_to_file_like_object at 0x7fdd05a05e18>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <function validate_bucket_name at 0x7fdd05a02950>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <bound method S3RegionRedirector.redirect_from_cache of <botocore.utils.S3RegionRedirector object at 0x7fdd0178cf28>>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <bound method S3ArnParamHandler.handle_arn of <botocore.utils.S3ArnParamHandler object at 0x7fdd01752358>>\n",
      "DEBUG:botocore.hooks:Event before-parameter-build.s3.PutObject: calling handler <function generate_idempotent_uuid at 0x7fdd05a02598>\n",
      "DEBUG:botocore.hooks:Event before-call.s3.PutObject: calling handler <function conditionally_calculate_md5 at 0x7fdd05a028c8>\n",
      "DEBUG:botocore.hooks:Event before-call.s3.PutObject: calling handler <function add_expect_header at 0x7fdd05a02c80>\n",
      "DEBUG:botocore.handlers:Adding expect 100 continue header to request.\n",
      "DEBUG:botocore.hooks:Event before-call.s3.PutObject: calling handler <bound method S3RegionRedirector.set_request_url of <botocore.utils.S3RegionRedirector object at 0x7fdd0178cf28>>\n",
      "DEBUG:botocore.hooks:Event before-call.s3.PutObject: calling handler <function inject_api_version_header_if_needed at 0x7fdd05a05f28>\n",
      "DEBUG:botocore.endpoint:Making request for OperationModel(name=PutObject) with params: {'url_path': '/public-eo-data/luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'query_string': {}, 'method': 'PUT', 'headers': {'User-Agent': 'Boto3/1.11.0 Python/3.7.3 Linux/4.4.0-139-generic Botocore/1.14.0', 'Content-MD5': 'xzy3GB7kMLjTDkpkO0/d4w==', 'Expect': '100-continue'}, 'body': <s3transfer.utils.ReadFileChunk object at 0x7fdd01752cf8>, 'url': 'http://s3-uk-1.sa-catapult.co.uk/public-eo-data/luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt', 'context': {'client_region': 'us-east-1', 'client_config': <botocore.config.Config object at 0x7fdd0178cc18>, 'has_streaming_input': True, 'auth_type': None, 'signing': {'bucket': 'public-eo-data'}}}\n",
      "DEBUG:botocore.hooks:Event request-created.s3.PutObject: calling handler <function signal_not_transferring at 0x7fdd01d96950>\n",
      "DEBUG:botocore.hooks:Event request-created.s3.PutObject: calling handler <bound method RequestSigner.handler of <botocore.signers.RequestSigner object at 0x7fdd0178cbe0>>\n",
      "DEBUG:botocore.hooks:Event choose-signer.s3.PutObject: calling handler <bound method ClientCreator._default_s3_presign_to_sigv2 of <botocore.client.ClientCreator object at 0x7fdd017da400>>\n",
      "DEBUG:botocore.hooks:Event choose-signer.s3.PutObject: calling handler <function set_operation_specific_signer at 0x7fdd05a02488>\n",
      "DEBUG:botocore.hooks:Event before-sign.s3.PutObject: calling handler <bound method S3EndpointSetter.set_endpoint of <botocore.utils.S3EndpointSetter object at 0x7fdd017523c8>>\n",
      "DEBUG:botocore.utils:Using S3 path style addressing.\n",
      "DEBUG:botocore.auth:Calculating signature using v4 auth.\n",
      "DEBUG:botocore.auth:CanonicalRequest:\n",
      "PUT\n",
      "/public-eo-data/luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt\n",
      "\n",
      "content-md5:xzy3GB7kMLjTDkpkO0/d4w==\n",
      "host:s3-uk-1.sa-catapult.co.uk\n",
      "x-amz-content-sha256:8b63d44aab9c06a9c3154deadb40a0320171025a9240602730a3100d1855592f\n",
      "x-amz-date:20200130T134715Z\n",
      "\n",
      "content-md5;host;x-amz-content-sha256;x-amz-date\n",
      "8b63d44aab9c06a9c3154deadb40a0320171025a9240602730a3100d1855592f\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:botocore.auth:StringToSign:\n",
      "AWS4-HMAC-SHA256\n",
      "20200130T134715Z\n",
      "20200130/us-east-1/s3/aws4_request\n",
      "19d20f1dd62dedff38c73a9ace0b46f99b134574da2fa651371bd745856ee3f4\n",
      "DEBUG:botocore.auth:Signature:\n",
      "e3d75dbc12385f7743399f68b3183a6e75fab6e445032ecf396042ba62e31bb9\n",
      "DEBUG:botocore.hooks:Event request-created.s3.PutObject: calling handler <function signal_transferring at 0x7fdd01d96d08>\n",
      "DEBUG:botocore.endpoint:Sending http request: <AWSPreparedRequest stream_output=False, method=PUT, url=http://s3-uk-1.sa-catapult.co.uk/public-eo-data/luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt, headers={'User-Agent': b'Boto3/1.11.0 Python/3.7.3 Linux/4.4.0-139-generic Botocore/1.14.0', 'Content-MD5': b'xzy3GB7kMLjTDkpkO0/d4w==', 'Expect': b'100-continue', 'X-Amz-Date': b'20200130T134715Z', 'X-Amz-Content-SHA256': b'8b63d44aab9c06a9c3154deadb40a0320171025a9240602730a3100d1855592f', 'Authorization': b'AWS4-HMAC-SHA256 Credential=16c353e3c1673eb9556e/20200130/us-east-1/s3/aws4_request, SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date, Signature=e3d75dbc12385f7743399f68b3183a6e75fab6e445032ecf396042ba62e31bb9', 'Content-Length': '1531139'}>\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): s3-uk-1.sa-catapult.co.uk:80\n",
      "DEBUG:botocore.awsrequest:Waiting for 100 Continue response.\n",
      "DEBUG:botocore.awsrequest:100 Continue response seen, now sending request body.\n",
      "DEBUG:urllib3.connectionpool:http://s3-uk-1.sa-catapult.co.uk:80 \"PUT /public-eo-data/luigi/landsat_8_wofs_testing/LC08_L1TP_072069_20130522/log_file.txt HTTP/1.1\" 200 0\n",
      "DEBUG:botocore.parsers:Response headers: {'Date': 'Thu, 30 Jan 2020 13:47:15 GMT', 'x-amz-request-id': '0d4f01a0-9f53-1a25-934e-d8c4970cd77c', 'ETag': '\"c73cb7181ee430b8d30e4a643b4fdde3\"', 'Content-Length': '0', 'Server': 'CloudianS3'}\n",
      "DEBUG:botocore.parsers:Response body:\n",
      "b''\n",
      "DEBUG:botocore.hooks:Event needs-retry.s3.PutObject: calling handler <botocore.retryhandler.RetryHandler object at 0x7fdd0178cf60>\n",
      "DEBUG:botocore.retryhandler:No retry needed.\n",
      "DEBUG:botocore.hooks:Event needs-retry.s3.PutObject: calling handler <bound method S3RegionRedirector.redirect_from_error of <botocore.utils.S3RegionRedirector object at 0x7fdd0178cf28>>\n",
      "DEBUG:s3transfer.utils:Releasing acquire 0/None\n",
      "INFO:root:Finish: /tmp/data/intermediate/LC08_L1TP_072069_20130522_tmp/LC08_L1TP_072069_20130522/log_file.txt 2020-01-30 13:47:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not boo\n"
     ]
    }
   ],
   "source": [
    "while not q.empty():\n",
    "    item = q.lease(lease_secs=1800, block=True, timeout=600)\n",
    "    if item is not None:\n",
    "        itemstr = item.decode(\"utf=8\")\n",
    "        print(\"Working on \" + itemstr)\n",
    "        #time.sleep(10) # Put your actual work here instead of sleep.\n",
    "        process_scene(itemstr)\n",
    "        q.complete(item)\n",
    "    else:\n",
    "        print(\"Waiting for work\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queue empty!\n"
     ]
    }
   ],
   "source": [
    "print(\"Queue empty!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}