resources/south-africa/orchestrator/elwazi-pilot-node-tests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "596a980a-391f-4594-bc1e-43c1c4c091dd",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "# Launching WES workflow on DRS objects based on a Data Connect query"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a94617fb-aa83-4c9b-abb6-818ba4e120e1",
   "metadata": {},
   "source": [
    "## Check if endpoints are available"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6521c800-c984-4bd5-8f67-888757783ba2",
   "metadata": {},
   "source": [
    "1) Check if WES endpoints are available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c2c5e3f7-110a-4c7b-89b0-cb06a21a6dd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "def pretty_print_json(response):\n",
    "    # pretty print JSON in blue color\n",
    "    print(\"\\033[38;2;8;75;138m\"+json.dumps(response.json(), indent=4)+\"\\033[0m\")\n",
    "    \n",
    "def print_head(text):\n",
    "    # print in green color\n",
    "    print(\"\\033[38;2;8;138;75m\"+text+\"\\033[0m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b8180a74-aa71-407c-84e8-b38904adbd31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.wes\",\n",
      "    \"name\": \"GA4GH Starter Kit WES Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Workflow Execution Service (WES)API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-wes\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"wes\",\n",
      "        \"version\": \"1.0.1\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    },\n",
      "    \"workflow_type_versions\": {\n",
      "        \"WDL\": [\n",
      "            \"1.0\"\n",
      "        ],\n",
      "        \"NEXTFLOW\": [\n",
      "            \"21.04.0\"\n",
      "        ]\n",
      "    },\n",
      "    \"workflow_engine_versions\": {\n",
      "        \"NATIVE\": \"1.0.0\"\n",
      "    }\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.wes\",\n",
      "    \"name\": \"GA4GH Starter Kit WES Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Workflow Execution Service (WES)API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-wes\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"wes\",\n",
      "        \"version\": \"1.0.1\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    },\n",
      "    \"workflow_type_versions\": {\n",
      "        \"WDL\": [\n",
      "            \"1.0\"\n",
      "        ],\n",
      "        \"NEXTFLOW\": [\n",
      "            \"21.04.0\"\n",
      "        ]\n",
      "    },\n",
      "    \"workflow_engine_versions\": {\n",
      "        \"NATIVE\": \"1.0.0\"\n",
      "    }\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.wes\",\n",
      "    \"name\": \"GA4GH Starter Kit WES Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Workflow Execution Service (WES)API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-wes\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"wes\",\n",
      "        \"version\": \"1.0.1\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    },\n",
      "    \"workflow_type_versions\": {\n",
      "        \"WDL\": [\n",
      "            \"1.0\"\n",
      "        ],\n",
      "        \"NEXTFLOW\": [\n",
      "            \"21.04.0\"\n",
      "        ]\n",
      "    },\n",
      "    \"workflow_engine_versions\": {\n",
      "        \"NATIVE\": \"1.0.0\"\n",
      "    }\n",
      "}\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "node_ips = ['ga4gh-starter-kit.ilifu.ac.za','osdp.ace.ac.ug','elwazi-node.icermali.org' ] #, '196.43.136.22'] \n",
    "wes_port = \"6000\"\n",
    "service_info_path = \"/service-info\"\n",
    "runs_path = \"/runs\"\n",
    "http_method = \"GET\"\n",
    "\n",
    "for node_ip in node_ips:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    wes_base_url = ga4gh_base_url.format(wes_port,\"wes\")\n",
    "    request_url = wes_base_url+service_info_path\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to service-info endpoint\n",
    "    wes_service_info_resp = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(wes_service_info_resp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbb2920d-4d39-4521-9ed8-5a2c9893bd92",
   "metadata": {},
   "source": [
    "2) Check if DRS endpoints are available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e99b4822-8d2a-4447-8bb7-db4618921199",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.drs\",\n",
      "    \"name\": \"GA4GH Starter Kit DRS Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Data Repository Service (DRS) API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-drs\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"drs\",\n",
      "        \"version\": \"1.3.0experimental\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    }\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.drs\",\n",
      "    \"name\": \"GA4GH Starter Kit DRS Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Data Repository Service (DRS) API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-drs\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"drs\",\n",
      "        \"version\": \"1.3.0experimental\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    }\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/service-info\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"org.ga4gh.starterkit.drs\",\n",
      "    \"name\": \"GA4GH Starter Kit DRS Service\",\n",
      "    \"description\": \"An open source, community-driven implementation of the GA4GH Data Repository Service (DRS) API specification.\",\n",
      "    \"contactUrl\": \"mailto:info@ga4gh.org\",\n",
      "    \"documentationUrl\": \"https://github.com/ga4gh/ga4gh-starter-kit-drs\",\n",
      "    \"createdAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"updatedAt\": \"2020-01-15T12:00:00Z\",\n",
      "    \"environment\": \"test\",\n",
      "    \"version\": \"0.3.2\",\n",
      "    \"type\": {\n",
      "        \"group\": \"org.ga4gh\",\n",
      "        \"artifact\": \"drs\",\n",
      "        \"version\": \"1.3.0experimental\"\n",
      "    },\n",
      "    \"organization\": {\n",
      "        \"name\": \"Global Alliance for Genomics and Health\",\n",
      "        \"url\": \"https://ga4gh.org\"\n",
      "    }\n",
      "}\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "node_ips = ['ga4gh-starter-kit.ilifu.ac.za','osdp.ace.ac.ug','elwazi-node.icermali.org'] #, '196.43.136.22'] \n",
    "drs_port = \"5000\"\n",
    "service_info_path = \"/service-info\"\n",
    "http_method = \"GET\"\n",
    "\n",
    "for node_ip in node_ips:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "    request_url = drs_base_url+service_info_path\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to service-info endpoint\n",
    "    drs_service_info_resp = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(drs_service_info_resp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1f89e9f-1653-4e96-bdcc-d9826bbb978d",
   "metadata": {},
   "source": [
    "## Check if DRS objects exists on endpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaa4a7d5-f7cb-4227-812e-1b16f85feba9",
   "metadata": {},
   "source": [
    "1. Check ga4gh-starter-kit.ilifu.ac.za"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5eb5b2f5-d57b-493a-8000-c6ffad3e7ac4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/objects/c542689dba1e54669335c8e25abe6207\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"c542689dba1e54669335c8e25abe6207\",\n",
      "    \"description\": \"Patient: HG01857, Country: KHV, Region: EAS, Sex: female\\n\",\n",
      "    \"created_time\": \"2023-07-24T13:45:58Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01857.final.chrX_15494566-15607236\",\n",
      "    \"size\": 500309,\n",
      "    \"updated_time\": \"2023-07-24T13:45:58Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"70ef6da9822aecf071acda427a61e31cbbf8f25b\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"b9e9f9a1f85de2ee57ba05d3f75de865458c2645284e42deef20de38b9e6a37c\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"c542689dba1e54669335c8e25abe6207\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://ga4gh-starter-kit.ilifu.ac.za:5000/c542689dba1e54669335c8e25abe6207\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01857/HG01857.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"153d1624-68a0-42a3-8c69-77a1e168bef8\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/objects/5a436bec951fab59dd975bcd10f316f1\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"5a436bec951fab59dd975bcd10f316f1\",\n",
      "    \"description\": \"Patient: HG01883, Country: ACB, Region: AFR, Sex: female\\n\",\n",
      "    \"created_time\": \"2023-07-24T13:45:58Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01883.final.chrX_15494566-15607236\",\n",
      "    \"size\": 513233,\n",
      "    \"updated_time\": \"2023-07-24T13:45:58Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"789133eb2d6c421fd0fb753f9cbf8093788f5f25\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"0ee02c1db461b8d41381c65aba46ed72777d86ed4cd4ea6982dc5f974aabeb2f\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"5a436bec951fab59dd975bcd10f316f1\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://ga4gh-starter-kit.ilifu.ac.za:5000/5a436bec951fab59dd975bcd10f316f1\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01883/HG01883.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"f27703b2-c964-4f2d-b985-45547db6927d\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "http_method = \"GET\"\n",
    "node_ip = 'ga4gh-starter-kit.ilifu.ac.za'\n",
    "drs_port = \"5000\"\n",
    "drs_ids = ['c542689dba1e54669335c8e25abe6207','5a436bec951fab59dd975bcd10f316f1']\n",
    "object_path_get = \"/objects/{}\"\n",
    "object_path_post = \"/objects\"\n",
    "access_path = \"/objects/{}/access/{}\"\n",
    "\n",
    "for drs_id in drs_ids:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "    request_url = drs_base_url+object_path_get.format(drs_id)\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to /objects/{object_id} endpoint\n",
    "    drs_object_response = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(drs_object_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14adb260-3107-4791-841b-831c2052cd6e",
   "metadata": {},
   "source": [
    "2.1 Check osdp.ace.ac.ug (ACE - Uganda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "52ad5806-4704-472c-8e1e-5012e3d1ddb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/objects/6fa43c7de04b60c1a73a42aa2efc977d\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"6fa43c7de04b60c1a73a42aa2efc977d\",\n",
      "    \"description\": \"Patient: HG01879, Country: ACB, Region: AFR, Sex: male\\n\",\n",
      "    \"created_time\": \"2023-10-30T16:27:28Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01879.final.chrX_15494566-15607236\",\n",
      "    \"size\": 0,\n",
      "    \"updated_time\": \"2023-10-30T16:27:28Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"af30841a49bba8733cf1f070ff725dd9cdfc91f1\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"a61d4570f6d210220fd14b4f7744df46d6dce61df0da35782b528660def996f1\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"6fa43c7de04b60c1a73a42aa2efc977d\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://osdp.ace.ac.ug:5000/6fa43c7de04b60c1a73a42aa2efc977d\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01879/HG01879.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"2a441692-4ed4-4373-bd13-025a71634173\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/objects/be145a60bc059c154475a2561af0df6b\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"be145a60bc059c154475a2561af0df6b\",\n",
      "    \"description\": \"Patient: HG01885, Country: ACB, Region: AFR, Sex: male\\n\",\n",
      "    \"created_time\": \"2023-10-30T16:27:28Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01885.final.chrX_15494566-15607236\",\n",
      "    \"size\": 0,\n",
      "    \"updated_time\": \"2023-10-30T16:27:28Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"43594b1eb5e0119242897b9177a234bab9607459\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"bbda8eadf8373c1d6f9fc131f1084ea8443e1a575d150782a6d9033f2a06b709\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"be145a60bc059c154475a2561af0df6b\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://osdp.ace.ac.ug:5000/be145a60bc059c154475a2561af0df6b\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01885/HG01885.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"26cb7cb1-c589-4e0c-bb07-18b96cdb3809\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "http_method = \"GET\"\n",
    "node_ip = 'osdp.ace.ac.ug'\n",
    "drs_port = \"5000\"\n",
    "drs_ids = ['6fa43c7de04b60c1a73a42aa2efc977d','be145a60bc059c154475a2561af0df6b']\n",
    "object_path_get = \"/objects/{}\"\n",
    "object_path_post = \"/objects\"\n",
    "access_path = \"/objects/{}/access/{}\"\n",
    "\n",
    "for drs_id in drs_ids:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "    request_url = drs_base_url+object_path_get.format(drs_id)\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to /objects/{object_id} endpoint\n",
    "    drs_object_response = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(drs_object_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2062a93b",
   "metadata": {},
   "source": [
    "2.2 Check 196.43.136.22 (UVRI - Uganda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bace8cca",
   "metadata": {},
   "outputs": [],
   "source": [
    "http_method = \"GET\"\n",
    "node_ip = '196.43.136.22' #UVRI - Uganda\n",
    "drs_port = \"5000\"\n",
    "drs_ids = ['1050d0443c2e83f9d9a8933481dcb405','be04a5a90617aeae9a05fb533f544ade']\n",
    "object_path_get = \"/objects/{}\"\n",
    "object_path_post = \"/objects\"\n",
    "access_path = \"/objects/{}/access/{}\"\n",
    "\n",
    "for drs_id in drs_ids:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "    request_url = drs_base_url+object_path_get.format(drs_id)\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to /objects/{object_id} endpoint\n",
    "    drs_object_response = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(drs_object_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f720489-f871-4c82-b73d-3249ba10ea02",
   "metadata": {},
   "source": [
    "3. Check elwazi-node.icermali.org (Mali)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d1a4eecd-3f34-4157-8822-d721cdb3daaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/objects/a68c60133f942881983d0e15827bf88f\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"a68c60133f942881983d0e15827bf88f\",\n",
      "    \"description\": \"Patient: HG01880, Country: ACB, Region: AFR, Sex: female\\n\",\n",
      "    \"created_time\": \"2023-10-31T09:16:37Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01880.final.chrX_15494566-15607236\",\n",
      "    \"size\": 0,\n",
      "    \"updated_time\": \"2023-10-31T09:16:37Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"5747e183e64f2fe4d87da568fd30d0159086d1fd\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"50284d24e63c3c859f1e3d46a8aa54869e82f6fb37099926f2968fe7e64c15d7\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"a68c60133f942881983d0e15827bf88f\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://elwazi-node.icermali.org:5000/a68c60133f942881983d0e15827bf88f\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01880/HG01880.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"5d80f1ea-f04f-4f58-90da-f2a92ef71961\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/objects/168d353c6f474ca72e35e9209f921a59\u001b[0m\n",
      "\u001b[38;2;8;75;138m{\n",
      "    \"id\": \"168d353c6f474ca72e35e9209f921a59\",\n",
      "    \"description\": \"Patient: HG01890, Country: ACB, Region: AFR, Sex: male\\n\",\n",
      "    \"created_time\": \"2023-10-31T09:16:37Z\",\n",
      "    \"mime_type\": \"application/cram\",\n",
      "    \"name\": \"HG01890.final.chrX_15494566-15607236\",\n",
      "    \"size\": 0,\n",
      "    \"updated_time\": \"2023-10-31T09:16:37Z\",\n",
      "    \"checksums\": [\n",
      "        {\n",
      "            \"checksum\": \"d1ed97df457f1fc8fa5cf5fd2ecd0b1fa6560996\",\n",
      "            \"type\": \"sha1\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"cba436e789297b20b642be7d1cf6a4363d9694019f907a0d41de4a239b543f5c\",\n",
      "            \"type\": \"sha256\"\n",
      "        },\n",
      "        {\n",
      "            \"checksum\": \"168d353c6f474ca72e35e9209f921a59\",\n",
      "            \"type\": \"md5\"\n",
      "        }\n",
      "    ],\n",
      "    \"self_uri\": \"drs://elwazi-node.icermali.org:5000/168d353c6f474ca72e35e9209f921a59\",\n",
      "    \"access_methods\": [\n",
      "        {\n",
      "            \"access_url\": {\n",
      "                \"url\": \"file:///share/elwazi/crams/HG01890/HG01890.final.chrX_15494566-15607236.cram\"\n",
      "            },\n",
      "            \"type\": \"file\"\n",
      "        },\n",
      "        {\n",
      "            \"access_id\": \"e00951c9-66f2-4932-97dc-4a11b8ed6dde\",\n",
      "            \"type\": \"https\"\n",
      "        }\n",
      "    ]\n",
      "}\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "http_method = \"GET\"\n",
    "node_ip = 'elwazi-node.icermali.org'\n",
    "drs_port = \"5000\"\n",
    "drs_ids = ['a68c60133f942881983d0e15827bf88f','168d353c6f474ca72e35e9209f921a59']\n",
    "object_path_get = \"/objects/{}\"\n",
    "object_path_post = \"/objects\"\n",
    "access_path = \"/objects/{}/access/{}\"\n",
    "\n",
    "for drs_id in drs_ids:\n",
    "    ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "    drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "    request_url = drs_base_url+object_path_get.format(drs_id)\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "    # GET request to /objects/{object_id} endpoint\n",
    "    drs_object_response = requests.request(http_method, request_url)\n",
    "    # print the response\n",
    "    pretty_print_json(drs_object_response)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1e61f33-2d47-4e25-a55d-80abd80a5665",
   "metadata": {},
   "source": [
    "## Launch workflow on test DRS objects from South-Africa, Mali and Uganda (using WES on ga4gh-starter-kit.ilifu.ac.za, elwazi-node.icermali.org or osdp.ace.ac.ug and 196.43.136.22)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "927a320d-db5f-499e-b933-1b081af3c3f3",
   "metadata": {},
   "source": [
    "1. Launch workflow\n",
    "- Change the node_ip to run on a different WES endpoint\n",
    "- Change the infput_file to run on different DRS objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df195001-4adf-4142-8800-a6a548cf1859",
   "metadata": {},
   "outputs": [],
   "source": [
    "# node_ip = 'ga4gh-starter-kit.ilifu.ac.za'\n",
    "# node_ip = 'elwazi-node.icermali.org'\n",
    "node_ip = 'osdp.ace.ac.ug'\n",
    "#node_ip = '196.43.136.22' #UVRI\n",
    "wes_port = \"6000\"\n",
    "service_info_path = \"/service-info\"\n",
    "runs_path = \"/runs\"\n",
    "http_method = \"GET\"\n",
    "ga4gh_base_url = \"http://\" + node_ip + \":{}/ga4gh/{}/v1\"\n",
    "wes_base_url = ga4gh_base_url.format(wes_port,\"wes\")\n",
    "\n",
    "http_method = \"POST\"\n",
    "request_url = wes_base_url + runs_path\n",
    "\n",
    "nextflow_workflow_url = \"https://github.com/grbot/cram-qc\"\n",
    "# input_file = \"drs://ga4gh-starter-kit.ilifu.ac.za:5000/c542689dba1e54669335c8e25abe6207 drs://ga4gh-starter-kit.ilifu.ac.za:5000/5a436bec951fab59dd975bcd10f316f1\"\n",
    "# input_file = \"drs://elwazi-node.icermali.org:5000/a68c60133f942881983d0e15827bf88f drs://elwazi-node.icermali.org:5000/168d353c6f474ca72e35e9209f921a59\"\n",
    "input_file = \"drs://osdp.ace.ac.ug:5000/6fa43c7de04b60c1a73a42aa2efc977d drs://osdp.ace.ac.ug:5000/be145a60bc059c154475a2561af0df6b\" \n",
    "# input_file = \"drs://196.43.136.22:5000/1050d0443c2e83f9d9a8933481dcb405 drs://196.43.136.22:5000/be04a5a90617aeae9a05fb533f544ade\"\n",
    "data = {\n",
    "    'workflow_type': 'NEXTFLOW',\n",
    "    'workflow_type_version': '21.04.0',\n",
    "    'workflow_url': nextflow_workflow_url,\n",
    "    'workflow_params': f'{{\"input\":\"{input_file}\"}}'\n",
    "}\n",
    "\n",
    "print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "# Post a Nextflow workflow\n",
    "wes_post_workflow_response = requests.request(http_method, request_url, data = data)\n",
    "\n",
    "# print the response\n",
    "pretty_print_json(wes_post_workflow_response)\n",
    "\n",
    "current_run_id = wes_post_workflow_response.json()[\"run_id\"]\n",
    "\n",
    "print_head(\"run_id = {}\".format(current_run_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76bc7ac8-fed5-4567-beea-61fa4a0c3958",
   "metadata": {},
   "source": [
    "2. Check output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abff2d1f-a3ea-42d3-b20d-1a036b4f89ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "http_method = \"GET\"\n",
    "request_url = wes_base_url + runs_path + \"/\" + current_run_id\n",
    "\n",
    "print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "# Get request to /runs/{run_id}\n",
    "monitor_run_response = requests.request(http_method, request_url)\n",
    "\n",
    "# print the response\n",
    "pretty_print_json(monitor_run_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0675b09-c15f-474f-8588-ba473df8b3d6",
   "metadata": {},
   "source": [
    "# Data Connect\n",
    "1. Check service-info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "dda9ab0a-34f5-4f60-a3e5-34757a9fe5d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "dc_port = \"8089\"\n",
    "dc_base_url = \"http://ga4gh-starter-kit.ilifu.ac.za:{}\".format(dc_port)\n",
    "\n",
    "\n",
    "service_info_path = \"/service-info\"\n",
    "tables_path = \"/tables\"\n",
    "table_info_path = \"/table/{}/info\"\n",
    "table_data_path = \"/table/{}/data\"\n",
    "search_path = \"/search\"\n",
    "\n",
    "def pretty_print_json(response):\n",
    "    print(json.dumps(response.json(), indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7a106ce7-2db0-4dc2-b93c-0973e4376926",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"id\": \"\",\n",
      "    \"name\": \"GA4GH Discovery Search API\",\n",
      "    \"description\": \"\",\n",
      "    \"documentationUrl\": \"\",\n",
      "    \"contactUrl\": \"\",\n",
      "    \"version\": \"\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dc_service_info_resp = requests.request(\"GET\", dc_base_url+service_info_path)\n",
    "pretty_print_json(dc_service_info_resp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfd92537-f058-48cc-a9f1-d5d46db93046",
   "metadata": {},
   "source": [
    "2. Check data table we will be querying"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "62d9b681-0384-4565-8ef3-b058784afcdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"trino.public.genome_ilifu\",\n",
      "    \"description\": \"Automatically generated schema\",\n",
      "    \"data_model\": {\n",
      "        \"$id\": \"http://ga4gh-starter-kit.ilifu.ac.za:8089/table/trino.public.genome_ilifu/info\",\n",
      "        \"description\": \"Automatically generated schema\",\n",
      "        \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n",
      "        \"properties\": {\n",
      "            \"sample_id\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            },\n",
      "            \"population_id\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            },\n",
      "            \"super_population_id\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            },\n",
      "            \"sex\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            },\n",
      "            \"cram_drs_id\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            },\n",
      "            \"crai_drs_id\": {\n",
      "                \"format\": \"varchar\",\n",
      "                \"type\": \"string\",\n",
      "                \"$comment\": \"varchar\"\n",
      "            }\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dc_service_info_resp = requests.request(\"GET\", dc_base_url+'/table/trino.public.genome_ilifu/info')\n",
    "pretty_print_json(dc_service_info_resp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7e52f9c-c2ea-4e89-9fe0-ecd7d4ebab52",
   "metadata": {},
   "source": [
    "### Select address and id functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c0e2adf4-1007-42c8-9c78-34874e69070b",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def get_address(s):\n",
    "    address = s.replace(\"drs://\",\"\")\n",
    "    address = re.sub(':.*', '', address)\n",
    "    return address\n",
    "    \n",
    "def get_drs_id(s):\n",
    "    drs_id = re.sub('.*/', '', s)\n",
    "    return drs_id"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c829dfa-e834-4b76-9626-0d1ec3bf3431",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "### Use case 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85df727b-41bd-474e-954b-385ec502d05a",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "![Use case 1](use_case_1.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd861c6-4831-4607-8016-46f689ce59d0",
   "metadata": {},
   "source": [
    "1. Do query\n",
    "Select CRAM DRS ids for all African samples. Limit search to 10 samples for now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1ace0399-e384-434d-8eb5-307a3ad0601e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "   \"data\": [],\n",
      "   \"pagination\": {\n",
      "      \"next_page_url\": \"http://ga4gh-starter-kit.ilifu.ac.za:8089/search/v1/statement/queued/20231106_090301_00003_69yj9/ya33281a1fd05b2c025f51cf66192d2ef57733093/1?queryJobId=20231106_090301_00003_69yj9\"\n",
      "   }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import requests, json\n",
    "q2 = {\n",
    "  \"query\": \"select cram_drs_id from trino.public.genome_ilifu where super_population_id='AFR' limit 10\",\n",
    "  \"parameters\": []\n",
    "}\n",
    "r = requests.post(\"http://ga4gh-starter-kit.ilifu.ac.za:8089/search\", json = q2)\n",
    "print(json.dumps(r.json(), indent=3))\n",
    "data = r.json()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbc322ff-7807-40eb-819e-30ebf2154add",
   "metadata": {},
   "source": [
    "2. Parse through pages to get results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "0d4c869a-5f9f-49ba-ba1e-3a11aff23a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "next_page = data['pagination']['next_page_url']\n",
    "# print (next_page)\n",
    "\n",
    "# Poll till we get results\n",
    "while not 'executing' in next_page:\n",
    "    dc_service_info_resp = requests.request(\"GET\", next_page)\n",
    "    data = dc_service_info_resp.json()\n",
    "    next_page = data['pagination']['next_page_url']\n",
    "    # print (data)\n",
    "\n",
    "dc_service_info_resp = requests.request(\"GET\", next_page)\n",
    "# pretty_print_json(dc_service_info_resp)\n",
    "\n",
    "data = dc_service_info_resp.json()\n",
    "next_page = data['pagination']['next_page_url']\n",
    "dc_service_info_resp = requests.request(\"GET\", next_page)\n",
    "data = dc_service_info_resp.json()\n",
    "# pretty_print_json(dc_service_info_resp)\n",
    "\n",
    "drs_str = \"\"\n",
    "drs_ids = []\n",
    "for i in range(len(data['data'])):\n",
    "    # print (data['data'][i]['cram_drs_id'])\n",
    "    drs_ids.append(data['data'][i]['cram_drs_id'])\n",
    "    drs_str = drs_str + data['data'][i]['cram_drs_id'] + \" \"\n",
    "\n",
    "drs_str = drs_str[:-1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65b21bc7-a283-4a00-8512-b80f10be88bd",
   "metadata": {},
   "source": [
    "3. Map DRS server to DRS objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "cc77cdcc-567b-4902-af59-89704e4c39ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'osdp.ace.ac.ug': {'drs_ids': ['drs://osdp.ace.ac.ug:5000/6fa43c7de04b60c1a73a42aa2efc977d', 'drs://osdp.ace.ac.ug:5000/be145a60bc059c154475a2561af0df6b', 'drs://osdp.ace.ac.ug:5000/9a45659fe478e5bb39d1dd1b08bd1807', 'drs://osdp.ace.ac.ug:5000/82adbcf7cc72c31a86e65d73bf6ef81b'], 'total_file_size': 0, 'ingress': 0, 'run_id': 0, 'drs_ids_str': '', 'outputs': {}}, 'elwazi-node.icermali.org': {'drs_ids': ['drs://elwazi-node.icermali.org:5000/a68c60133f942881983d0e15827bf88f', 'drs://elwazi-node.icermali.org:5000/45ca586b0921ffedf6a63679fbaacb68', 'drs://elwazi-node.icermali.org:5000/d36019bb63182abad672205a140f7e83', 'drs://elwazi-node.icermali.org:5000/b809bb9b81a9583ec67e787b0449e9bd', 'drs://elwazi-node.icermali.org:5000/168d353c6f474ca72e35e9209f921a59'], 'total_file_size': 0, 'ingress': 0, 'run_id': 0, 'drs_ids_str': '', 'outputs': {}}, 'ga4gh-starter-kit.ilifu.ac.za': {'drs_ids': ['drs://ga4gh-starter-kit.ilifu.ac.za:5000/5a436bec951fab59dd975bcd10f316f1'], 'total_file_size': 0, 'ingress': 0, 'run_id': 0, 'drs_ids_str': '', 'outputs': {}}}\n"
     ]
    }
   ],
   "source": [
    "drs_servers = {}\n",
    "for drs_id in drs_ids:\n",
    "    address = get_address(drs_id)\n",
    "    if address not in drs_servers:\n",
    "        drs_servers[address] = {}\n",
    "        drs_servers[address]['drs_ids'] = []\n",
    "        drs_servers[address]['drs_ids'].append(drs_id)\n",
    "        drs_servers[address]['total_file_size'] = 0\n",
    "        drs_servers[address]['ingress'] = 0\n",
    "        drs_servers[address]['run_id'] = 0\n",
    "        drs_servers[address]['drs_ids_str'] = \"\"\n",
    "        drs_servers[address]['outputs'] = {}\n",
    "    else:\n",
    "        drs_servers[address]['drs_ids'].append(drs_id)\n",
    "print (drs_servers)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15c69244-4768-459f-909f-f2b6b9a40ad3",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "### Use case 2 \n",
    "### Now we initiate a run in a gather/scatter/federated manner\n",
    "#### - Runs are initiated on individual nodes (calculate flagstat)\n",
    "#### - Output is gathered and MultiQC are run on the flagstat results on one WES endpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea5ff1a4-5687-4e73-a6cb-81ac30a4c807",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "![Use case 1](use_case_2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "535d9de6-fcdf-4318-9e7b-2c9ee5172818",
   "metadata": {},
   "source": [
    "1) Launch flagstat runs. Use the dictionary structure generated previously. Launch workflow at WES endpoint on DRS objects only from that endpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1e940de3-2d85-4fe8-b2d6-927029669719",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching jobs on server: osdp.ace.ac.ug\n",
      "Launching jobs on server: elwazi-node.icermali.org\n",
      "Launching jobs on server: ga4gh-starter-kit.ilifu.ac.za\n",
      "\u001b[38;2;8;138;75mPOST request to http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/runs\u001b[0m\n",
      "{\n",
      "    \"run_id\": \"1e68a829-4d66-480a-b56d-c494a911c626\"\n",
      "}\n",
      "\u001b[38;2;8;138;75mrun_id = 1e68a829-4d66-480a-b56d-c494a911c626\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/runs/1e68a829-4d66-480a-b56d-c494a911c626\u001b[0m\n",
      "{'run_id': '1e68a829-4d66-480a-b56d-c494a911c626', 'request': {'workflow_params': {'input': 'drs://osdp.ace.ac.ug:5000/6fa43c7de04b60c1a73a42aa2efc977d drs://osdp.ace.ac.ug:5000/be145a60bc059c154475a2561af0df6b drs://osdp.ace.ac.ug:5000/9a45659fe478e5bb39d1dd1b08bd1807 drs://osdp.ace.ac.ug:5000/82adbcf7cc72c31a86e65d73bf6ef81b'}, 'workflow_type': 'NEXTFLOW', 'workflow_type_version': '21.04.0', 'workflow_url': 'https://github.com/grbot/flagstat'}, 'state': 'RUNNING', 'run_log': {'name': 'grbot/flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01879.final.chrX_15494566-15607236.cram > HG01879.final.chrX_15494566-15607236.cram.flagstat', '#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01885.final.chrX_15494566-15607236.cram > HG01885.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:05:20Z', 'stdout': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626?workdirs=c3%2F50b0b66075c87ce53d593d0fd659d7%2Ced%2F9e081e0ca93842e9b6cf55ffe3da77%2C24%2F244b9c2f11b2dfeb68b4bd96ebe377%2C7a%2F6f1bbc0d16830fe2549866f2418d69', 'stderr': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626?workdirs=c3%2F50b0b66075c87ce53d593d0fd659d7%2Ced%2F9e081e0ca93842e9b6cf55ffe3da77%2C24%2F244b9c2f11b2dfeb68b4bd96ebe377%2C7a%2F6f1bbc0d16830fe2549866f2418d69'}, 'task_logs': [{'start_time': '2023-11-06T09:05:20Z', 'stdout': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/c3/50b0b66075c87ce53d593d0fd659d7', 'stderr': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/c3/50b0b66075c87ce53d593d0fd659d7'}, {'start_time': '2023-11-06T09:05:20Z', 'stdout': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/ed/9e081e0ca93842e9b6cf55ffe3da77', 'stderr': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/ed/9e081e0ca93842e9b6cf55ffe3da77'}, {'name': 'run_flagstat (1)', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01879.final.chrX_15494566-15607236.cram > HG01879.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:05:28Z', 'end_time': '2023-11-06T09:05:28Z', 'stdout': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/24/244b9c2f11b2dfeb68b4bd96ebe377', 'stderr': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/24/244b9c2f11b2dfeb68b4bd96ebe377', 'exit_code': 0}, {'name': 'run_flagstat (2)', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01885.final.chrX_15494566-15607236.cram > HG01885.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:05:30Z', 'stdout': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/7a/6f1bbc0d16830fe2549866f2418d69', 'stderr': 'http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/7a/6f1bbc0d16830fe2549866f2418d69'}], 'outputs': {'HG01885.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/7a/6f1bbc0d16830fe2549866f2418d69/HG01885.final.chrX_15494566-15607236.cram', 'HG01879.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/24/244b9c2f11b2dfeb68b4bd96ebe377/HG01879.final.chrX_15494566-15607236.cram', 'HG01879.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/24/244b9c2f11b2dfeb68b4bd96ebe377/HG01879.final.chrX_15494566-15607236.cram.flagstat'}}\n",
      "Current job status: RUNNING\n",
      "Current job status: RUNNING\n",
      "Current job status: RUNNING\n",
      "Job running status: COMPLETE\n",
      "{\n",
      "    \"run_id\": \"1e68a829-4d66-480a-b56d-c494a911c626\",\n",
      "    \"request\": {\n",
      "        \"workflow_params\": {\n",
      "            \"input\": \"drs://osdp.ace.ac.ug:5000/6fa43c7de04b60c1a73a42aa2efc977d drs://osdp.ace.ac.ug:5000/be145a60bc059c154475a2561af0df6b drs://osdp.ace.ac.ug:5000/9a45659fe478e5bb39d1dd1b08bd1807 drs://osdp.ace.ac.ug:5000/82adbcf7cc72c31a86e65d73bf6ef81b\"\n",
      "        },\n",
      "        \"workflow_type\": \"NEXTFLOW\",\n",
      "        \"workflow_type_version\": \"21.04.0\",\n",
      "        \"workflow_url\": \"https://github.com/grbot/flagstat\"\n",
      "    },\n",
      "    \"state\": \"COMPLETE\",\n",
      "    \"run_log\": {\n",
      "        \"name\": \"grbot/flagstat\",\n",
      "        \"cmd\": [\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01879.final.chrX_15494566-15607236.cram > HG01879.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01885.final.chrX_15494566-15607236.cram > HG01885.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01896.final.chrX_15494566-15607236.cram > HG01896.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01894.final.chrX_15494566-15607236.cram > HG01894.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "        ],\n",
      "        \"start_time\": \"2023-11-06T09:05:20Z\",\n",
      "        \"end_time\": \"2023-11-06T09:05:59Z\",\n",
      "        \"stdout\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626?workdirs=24%2F244b9c2f11b2dfeb68b4bd96ebe377%2C7a%2F6f1bbc0d16830fe2549866f2418d69%2Cc3%2F50b0b66075c87ce53d593d0fd659d7%2Ced%2F9e081e0ca93842e9b6cf55ffe3da77\",\n",
      "        \"stderr\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626?workdirs=24%2F244b9c2f11b2dfeb68b4bd96ebe377%2C7a%2F6f1bbc0d16830fe2549866f2418d69%2Cc3%2F50b0b66075c87ce53d593d0fd659d7%2Ced%2F9e081e0ca93842e9b6cf55ffe3da77\",\n",
      "        \"exit_code\": 0\n",
      "    },\n",
      "    \"task_logs\": [\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01879.final.chrX_15494566-15607236.cram > HG01879.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:05:20Z\",\n",
      "            \"end_time\": \"2023-11-06T09:05:30Z\",\n",
      "            \"stdout\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/24/244b9c2f11b2dfeb68b4bd96ebe377\",\n",
      "            \"stderr\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/24/244b9c2f11b2dfeb68b4bd96ebe377\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01885.final.chrX_15494566-15607236.cram > HG01885.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:05:30Z\",\n",
      "            \"end_time\": \"2023-11-06T09:05:40Z\",\n",
      "            \"stdout\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/7a/6f1bbc0d16830fe2549866f2418d69\",\n",
      "            \"stderr\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/7a/6f1bbc0d16830fe2549866f2418d69\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01896.final.chrX_15494566-15607236.cram > HG01896.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:05:40Z\",\n",
      "            \"end_time\": \"2023-11-06T09:05:50Z\",\n",
      "            \"stdout\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/c3/50b0b66075c87ce53d593d0fd659d7\",\n",
      "            \"stderr\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/c3/50b0b66075c87ce53d593d0fd659d7\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01894.final.chrX_15494566-15607236.cram > HG01894.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:05:50Z\",\n",
      "            \"end_time\": \"2023-11-06T09:05:59Z\",\n",
      "            \"stdout\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stdout/1e68a829-4d66-480a-b56d-c494a911c626/ed/9e081e0ca93842e9b6cf55ffe3da77\",\n",
      "            \"stderr\": \"http://osdp.ace.ac.ug:6000/ga4gh/wes/v1/logs/nextflow/stderr/1e68a829-4d66-480a-b56d-c494a911c626/ed/9e081e0ca93842e9b6cf55ffe3da77\",\n",
      "            \"exit_code\": 0\n",
      "        }\n",
      "    ],\n",
      "    \"outputs\": {\n",
      "        \"HG01896.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/c3/50b0b66075c87ce53d593d0fd659d7/HG01896.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01894.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/ed/9e081e0ca93842e9b6cf55ffe3da77/HG01894.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01896.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/c3/50b0b66075c87ce53d593d0fd659d7/HG01896.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01885.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/7a/6f1bbc0d16830fe2549866f2418d69/HG01885.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01894.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/ed/9e081e0ca93842e9b6cf55ffe3da77/HG01894.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01885.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/7a/6f1bbc0d16830fe2549866f2418d69/HG01885.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01879.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/24/244b9c2f11b2dfeb68b4bd96ebe377/HG01879.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01879.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/1e/68/a8/1e68a829-4d66-480a-b56d-c494a911c626/work/24/244b9c2f11b2dfeb68b4bd96ebe377/HG01879.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "    }\n",
      "}\n",
      "\u001b[38;2;8;138;75mPOST request to http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/runs\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"run_id\": \"7089aa4a-2b68-4091-af80-21bfcd231a31\"\n",
      "}\n",
      "\u001b[38;2;8;138;75mrun_id = 7089aa4a-2b68-4091-af80-21bfcd231a31\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/runs/7089aa4a-2b68-4091-af80-21bfcd231a31\u001b[0m\n",
      "{'run_id': '7089aa4a-2b68-4091-af80-21bfcd231a31', 'request': {'workflow_params': {'input': 'drs://elwazi-node.icermali.org:5000/a68c60133f942881983d0e15827bf88f drs://elwazi-node.icermali.org:5000/45ca586b0921ffedf6a63679fbaacb68 drs://elwazi-node.icermali.org:5000/d36019bb63182abad672205a140f7e83 drs://elwazi-node.icermali.org:5000/b809bb9b81a9583ec67e787b0449e9bd drs://elwazi-node.icermali.org:5000/168d353c6f474ca72e35e9209f921a59'}, 'workflow_type': 'NEXTFLOW', 'workflow_type_version': '21.04.0', 'workflow_url': 'https://github.com/grbot/flagstat'}, 'state': 'COMPLETE', 'run_log': {'name': 'grbot/flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01882.final.chrX_15494566-15607236.cram > HG01882.final.chrX_15494566-15607236.cram.flagstat', '#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01880.final.chrX_15494566-15607236.cram > HG01880.final.chrX_15494566-15607236.cram.flagstat', '#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01886.final.chrX_15494566-15607236.cram > HG01886.final.chrX_15494566-15607236.cram.flagstat', '#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01889.final.chrX_15494566-15607236.cram > HG01889.final.chrX_15494566-15607236.cram.flagstat', '#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01890.final.chrX_15494566-15607236.cram > HG01890.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:15Z', 'end_time': '2023-11-06T09:06:20Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31?workdirs=45%2F8f9f63b2938b30cc527cab467ef56e%2Ce1%2Fe1adebebfba7717565c0bd5660e793%2C92%2Fa82441390fc544d77b0cfb1dd8743e%2Cc1%2Fdeb87d9202a5f8183b4e28edb6bb0b%2Cdb%2F4b8b5ba44b74873583a6e525e32882', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31?workdirs=45%2F8f9f63b2938b30cc527cab467ef56e%2Ce1%2Fe1adebebfba7717565c0bd5660e793%2C92%2Fa82441390fc544d77b0cfb1dd8743e%2Cc1%2Fdeb87d9202a5f8183b4e28edb6bb0b%2Cdb%2F4b8b5ba44b74873583a6e525e32882', 'exit_code': 0}, 'task_logs': [{'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01882.final.chrX_15494566-15607236.cram > HG01882.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:15Z', 'end_time': '2023-11-06T09:06:16Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/45/8f9f63b2938b30cc527cab467ef56e', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/45/8f9f63b2938b30cc527cab467ef56e', 'exit_code': 0}, {'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01880.final.chrX_15494566-15607236.cram > HG01880.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:17Z', 'end_time': '2023-11-06T09:06:17Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/e1/e1adebebfba7717565c0bd5660e793', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/e1/e1adebebfba7717565c0bd5660e793', 'exit_code': 0}, {'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01886.final.chrX_15494566-15607236.cram > HG01886.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:17Z', 'end_time': '2023-11-06T09:06:18Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/92/a82441390fc544d77b0cfb1dd8743e', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/92/a82441390fc544d77b0cfb1dd8743e', 'exit_code': 0}, {'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01889.final.chrX_15494566-15607236.cram > HG01889.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:18Z', 'end_time': '2023-11-06T09:06:19Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/c1/deb87d9202a5f8183b4e28edb6bb0b', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/c1/deb87d9202a5f8183b4e28edb6bb0b', 'exit_code': 0}, {'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01890.final.chrX_15494566-15607236.cram > HG01890.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T09:06:19Z', 'end_time': '2023-11-06T09:06:20Z', 'stdout': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/db/4b8b5ba44b74873583a6e525e32882', 'stderr': 'http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/db/4b8b5ba44b74873583a6e525e32882', 'exit_code': 0}], 'outputs': {'HG01882.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/45/8f9f63b2938b30cc527cab467ef56e/HG01882.final.chrX_15494566-15607236.cram', 'HG01880.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/e1/e1adebebfba7717565c0bd5660e793/HG01880.final.chrX_15494566-15607236.cram', 'HG01880.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/e1/e1adebebfba7717565c0bd5660e793/HG01880.final.chrX_15494566-15607236.cram.flagstat', 'HG01890.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/db/4b8b5ba44b74873583a6e525e32882/HG01890.final.chrX_15494566-15607236.cram.flagstat', 'HG01886.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/92/a82441390fc544d77b0cfb1dd8743e/HG01886.final.chrX_15494566-15607236.cram', 'HG01882.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/45/8f9f63b2938b30cc527cab467ef56e/HG01882.final.chrX_15494566-15607236.cram.flagstat', 'HG01889.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/c1/deb87d9202a5f8183b4e28edb6bb0b/HG01889.final.chrX_15494566-15607236.cram', 'HG01889.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/c1/deb87d9202a5f8183b4e28edb6bb0b/HG01889.final.chrX_15494566-15607236.cram.flagstat', 'HG01886.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/92/a82441390fc544d77b0cfb1dd8743e/HG01886.final.chrX_15494566-15607236.cram.flagstat', 'HG01890.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/db/4b8b5ba44b74873583a6e525e32882/HG01890.final.chrX_15494566-15607236.cram'}}\n",
      "Job running status: COMPLETE\n",
      "{\n",
      "    \"run_id\": \"7089aa4a-2b68-4091-af80-21bfcd231a31\",\n",
      "    \"request\": {\n",
      "        \"workflow_params\": {\n",
      "            \"input\": \"drs://elwazi-node.icermali.org:5000/a68c60133f942881983d0e15827bf88f drs://elwazi-node.icermali.org:5000/45ca586b0921ffedf6a63679fbaacb68 drs://elwazi-node.icermali.org:5000/d36019bb63182abad672205a140f7e83 drs://elwazi-node.icermali.org:5000/b809bb9b81a9583ec67e787b0449e9bd drs://elwazi-node.icermali.org:5000/168d353c6f474ca72e35e9209f921a59\"\n",
      "        },\n",
      "        \"workflow_type\": \"NEXTFLOW\",\n",
      "        \"workflow_type_version\": \"21.04.0\",\n",
      "        \"workflow_url\": \"https://github.com/grbot/flagstat\"\n",
      "    },\n",
      "    \"state\": \"COMPLETE\",\n",
      "    \"run_log\": {\n",
      "        \"name\": \"grbot/flagstat\",\n",
      "        \"cmd\": [\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01882.final.chrX_15494566-15607236.cram > HG01882.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01880.final.chrX_15494566-15607236.cram > HG01880.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01886.final.chrX_15494566-15607236.cram > HG01886.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01889.final.chrX_15494566-15607236.cram > HG01889.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01890.final.chrX_15494566-15607236.cram > HG01890.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "        ],\n",
      "        \"start_time\": \"2023-11-06T09:06:15Z\",\n",
      "        \"end_time\": \"2023-11-06T09:06:20Z\",\n",
      "        \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31?workdirs=45%2F8f9f63b2938b30cc527cab467ef56e%2Ce1%2Fe1adebebfba7717565c0bd5660e793%2C92%2Fa82441390fc544d77b0cfb1dd8743e%2Cc1%2Fdeb87d9202a5f8183b4e28edb6bb0b%2Cdb%2F4b8b5ba44b74873583a6e525e32882\",\n",
      "        \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31?workdirs=45%2F8f9f63b2938b30cc527cab467ef56e%2Ce1%2Fe1adebebfba7717565c0bd5660e793%2C92%2Fa82441390fc544d77b0cfb1dd8743e%2Cc1%2Fdeb87d9202a5f8183b4e28edb6bb0b%2Cdb%2F4b8b5ba44b74873583a6e525e32882\",\n",
      "        \"exit_code\": 0\n",
      "    },\n",
      "    \"task_logs\": [\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01882.final.chrX_15494566-15607236.cram > HG01882.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:06:15Z\",\n",
      "            \"end_time\": \"2023-11-06T09:06:16Z\",\n",
      "            \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/45/8f9f63b2938b30cc527cab467ef56e\",\n",
      "            \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/45/8f9f63b2938b30cc527cab467ef56e\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01880.final.chrX_15494566-15607236.cram > HG01880.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:06:17Z\",\n",
      "            \"end_time\": \"2023-11-06T09:06:17Z\",\n",
      "            \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/e1/e1adebebfba7717565c0bd5660e793\",\n",
      "            \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/e1/e1adebebfba7717565c0bd5660e793\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01886.final.chrX_15494566-15607236.cram > HG01886.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:06:17Z\",\n",
      "            \"end_time\": \"2023-11-06T09:06:18Z\",\n",
      "            \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/92/a82441390fc544d77b0cfb1dd8743e\",\n",
      "            \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/92/a82441390fc544d77b0cfb1dd8743e\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01889.final.chrX_15494566-15607236.cram > HG01889.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:06:18Z\",\n",
      "            \"end_time\": \"2023-11-06T09:06:19Z\",\n",
      "            \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/c1/deb87d9202a5f8183b4e28edb6bb0b\",\n",
      "            \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/c1/deb87d9202a5f8183b4e28edb6bb0b\",\n",
      "            \"exit_code\": 0\n",
      "        },\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01890.final.chrX_15494566-15607236.cram > HG01890.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T09:06:19Z\",\n",
      "            \"end_time\": \"2023-11-06T09:06:20Z\",\n",
      "            \"stdout\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stdout/7089aa4a-2b68-4091-af80-21bfcd231a31/db/4b8b5ba44b74873583a6e525e32882\",\n",
      "            \"stderr\": \"http://elwazi-node.icermali.org:6000/ga4gh/wes/v1/logs/nextflow/stderr/7089aa4a-2b68-4091-af80-21bfcd231a31/db/4b8b5ba44b74873583a6e525e32882\",\n",
      "            \"exit_code\": 0\n",
      "        }\n",
      "    ],\n",
      "    \"outputs\": {\n",
      "        \"HG01882.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/45/8f9f63b2938b30cc527cab467ef56e/HG01882.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01880.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/e1/e1adebebfba7717565c0bd5660e793/HG01880.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01880.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/e1/e1adebebfba7717565c0bd5660e793/HG01880.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01890.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/db/4b8b5ba44b74873583a6e525e32882/HG01890.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01886.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/92/a82441390fc544d77b0cfb1dd8743e/HG01886.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01882.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/45/8f9f63b2938b30cc527cab467ef56e/HG01882.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01889.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/c1/deb87d9202a5f8183b4e28edb6bb0b/HG01889.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01889.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/c1/deb87d9202a5f8183b4e28edb6bb0b/HG01889.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01886.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/92/a82441390fc544d77b0cfb1dd8743e/HG01886.final.chrX_15494566-15607236.cram.flagstat\",\n",
      "        \"HG01890.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/70/89/aa/7089aa4a-2b68-4091-af80-21bfcd231a31/work/db/4b8b5ba44b74873583a6e525e32882/HG01890.final.chrX_15494566-15607236.cram\"\n",
      "    }\n",
      "}\n",
      "\u001b[38;2;8;138;75mPOST request to http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/runs\u001b[0m\n",
      "{\n",
      "    \"run_id\": \"09edc840-b9d0-4769-b434-d5c951e24170\"\n",
      "}\n",
      "\u001b[38;2;8;138;75mrun_id = 09edc840-b9d0-4769-b434-d5c951e24170\u001b[0m\n",
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/runs/09edc840-b9d0-4769-b434-d5c951e24170\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'run_id': '09edc840-b9d0-4769-b434-d5c951e24170', 'request': {'workflow_params': {'input': 'drs://ga4gh-starter-kit.ilifu.ac.za:5000/5a436bec951fab59dd975bcd10f316f1'}, 'workflow_type': 'NEXTFLOW', 'workflow_type_version': '21.04.0', 'workflow_url': 'https://github.com/grbot/flagstat'}, 'state': 'COMPLETE', 'run_log': {'name': 'grbot/flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01883.final.chrX_15494566-15607236.cram > HG01883.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T11:05:50Z', 'end_time': '2023-11-06T11:05:50Z', 'stdout': 'http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/09edc840-b9d0-4769-b434-d5c951e24170?workdirs=73%2F65a4b12cf72837c631610ec5de2fc6', 'stderr': 'http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/09edc840-b9d0-4769-b434-d5c951e24170?workdirs=73%2F65a4b12cf72837c631610ec5de2fc6', 'exit_code': 0}, 'task_logs': [{'name': 'run_flagstat', 'cmd': ['#!/bin/bash -ue', 'samtools flagstat     -@ 1     HG01883.final.chrX_15494566-15607236.cram > HG01883.final.chrX_15494566-15607236.cram.flagstat'], 'start_time': '2023-11-06T11:05:50Z', 'end_time': '2023-11-06T11:05:50Z', 'stdout': 'http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/09edc840-b9d0-4769-b434-d5c951e24170/73/65a4b12cf72837c631610ec5de2fc6', 'stderr': 'http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/09edc840-b9d0-4769-b434-d5c951e24170/73/65a4b12cf72837c631610ec5de2fc6', 'exit_code': 0}], 'outputs': {'HG01883.final.chrX_15494566-15607236.cram': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/09/ed/c8/09edc840-b9d0-4769-b434-d5c951e24170/work/73/65a4b12cf72837c631610ec5de2fc6/HG01883.final.chrX_15494566-15607236.cram', 'HG01883.final.chrX_15494566-15607236.cram.flagstat': 'file:///opt/ga4gh-starter-kit-wes/wes_runs/09/ed/c8/09edc840-b9d0-4769-b434-d5c951e24170/work/73/65a4b12cf72837c631610ec5de2fc6/HG01883.final.chrX_15494566-15607236.cram.flagstat'}}\n",
      "Job running status: COMPLETE\n",
      "{\n",
      "    \"run_id\": \"09edc840-b9d0-4769-b434-d5c951e24170\",\n",
      "    \"request\": {\n",
      "        \"workflow_params\": {\n",
      "            \"input\": \"drs://ga4gh-starter-kit.ilifu.ac.za:5000/5a436bec951fab59dd975bcd10f316f1\"\n",
      "        },\n",
      "        \"workflow_type\": \"NEXTFLOW\",\n",
      "        \"workflow_type_version\": \"21.04.0\",\n",
      "        \"workflow_url\": \"https://github.com/grbot/flagstat\"\n",
      "    },\n",
      "    \"state\": \"COMPLETE\",\n",
      "    \"run_log\": {\n",
      "        \"name\": \"grbot/flagstat\",\n",
      "        \"cmd\": [\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"samtools flagstat     -@ 1     HG01883.final.chrX_15494566-15607236.cram > HG01883.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "        ],\n",
      "        \"start_time\": \"2023-11-06T11:05:50Z\",\n",
      "        \"end_time\": \"2023-11-06T11:05:50Z\",\n",
      "        \"stdout\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/09edc840-b9d0-4769-b434-d5c951e24170?workdirs=73%2F65a4b12cf72837c631610ec5de2fc6\",\n",
      "        \"stderr\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/09edc840-b9d0-4769-b434-d5c951e24170?workdirs=73%2F65a4b12cf72837c631610ec5de2fc6\",\n",
      "        \"exit_code\": 0\n",
      "    },\n",
      "    \"task_logs\": [\n",
      "        {\n",
      "            \"name\": \"run_flagstat\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"samtools flagstat     -@ 1     HG01883.final.chrX_15494566-15607236.cram > HG01883.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T11:05:50Z\",\n",
      "            \"end_time\": \"2023-11-06T11:05:50Z\",\n",
      "            \"stdout\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/09edc840-b9d0-4769-b434-d5c951e24170/73/65a4b12cf72837c631610ec5de2fc6\",\n",
      "            \"stderr\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/09edc840-b9d0-4769-b434-d5c951e24170/73/65a4b12cf72837c631610ec5de2fc6\",\n",
      "            \"exit_code\": 0\n",
      "        }\n",
      "    ],\n",
      "    \"outputs\": {\n",
      "        \"HG01883.final.chrX_15494566-15607236.cram\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/09/ed/c8/09edc840-b9d0-4769-b434-d5c951e24170/work/73/65a4b12cf72837c631610ec5de2fc6/HG01883.final.chrX_15494566-15607236.cram\",\n",
      "        \"HG01883.final.chrX_15494566-15607236.cram.flagstat\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/09/ed/c8/09edc840-b9d0-4769-b434-d5c951e24170/work/73/65a4b12cf72837c631610ec5de2fc6/HG01883.final.chrX_15494566-15607236.cram.flagstat\"\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "# Populate with drs_ids_str\n",
    "for drs_server in drs_servers:\n",
    "    print (\"Launching jobs on server: \" + drs_server)\n",
    "    drs_ids = drs_servers[drs_server]['drs_ids']\n",
    "    drs_ids_str = \"\"\n",
    "    for drs_id in drs_ids:\n",
    "        drs_ids_str = drs_ids_str + drs_id + \" \"\n",
    "    drs_ids_str = drs_ids_str[:-1]\n",
    "#     print (drs_ids_str)\n",
    "    drs_servers[drs_server]['drs_ids_str'] = drs_ids_str\n",
    "\n",
    "# Launch workflow\n",
    "for drs_server in drs_servers:\n",
    "    wes_port = \"6000\"\n",
    "    ga4gh_base_url = \"http://\" + drs_server + \":{}/ga4gh/{}/v1\"\n",
    "    wes_base_url = ga4gh_base_url.format(wes_port,\"wes\")\n",
    "\n",
    "    runs_path = \"/runs\"\n",
    "\n",
    "    http_method = \"POST\"\n",
    "    request_url = wes_base_url + runs_path\n",
    "\n",
    "    nextflow_workflow_url = \"https://github.com/grbot/flagstat\"\n",
    "\n",
    "    input_file = drs_servers[drs_server]['drs_ids_str']\n",
    "    \n",
    "    data = {\n",
    "        'workflow_type': 'NEXTFLOW',\n",
    "        'workflow_type_version': '21.04.0',\n",
    "        'workflow_url': nextflow_workflow_url,\n",
    "        'workflow_params': f'{{\"input\":\"{input_file}\"}}'\n",
    "    }\n",
    "\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "    # Post a Nextflow workflow\n",
    "    wes_post_workflow_response = requests.request(http_method, request_url, data = data)\n",
    "\n",
    "    # print the response\n",
    "    pretty_print_json(wes_post_workflow_response)\n",
    "\n",
    "    current_run_id = wes_post_workflow_response.json()[\"run_id\"]\n",
    "\n",
    "    print_head(\"run_id = {}\".format(current_run_id))\n",
    "\n",
    "    ## We don't launch in parallel for now. Just poll a submitted job and retrieve the results\n",
    "    http_method = \"GET\"\n",
    "    request_url = wes_base_url + runs_path + \"/\" + current_run_id\n",
    "\n",
    "    print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "    # Get request to /runs/{run_id}\n",
    "#     monitor_run_response = requests.request(http_method, request_url) \n",
    "    time.sleep(15)\n",
    "    monitor_run_response = requests.request(http_method, request_url)\n",
    "    \n",
    "    # Poll until job is complete\n",
    "\n",
    "    print(monitor_run_response.json())\n",
    "    while monitor_run_response.json()[\"state\"] != \"COMPLETE\":\n",
    "        print(\"Current job status: \" + monitor_run_response.json()[\"state\"])\n",
    "        time.sleep(5)\n",
    "        monitor_run_response = requests.request(http_method, request_url)\n",
    "\n",
    "    print(\"Job running status: \" + monitor_run_response.json()[\"state\"])\n",
    "    pretty_print_json(monitor_run_response)\n",
    "\n",
    "    drs_servers[drs_server]['run_id'] = current_run_id\n",
    "    drs_servers[drs_server]['outputs'] = monitor_run_response.json()[\"outputs\"]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98535823-792a-493a-b03e-3189c34affc8",
   "metadata": {},
   "source": [
    "2. a) Upload results to individual DRS servers and get a list of DRS objects (this will not work need to run 2 b) and explained there)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c578cdb5-0f8c-49d6-8725-e3bd5cef3fcc",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# importlib.reload(upload_to_drs)\n",
    "\n",
    "# drs_ids_str = \"\"\n",
    "\n",
    "# for drs_server in drs_servers:\n",
    "\n",
    "#     run_id = drs_servers[drs_server]['run_id']\n",
    "#     outputs = drs_servers[drs_server][\"outputs\"]\n",
    "\n",
    "#     for key in outputs:\n",
    "#         if '.flagstat' in key:\n",
    "#             file = outputs[key][7:]\n",
    "#             file_ext = file.split(\".\")[-1]\n",
    "#             print (file)\n",
    "#             meta_d = upload_to_drs.files_metadata_test(run_id, file, file_ext)\n",
    "#             upload_to_drs.add_file_to_server(meta_d, file_ext, drs_server,'5001') #adds drs object\n",
    "#             drs_id = \"drs://\" + drs_server + \":5000/\" + meta_d[0][3]\n",
    "#             drs_ids_str = drs_ids_str + \" \"  + drs_id\n",
    "            \n",
    "# drs_ids_str = drs_ids_str[:-1]\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00d81ec0-cb6d-45cb-b52b-b53adb494c9d",
   "metadata": {},
   "source": [
    "2. b) Upload results to individual DRS servers and get a list of DRS access URLs\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "61ba77f0-0e73-4d37-9513-bedb969853b9",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HG01896.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01885.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01894.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01879.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01880.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01890.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01882.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01889.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01886.final.chrX_15494566-15607236.cram.flagstat\n",
      "HG01883.final.chrX_15494566-15607236.cram.flagstat\n",
      "http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/ea4954e23c25366d466b430731589866/493eadf8-2dd8-4bd9-b821-c272cdbf68c2 http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/f07b256a7dd645b7c4a08e7249a4de79/dd7b0f32-7093-47ee-8389-441e34711e97 http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/aa26c66b0179efa8b331ad8ccb4f82fb/4ff93fe4-c155-4481-b1cb-a0c6b8b8f73d http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/4e28d519cd796ffe408adf054e1e71dc/b9aa8422-64de-40b8-ad71-0804f02d8807 http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/1651d9f8432c5091be385648f418a4f7/9910c910-c24b-4886-b34d-4554ad258f2b http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/daf41a6c2067981f1ca096fd3823fc0a/ed79d3ec-ec7f-4922-8456-1e6ef099a5bb http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/e5c12cb037eb0b3b7a570dfa82d643b9/7d2885e8-dc63-46b0-9437-5717456b6ad7 http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/76b1db6974285a92ba2586f5ded510ca/224820e3-2dc9-41d9-a87f-1fd0b9135f0e http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/9a4ed11618712c2da5d95c116768529f/6f520c56-b142-4955-976e-02de6350a043 http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/stream/737dbe556113884ad23d48f835e26ed9/4173856b-6608-44a7-885c-920c4c001633\n"
     ]
    }
   ],
   "source": [
    "import importlib\n",
    "import upload_to_drs\n",
    "\n",
    "importlib.reload(upload_to_drs)\n",
    "\n",
    "drs_urls_str = \"\"\n",
    "\n",
    "for drs_server in drs_servers:\n",
    "\n",
    "    run_id = drs_servers[drs_server]['run_id']\n",
    "    outputs = drs_servers[drs_server][\"outputs\"]\n",
    "\n",
    "    for key in outputs:\n",
    "        if '.flagstat' in key:\n",
    "            print (key)\n",
    "            file = outputs[key][7:]\n",
    "            file_ext = file.split(\".\")[-1]\n",
    "            meta_d = upload_to_drs.files_metadata_test(run_id, file, file_ext)\n",
    "            upload_to_drs.add_file_to_server(meta_d, file_ext, drs_server,'5001') #adds drs object\n",
    "            drs_id = meta_d[0][3]\n",
    "            drs_port = 5000\n",
    "            object_path_get = \"/objects/{}\"\n",
    "            http_method = \"GET\"\n",
    "            ga4gh_base_url = \"http://\" + drs_server + \":{}/ga4gh/{}/v1\"\n",
    "            drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "            request_url = drs_base_url + object_path_get.format(drs_id)\n",
    "            #print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "            drs_object_response = requests.request(http_method, request_url)\n",
    "            #pretty_print_json(drs_object_response)\n",
    "            data = drs_object_response.json()\n",
    "            # We cannot use DRS objects here and need to directly stream. DRS objects only resolve to local path\n",
    "            # and if path is not on server their would be a failure. The disadvantage of using stream is that you loose\n",
    "            # the file naming.\n",
    "            access_url = request_url + \"/access/\" + (data['access_methods'][1]['access_id'])\n",
    "            drs_object_response = requests.request(http_method, access_url)\n",
    "            drs_url = drs_object_response.json()[\"url\"]\n",
    "            drs_urls_str = drs_urls_str + drs_url + \" \"\n",
    "    \n",
    "drs_urls_str = drs_urls_str[:-1]\n",
    "print(drs_urls_str)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "316e80bf-9a38-4c91-bdf2-9f449937b189",
   "metadata": {},
   "source": [
    "3. Launch the workflow that will combine the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "fe0efeed-2710-4bb6-98d6-945a45db8714",
   "metadata": {
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mPOST request to http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/runs\u001b[0m\n",
      "{\n",
      "    \"run_id\": \"62541103-b540-4bae-9df7-a348a887f720\"\n",
      "}\n",
      "\u001b[38;2;8;138;75mrun_id = 62541103-b540-4bae-9df7-a348a887f720\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "wes_port = \"6000\"\n",
    "drs_server_central = \"ga4gh-starter-kit.ilifu.ac.za\"\n",
    "ga4gh_base_url = \"http://\" + drs_server_central + \":{}/ga4gh/{}/v1\"\n",
    "wes_base_url = ga4gh_base_url.format(wes_port,\"wes\")\n",
    "\n",
    "service_info_path = \"/service-info\"\n",
    "runs_path = \"/runs\"\n",
    "\n",
    "http_method = \"POST\"\n",
    "request_url = wes_base_url + runs_path\n",
    "\n",
    "nextflow_workflow_url = \"https://github.com/grbot/multiqc\"\n",
    "input_file = drs_urls_str\n",
    "\n",
    "#print (drs_str)\n",
    "\n",
    "data = {\n",
    "    'workflow_type': 'NEXTFLOW',\n",
    "    'workflow_type_version': '21.04.0',\n",
    "    'workflow_url': nextflow_workflow_url,\n",
    "    'workflow_params': f'{{\"input\":\"{input_file}\"}}'\n",
    "}\n",
    "\n",
    "print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "# Post a Nextflow workflow\n",
    "wes_post_workflow_response = requests.request(http_method, request_url, data = data)\n",
    "\n",
    "# print the response\n",
    "pretty_print_json(wes_post_workflow_response)\n",
    "\n",
    "current_run_id = wes_post_workflow_response.json()[\"run_id\"]\n",
    "\n",
    "print_head(\"run_id = {}\".format(current_run_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dee6ed9-aa2a-4504-a2d1-fedea50776b7",
   "metadata": {},
   "source": [
    "4. Poll for results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "279660a1-ccb2-47a7-b995-9285b1d602ad",
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;8;138;75mGET request to http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/runs/62541103-b540-4bae-9df7-a348a887f720\u001b[0m\n",
      "<Response [200]>\n",
      "Job running status: COMPLETE\n",
      "{\n",
      "    \"run_id\": \"62541103-b540-4bae-9df7-a348a887f720\",\n",
      "    \"request\": {\n",
      "        \"workflow_params\": {\n",
      "            \"input\": \"http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/ea4954e23c25366d466b430731589866/493eadf8-2dd8-4bd9-b821-c272cdbf68c2 http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/f07b256a7dd645b7c4a08e7249a4de79/dd7b0f32-7093-47ee-8389-441e34711e97 http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/aa26c66b0179efa8b331ad8ccb4f82fb/4ff93fe4-c155-4481-b1cb-a0c6b8b8f73d http://osdp.ace.ac.ug:5000/ga4gh/drs/v1/stream/4e28d519cd796ffe408adf054e1e71dc/b9aa8422-64de-40b8-ad71-0804f02d8807 http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/1651d9f8432c5091be385648f418a4f7/9910c910-c24b-4886-b34d-4554ad258f2b http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/daf41a6c2067981f1ca096fd3823fc0a/ed79d3ec-ec7f-4922-8456-1e6ef099a5bb http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/e5c12cb037eb0b3b7a570dfa82d643b9/7d2885e8-dc63-46b0-9437-5717456b6ad7 http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/76b1db6974285a92ba2586f5ded510ca/224820e3-2dc9-41d9-a87f-1fd0b9135f0e http://elwazi-node.icermali.org:5000/ga4gh/drs/v1/stream/9a4ed11618712c2da5d95c116768529f/6f520c56-b142-4955-976e-02de6350a043 http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/stream/737dbe556113884ad23d48f835e26ed9/4173856b-6608-44a7-885c-920c4c001633\"\n",
      "        },\n",
      "        \"workflow_type\": \"NEXTFLOW\",\n",
      "        \"workflow_type_version\": \"21.04.0\",\n",
      "        \"workflow_url\": \"https://github.com/grbot/multiqc\"\n",
      "    },\n",
      "    \"state\": \"COMPLETE\",\n",
      "    \"run_log\": {\n",
      "        \"name\": \"grbot/multiqc\",\n",
      "        \"cmd\": [\n",
      "            \"#!/bin/bash -ue\",\n",
      "            \"multiqc .\"\n",
      "        ],\n",
      "        \"start_time\": \"2023-11-06T11:09:03Z\",\n",
      "        \"end_time\": \"2023-11-06T11:09:07Z\",\n",
      "        \"stdout\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/62541103-b540-4bae-9df7-a348a887f720?workdirs=1d%2F2d713cd24ff517d691cabad66d33f0\",\n",
      "        \"stderr\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/62541103-b540-4bae-9df7-a348a887f720?workdirs=1d%2F2d713cd24ff517d691cabad66d33f0\",\n",
      "        \"exit_code\": 0\n",
      "    },\n",
      "    \"task_logs\": [\n",
      "        {\n",
      "            \"name\": \"run_multiqc\",\n",
      "            \"cmd\": [\n",
      "                \"#!/bin/bash -ue\",\n",
      "                \"multiqc .\"\n",
      "            ],\n",
      "            \"start_time\": \"2023-11-06T11:09:03Z\",\n",
      "            \"end_time\": \"2023-11-06T11:09:07Z\",\n",
      "            \"stdout\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stdout/62541103-b540-4bae-9df7-a348a887f720/1d/2d713cd24ff517d691cabad66d33f0\",\n",
      "            \"stderr\": \"http://ga4gh-starter-kit.ilifu.ac.za:6000/ga4gh/wes/v1/logs/nextflow/stderr/62541103-b540-4bae-9df7-a348a887f720/1d/2d713cd24ff517d691cabad66d33f0\",\n",
      "            \"exit_code\": 0\n",
      "        }\n",
      "    ],\n",
      "    \"outputs\": {\n",
      "        \"224820e3-2dc9-41d9-a87f-1fd0b9135f0e\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/224820e3-2dc9-41d9-a87f-1fd0b9135f0e\",\n",
      "        \"ed79d3ec-ec7f-4922-8456-1e6ef099a5bb\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/ed79d3ec-ec7f-4922-8456-1e6ef099a5bb\",\n",
      "        \"b9aa8422-64de-40b8-ad71-0804f02d8807\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/b9aa8422-64de-40b8-ad71-0804f02d8807\",\n",
      "        \"9910c910-c24b-4886-b34d-4554ad258f2b\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/9910c910-c24b-4886-b34d-4554ad258f2b\",\n",
      "        \"6f520c56-b142-4955-976e-02de6350a043\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/6f520c56-b142-4955-976e-02de6350a043\",\n",
      "        \"4173856b-6608-44a7-885c-920c4c001633\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/4173856b-6608-44a7-885c-920c4c001633\",\n",
      "        \"multiqc_data/multiqc_citations.txt\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc_citations.txt\",\n",
      "        \"multiqc_data/multiqc_samtools_flagstat.txt\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc_samtools_flagstat.txt\",\n",
      "        \"dd7b0f32-7093-47ee-8389-441e34711e97\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/dd7b0f32-7093-47ee-8389-441e34711e97\",\n",
      "        \"multiqc_data/multiqc_general_stats.txt\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc_general_stats.txt\",\n",
      "        \"multiqc_data/multiqc.log\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc.log\",\n",
      "        \"multiqc_data/multiqc_sources.txt\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc_sources.txt\",\n",
      "        \"multiqc_data/multiqc_data.json\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_data/multiqc_data.json\",\n",
      "        \"7d2885e8-dc63-46b0-9437-5717456b6ad7\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/7d2885e8-dc63-46b0-9437-5717456b6ad7\",\n",
      "        \"4ff93fe4-c155-4481-b1cb-a0c6b8b8f73d\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/4ff93fe4-c155-4481-b1cb-a0c6b8b8f73d\",\n",
      "        \"493eadf8-2dd8-4bd9-b821-c272cdbf68c2\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/493eadf8-2dd8-4bd9-b821-c272cdbf68c2\",\n",
      "        \"multiqc_report.html\": \"file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_report.html\"\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "# On first run might get the error not able to find [\"state\"]. This is due to underlying isssue below.\n",
    "#{\n",
    "#    \"timestamp\": \"2023-07-27T17:10:08Z\",\n",
    "#    \"status_code\": 400,\n",
    "#    \"error\": \"Bad Request\",\n",
    "#    \"msg\": \"Could not load WES run log\"\n",
    "#}\n",
    "# Just rerun for now and then polling will start\n",
    "\n",
    "http_method = \"GET\"\n",
    "request_url = wes_base_url + runs_path + \"/\" + current_run_id\n",
    "\n",
    "print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "\n",
    "# Get request to /runs/{run_id}\n",
    "monitor_run_response = requests.request(http_method, request_url)\n",
    "time.sleep(15) # This delay resolves the issue mentioned above\n",
    "monitor_run_response = requests.request(http_method, request_url)\n",
    "\n",
    "print(monitor_run_response)\n",
    "\n",
    "# Poll until job is complete\n",
    "while monitor_run_response.json()[\"state\"]!=\"COMPLETE\":\n",
    "    print(\"Current job status: \" + monitor_run_response.json()[\"state\"])\n",
    "    time.sleep(5)\n",
    "    monitor_run_response = requests.request(http_method, request_url)\n",
    "\n",
    "print(\"Job running status: \" + monitor_run_response.json()[\"state\"])\n",
    "pretty_print_json(monitor_run_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89fb42f2-f2c7-4302-9cff-08d81beb28dc",
   "metadata": {},
   "source": [
    "5. Upload `multiqc_report.html` to central DRS server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "76967eca-51f0-4cd0-a644-6dc9ee3ec57f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file:///opt/ga4gh-starter-kit-wes/wes_runs/62/54/11/62541103-b540-4bae-9df7-a348a887f720/work/1d/2d713cd24ff517d691cabad66d33f0/multiqc_report.html\n"
     ]
    }
   ],
   "source": [
    "import importlib\n",
    "import upload_to_drs\n",
    "importlib.reload(upload_to_drs)\n",
    "\n",
    "run_id = monitor_run_response.json()['run_id']\n",
    "outputs = monitor_run_response.json()[\"outputs\"]\n",
    "\n",
    "for key in outputs:\n",
    "    if 'multiqc_report.html' in key:\n",
    "        print (outputs[key])\n",
    "        file = outputs[key][7:]\n",
    "        file_ext = file.split(\".\")[-1]\n",
    "        meta_d = upload_to_drs.files_metadata_test(run_id, file, file_ext)\n",
    "        upload_to_drs.add_file_to_server(meta_d, file_ext, drs_server_central,'5001') #adds drs object\n",
    "        drs_id = meta_d[0][3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cc01dc0-61ed-4d4b-aad0-4ddccd5a5510",
   "metadata": {},
   "source": [
    "6. Download `multiqc_report.html`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "51462c75-2835-47a6-99bc-c27bf69578bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://ga4gh-starter-kit.ilifu.ac.za:5000/ga4gh/drs/v1/stream/e6c29b7942a4b2875ee31368ac70874e/caced522-375f-45ee-937c-100277db7a4e\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('multiqc_report_2.html', <http.client.HTTPMessage at 0x7fd02edc6470>)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib.request\n",
    "\n",
    "drs_port = 5000\n",
    "\n",
    "object_path_get = \"/objects/{}\"\n",
    "http_method = \"GET\"\n",
    "ga4gh_base_url = \"http://\" + drs_server_central + \":{}/ga4gh/{}/v1\"\n",
    "drs_base_url = ga4gh_base_url.format(drs_port,\"drs\")\n",
    "request_url = drs_base_url + object_path_get.format(drs_id)\n",
    "#print_head(\"{} request to {}\".format(http_method, request_url))\n",
    "drs_object_response = requests.request(http_method, request_url)\n",
    "#pretty_print_json(drs_object_response)\n",
    "data = drs_object_response.json()\n",
    "access_url = request_url + \"/access/\" + (data['access_methods'][1]['access_id'])\n",
    "#print(access_path)\n",
    "drs_object_response = requests.request(http_method, access_url)\n",
    "download_url = drs_object_response.json()[\"url\"]\n",
    "print(download_url)\n",
    "urllib.request.urlretrieve(download_url, \"multiqc_report_2.html\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}