Fixes #30928 - Add chunk size parameter to export

[parthaa]

Pulp3 provides the ability to chunk an export. This commit adds functionality related to that. It provides a chunk_size_mb option to
specify the chunk size in megabytes

$ curl -X POST -s -k -u admin:changeme http://lambda.sparta.example.com:3000/katello/api/content_view_versions/3/export -d '{"destination_server": "downstream1.example.com", "chunk_size_mb": 10}' -H 'Content-Type: application/json' | jq
{
  "id": "99dd31be-e41d-4e5c-b3c8-c544efa92785",
  "label": "Actions::Pulp3::Orchestration::ContentViewVersion::Export",
  "pending": true,
  "action": "Export content view version 'view 1.0'; content view 'view'; organization 'export-15903'",
  "username": "admin",
  "started_at": "2020-09-28 21:29:37 UTC",
  "ended_at": null,
  "state": "planned",
  "result": "pending",
  "progress": 0,
  "input": {
    "content_view_version": {
      "id": 3,
      "name": "view 1.0"
    },
    "content_view": {
      "id": 3,
      "name": "view",
      "label": "view"
    },
    "organization": {
      "id": 3,
      "name": "export-15903",
      "label": "export-15903"
    },
    "services_checked": [
      "pulp3"
    ],
    "exporter_data": {
      "class": "Dynflow::ExecutionPlan::OutputReference",
      "execution_plan_id": "6a9dbf2d-7474-41d7-955a-b1f558cb2ec2",
      "step_id": 3,
      "action_id": 2,
      "subkeys": [
        "exporter_data"
      ]
    },
    "smart_proxy_id": 1,
    "destination_server": "downstream1.example.com",
    "content_view_version_id": 3,
    "locale": "en",
    "current_request_id": null,
    "current_timezone": "UTC",
    "current_user_id": 4,
    "current_organization_id": null,
    "current_location_id": null
  },
  "output": {},
  "humanized": {
    "action": "Export",
    "input": [
      [
        "content_view_version",
        {
          "text": "content view version 'view 1.0'",
          "link": null
        }
      ],
      [
        "content_view",
        {
          "text": "content view 'view'",
          "link": "/content_views/3/versions"
        }
      ],
      [
        "organization",
        {
          "text": "organization 'export-15903'",
          "link": "/organizations/3/edit"
        }
      ]
    ],
    "output": "",
    "errors": []
  },
  "cli_example": null,
  "start_at": "2020-09-28 21:29:37 UTC",
  "available_actions": {
    "cancellable": false,
    "resumable": false
  }
}

$ curl -X GET -s -k -u admin:changeme http://lambda.sparta.example.com:3000/katello/api/content_view_versions/export_histories|jq
{
  "total": 1,
  "subtotal": 1,
  "page": 1,
  "per_page": 20,
  "error": null,
  "search": null,
  "sort": {
    "by": "id",
    "order": "asc"
  },
  "results": [
    {
      "destination_server": "downstream1.example.com",
      "path": "/var/lib/pulp/exports/export-15903/view/1.0/downstream1.example.com/2020-09-28T21-29-38-00-00",
      "id": 3,
      "created_at": "2020-09-28 21:29:40 UTC",
      "updated_at": "2020-09-28 21:29:40 UTC",
      "content_view_version": "view 1.0",
      "content_view_version_id": 3
    }
  ]
}

$ ls /var/lib/pulp/exports/export-15903/view/1.0/downstream1.example.com/2020-09-28T21-29-38-00-00
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0000  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0012  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0024
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0001  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0013  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0025
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0002  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0014  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0026
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0003  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0015  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0027
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0004  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0016  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0028
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0005  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0017  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0029
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0006  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0018  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0030
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0007  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0019  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0031
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0008  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0020  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0032
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0009  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0021  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129-toc.json
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0010  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0022  metadata.json
export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0011  export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0023

$ cat /var/lib/pulp/exports/export-15903/view/1.0/downstream1.example.com/2020-09-28T21-29-38-00-00/export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129-toc.json|jq
{
  "meta": {
    "chunk_size": 10240,
    "file": "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz",
    "global_hash": "9c4efbd635f9d6c3ece63d65e468ddbb4c6b7d19a9af31f4b6ea04af7a25e408"
  },
  "files": {
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0000": "8873e34a798034d790d9554c9352e0c48d3799ee5ac53dcc5549a8b886e44485",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0001": "de7ad68897ef4f0996d5846f070caeda5e714d233204e6846f07964ad11612f1",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0002": "d9886fec42e3089462d27d2926e2bda6196b56cbf083c540c39aaa29595e5b96",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0003": "894af6ab239102a0074b39c1a4721440be16acdb9e81b243e8e811b4db035443",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0004": "801b8a9e115075ae464858d8071c0ec85044986e17302a6cdad1b1517fadf1cb",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0005": "ffcd141d5ab5dd26c72f4521f9c066677ba4503481b7cada4ac8661fe627a485",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0006": "336711a2448a49892386781aa57c6d38bd984a3ad4b832d1beb7221cf8180401",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0007": "344d31e965419662c484bad93124059ae5b23865ddde262df1457703bef9e066",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0008": "3e52dd59a1b6c3355d4d74e34833337bf955dbe7df792b5c2bb95db7c88083dc",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0009": "cdf85532944ec2f48dad79a8deacca5486e36c87e7ced7b7e0a63fd4ac35cb38",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0010": "6b3907269712c9f892ca96cd834c7f445ccbb0b5f367f25aeae8eeb53b5fa69d",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0011": "768f68ec2a86bcb94ce3b29a58cd6ad7987ab4669332c027009e56b371f0bbb0",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0012": "7a8e5f9dfef3e863eddc840d094a304c9433bf66394c2bb01d11bec2ecfe51a7",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0013": "cad2e4450a132f494a49c087cdf51bd6b39b6d72821954b071f9ed631e997a77",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0014": "2f1d35280fcbb11f3befa843e515ceb3139e352b1982406f2466c9d4b42963f2",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0015": "4429f5c858684d890589e0a5b8433a1488493d5e92dc50c44b452ad1097f8331",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0016": "fdac85bdd11045c4b941bb1b23e2714d38874d90aed03f7467e3de63eae9df53",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0017": "70d95b558ebef44ff8c579c514cb40d358e670597f4989f06adfa4db001cf2c7",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0018": "a07e2739ee8a6eb99423741a5360d34fe8e78c649234abfe4afd4b4f61168ad2",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0019": "c50c59acc95903e0282bd67ed5da310d293e75ed138adf9b1397d63332b4fdce",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0020": "0d27eea3a035f079e38a0d2b9f5f98afaf5a09995cd626cf735d5e77677715e0",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0021": "589e56614ab389623428d705b5e667e06a0ebc4b693e890b30585d5c03825418",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0022": "6ec269a3a1793990eadfac75b56e824faf8dbf8d30657947440c15e51d6556cb",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0023": "b49fd086d46fff81d53d2eb44451c41e2a0458062cc2d9b41c6a7c6b5b605de0",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0024": "38a25bb1219fc6541dfb09f64b0f3a2a2cda9ce4e212630fafb582ad2b0b1c5b",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0025": "6d330584343c88d8397941fb16e91fca5d863f2106dd30d82b544c395a19ef63",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0026": "3887680a8ca5c10461cb86e5b6fec735343a9e38fcecca8882e93b84185eb201",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0027": "3e0ece16d92598b0fcae9c79452c41cabd968d03a6f88d35cbbfb7119dc6a2cf",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0028": "8caad0fb7da80d14c1937feca835a638900090b8092dba19d6915b2075ba87b4",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0029": "0ead8b189750bdfb02adf19d188820e877ba407672755a3595142f114c2952d9",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0030": "e371ad276fe6dd55f04474ae1f85c25252ef824177ce6c24d7fb0f02ab2b9cb4",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0031": "aa4de350725a7d358bb7161fd9a64c2d7326ae44e988178da521db280534d1ab",
    "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129.tar.gz.0032": "b16e477308de52a207e3b2a068016747ee68c78b62fb9e51f08dc8f1bb6c67bc"
  }
}

$ cat /var/lib/pulp/exports/export-15903/view/1.0/downstream1.example.com/2020-09-28T21-29-38-00-00/metadata.json|jq
{
  "organization": "export-15903",
  "repository_mapping": {
    "katello-1919": {
      "repository": "katello",
      "product": "prod",
      "redhat": false
    },
    "misc-21682": {
      "repository": "misc",
      "product": "prod",
      "redhat": false
    }
  },
  "content_view": "view",
  "content_view_version": {
    "major": 1,
    "minor": 0
  },
  "toc": "export-7eb6834e-2c77-4f40-9e8e-36997a93fdf2-20200928_2129-toc.json"
}

[theforeman-bot]

Issues: #30928

[parthaa]

Blocked by #8932

[jlsherrill]

APJ tested both with and without the chunk size parameter and results behaved as expected.