commonsbot/convert_to_pyramidal_tiled_tiff.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Script to convert generic TIFFs to pyramidal tiled\n",
    "\n",
    "See key post at https://legacy.imagemagick.org/discourse-server/viewtopic.php?t=20193 for background on using command-line ImageMagick to do the conversion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "source_image_root_directory = '/users/baskausj/gallery_digital_image_archive/'\n",
    "destination_image_root_directory = '/users/baskausj/gallery_pyramidal_tiffs/'\n",
    "\n",
    "image_data_frame = pd.read_csv('/users/baskausj/github/vandycite/gallery_works/image_upload/image_dimensions.csv')\n",
    "# Convert some columns to integers\n",
    "image_data_frame[['kilobytes', 'height', 'width']] = image_data_frame[['kilobytes', 'height', 'width']].astype(int)\n",
    "image_data_frame[['subdir']] = image_data_frame[['subdir']].astype(str)\n",
    "\n",
    "'''\n",
    "# For testing purposes, just use the first few rows of the works metadata\n",
    "test_rows = 4\n",
    "image_data_frame = image_data_frame.head(test_rows).copy()\n",
    "'''\n",
    "\n",
    "image_data_frame.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter only rows that have one of the TIFF file extension variants\n",
    "tiff_alt_extensions = ['tif', 'TIF', 'tiff', 'TIFF']   \n",
    "tiffs_frame = image_data_frame[image_data_frame['extension'].isin(tiff_alt_extensions)]\n",
    "tiffs_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic conversion command is\n",
    "# os.system('convert 1979.0342P.tif -define tiff:tile-geometry=256x256 -depth 8 ptif:1979.0342P_tiled.tif')\n",
    "\n",
    "for index, image in tiffs_frame.iterrows():\n",
    "    in_path = source_image_root_directory + image['subdir'] + '/' + image['name']\n",
    "    \n",
    "    # Create the subdirectory if it doesn't already exist\n",
    "    os.makedirs(destination_image_root_directory + image['subdir'], exist_ok=True)\n",
    "    out_path = destination_image_root_directory + image['subdir'] + '/' + image['name']\n",
    "    print(image['name'])\n",
    "    \n",
    "    # Redirect (append) errors to a log file. Based on practical experience, most errors can be ignored.\n",
    "    # Note: need to enclose file paths in quotes because filenames sometimes include spaces.\n",
    "    command_string = 'convert \"' + in_path + '\" -define tiff:tile-geometry=256x256 -depth 8 ptif:\"' + out_path + '\" 2>> ~/gallery_image_conversion_log.txt'\n",
    "    #print(command_string)\n",
    "    os.system(command_string)\n",
    "\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compressing TIFFs using LZW or deflate (a.k.a. zip)\n",
    "\n",
    "See post at https://legacy.imagemagick.org/discourse-server/viewtopic.php?t=13484 for syntax.\n",
    "\n",
    "Note: did a test upload and using either on TIFF is file with Commons. The LZW is relatively ineffective and sometimes actually makes the filesize larger."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic conversion command is\n",
    "# os.system('convert 1994.012_original.tif -compress lzw 1994.012.tif')\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "source_image_root_directory = '/users/baskausj/uncompressed/'\n",
    "destination_image_root_directory = '/users/baskausj/compressed/'\n",
    "\n",
    "image_list_frame = pd.read_csv('/users/baskausj/github/vandycite/gallery_works/image_upload/large_tiffs.csv')\n",
    "image_list_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, image in image_list_frame.iterrows():\n",
    "    in_path = source_image_root_directory + image['name']\n",
    "    out_path = destination_image_root_directory + image['name']\n",
    "    \n",
    "    #command_string = 'convert \"' + in_path + '\" -compress lzw \"' + out_path + '\"'\n",
    "    command_string = 'convert \"' + in_path + '\" -compress zip \"' + out_path + '\"'\n",
    "    print(command_string)\n",
    "    os.system(command_string)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code to grab the filesize of the compressed files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "def write_dicts_to_csv(table, filename, fieldnames):\n",
    "    with open(filename, 'w', newline='', encoding='utf-8') as csv_file_object:\n",
    "        writer = csv.DictWriter(csv_file_object, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "        for row in table:\n",
    "            writer.writerow(row)\n",
    "\n",
    "out_images = []\n",
    "for index, image in image_list_frame.iterrows():\n",
    "    out_image = {'name': image['name']}\n",
    "    in_path = source_image_root_directory + image['name']\n",
    "    out_path = destination_image_root_directory + image['name']\n",
    "\n",
    "    out_image['original_kilobytes'] = round(os.path.getsize(in_path)/1024)\n",
    "    out_image['compressed_kilobytes'] = round(os.path.getsize(out_path)/1024)    \n",
    "    out_images.append(out_image)\n",
    "    \n",
    "fieldnames = list(out_images[0].keys())\n",
    "write_dicts_to_csv(out_images, source_image_root_directory + 'image_dimensions.csv', fieldnames)\n",
    "\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do an inner join with the original file information CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "source_image_root_directory = '/users/baskausj/uncompressed/'\n",
    "source_directory = '/users/baskausj/github/vandycite/gallery_works/image_upload/'\n",
    "\n",
    "left_url = source_image_root_directory + 'image_dimensions_full.csv'\n",
    "right_url = source_directory + 'images.csv'\n",
    "\n",
    "left = pd.read_csv(left_url, na_filter=False, dtype = str)\n",
    "right = pd.read_csv(right_url, na_filter=False, dtype = str)\n",
    "merged_frame = pd.merge(right, left, on='local_filename', how='inner')\n",
    "merged_frame.head()\n",
    "\n",
    "merged_frame.to_csv(source_directory + 'oversized_images.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}