youtubeassist

#!/bin/bash

##
## command line tools for working with YouTube.
## See: https://developers.google.com/youtube/v3/docs/
##

config=$HOME/.config/youtubeassist/config

if [ -f "$config" ] ; then
  . $config
else
  echo "Missing config file: $config"
  exit
fi

API_BASE=https://www.googleapis.com/youtube/v3

function usage() {
  cmd="Usage: $(basename $0)"
  echo "$cmd: -a init"
  echo "$cmd: -a subscriptions"
  echo "$cmd: -a subscription-delete -i id"
  echo "$cmd: -a playlists"
  echo "$cmd: -a playlist {-i id} {-n name}"
  echo "$cmd: -a playlist-delete {-i id} {-n name}"
  echo "$cmd: -a playlist-rename -i id -n new-name"
  echo "$cmd: -a playlist-create -n name"
  echo "$cmd: -a playlist-update {-n name} {-i id} [-p public|private|unlisted] [-t title]"
  echo "$cmd: -a playlist-age -n playlist-name -t ttl-in-days  -p preview|process [ -d directory-to-save-to ] [ -o playlist-to-output-to ] "
  echo "$cmd: -a playlist-item-add -n playlist-name -i video-id"
  echo "video_id ... | $cmd: -a playlist-items-add -n playlist-name"
  echo "$cmd: -a playlist-item-delete -n playlist-name -i video-id"
  echo "$cmd: -a playlists-rotate"
  echo "$cmd: -a video -i id"
  echo "$cmd: -a my-channel -i id"
  echo "$cmd: -a playlists-snapshot -d directory"
  echo "$cmd: -a search -q query [-o output] [-F] [-s date|rating|relevance|viewCount]"
  exit
}


date_cmd() {
  case $(uname) in
    Darwin) echo gdate ;;
    *) echo date ;;
  esac

}

age() {
  date=$(date_cmd)
  before=$($date +%s -d "$1")
  now=$($date +%s)
  echo $(((now - before) / (60 * 60 * 24)))
}

playlist_id_by_name() {
  n=$1
  if [ -n "$n" ] ; then
    is_id=$(echo $n | grep ^PL..........................)

    if [ -n "$is_id" ] ; then
      echo $n
    else
      found=$(youtubeassist -a playlists | grep "[|]${n}$" | cut -d'|' -f1 | head -1)
      if [ -n "$found" ] ; then
        echo $found
      else
        echo ""
      fi
    fi
  else
    echo ""
  fi
}


resolve_dir() {
  d="$1"
  if [ -d "$d" -a -w "$d" ];then
    echo $d
  else
    echo ""
  fi
}

search_history_dir() {
  q="$1"
  s="$2"
  search_name=$(echo "${q}-${s}" | sed 's/[^A-Za-z0-9-]//g')
  search_id=$(echo -n "${q}-${s}" | cksum | cut -d' '  -f1)
  echo "$SEARCH_HISTORY_DIR/${search_name}.${search_id}"
}

last_modified() {
  file="$1"

  case $(uname) in
    Darwin) stat -f '%m' $file ;;
    *) stat --format '%Y' $file ;;
  esac
}

search_cache_age() {
  q="$1"
  s="$2"
  dir=$(search_history_dir "$q" "$s")
  a_year_ago=$((60*60*24*365))

  if [ -d "$dir" ] ; then
    last_file=$(cd $dir ; ls -t | head -1)
    if [ -z "$last_file" ] ; then
      echo $a_year_ago
    else
      now=$($(date_cmd) +%s)
      t=$(echo $last_file | sed 's/[.].*//')
      echo $((now - t))
    fi
  else
    echo $a_year_ago
  fi
}

search_execute() {
  q="$1"
  s="$2"
  history_dir=$(search_history_dir "$q" "$s")
  mkdir -p $history_dir
  history_file=$history_dir/$($(date_cmd) +%s.%N).history
  touch $history_file
  
  if [ ! -w "$history_file" ] ; then
    echo "Can't write to $history_file"
    exit
  fi
  
  MAX_PAGES=4
  invoke -G $API_BASE/search \
         -d part=snippet \
         -d maxResults=50 \
         -d order=$sort_by \
         --data-urlencode q="$query" \
         -d safeSearch=none \
         -d videoDefinition=high \
         -d type=video > $HOME/.yt.serp
  cat $HOME/.yt.serp | jq -r '.items[] | .id.videoId' > $HOME/.yt.serp.ids
  cat $HOME/.yt.serp.ids | while read id ; do
    found=$(grep ^${id}$ $history_dir/*.history)
    if [ -z "$found" ]; then
      echo $id >> $history_file
      cat $HOME/.yt.serp | jq "[.items[] | select(.id.videoId == \"$id\")][0]" > $SEARCH_CACHE_DIR/$id.hit
    fi
  done

  if [ ! -s "$history_file" ] ; then
    rm $history_file
  fi
}
    
search_data() {
  q="$1"
  s="$2"
  f="$3"
  history_dir=$(search_history_dir "$q" "$s")


  if [ "$f" = "yes" ] ; then
    limit_cmd="cat"
  else
    limit_cmd="head -1"
  fi

  echo '{ "items": [ '
  sep=""
  ls -t $history_dir | $limit_cmd | while read file_name ; do
    ids_file=$history_dir/$file_name
    cat $ids_file | while read id ; do
      echo $sep
      cat $SEARCH_CACHE_DIR/$id.hit
      sep=","
    done
    sep=","
  done

  echo '] }'
}


filter() {
  if [ -z "$VERBOSE" ] ; then
    jq "$@"
  else
    cat
  fi
}

while getopts ":a:i:n:vd:l:o:t:p:q:s:F" opt ; do
  case $opt in
    a) ACTION=$OPTARG ;;
    v) VERBOSE=yes    ;;
    i) id=$OPTARG     ;;
    n) name=$OPTARG     ;;
    d) dir=$OPTARG ;;
    t) ttl=$OPTARG ;;
    o) output=$OPTARG ;;
    p) privacy=$OPTARG ;;
    s) sort_by=$OPTARG ;;
    q) query=$OPTARG ;;
    F) full_results=yes ;;
    \?) usage         ;;
  esac
done

MAX_PAGES=10000

function invoke {
  curl -s -H "Authorization: Bearer $AUTH_TOKEN" "$@" > /tmp/yt.buffer.$$
  next_page=`jq -r '.nextPageToken' < /tmp/yt.buffer.$$`
  num_pages=1

  if [ "$next_page" = "null" -o -z "$next_page" ] ; then
    cat /tmp/yt.buffer.$$
    rm -f /tmp/yt.buffer.$$
  else
    jq '.items'  < /tmp/yt.buffer.$$ | sed 's/^.//'> /tmp/yt.master.$$
    while [ "$next_page" != "null" -a $num_pages -lt $MAX_PAGES ] ; do
      curl -s -H "Authorization: Bearer $AUTH_TOKEN" "$@" -d pageToken=$next_page |
        tee /tmp/yt.buffer.$$ |
        ( echo "," ; jq '.items' | sed 's/^.//' ) >> /tmp/yt.master.$$
      next_page=`jq -r '.nextPageToken' < /tmp/yt.buffer.$$`
      num_pages=$((num_pages + 1))
    done
    rm -f /tmp/yt.buffer.$$
    echo '{ "items" : [ '
    cat /tmp/yt.master.$$
    echo ' ] }'
    rm -f /tmp/yt.master.$$
  fi
}

case $ACTION in
  init)
    gapi_auth -i $CLIENT_ID -p $CLIENT_SECRET  -r $AUTH_REDIRECT_URI -s "$API_SCOPE" init
    exit
    ;;

  my-channel)
    invoke -G $API_BASE/channels \
           -d mine=true \
           -d part=snippet,contentDetails,statistics \
           -d maxResults=50 |
      filter -r '.'
    ;;
  
  subscriptions)
    invoke -G $API_BASE/subscriptions \
           -d mine=true \
           -d part=snippet \
           -d maxResults=50 | 
      filter -r ' .items[] |  .id + "|" + .snippet.title'
    ;;

  subscription-delete)
    if [ -z "$id" ] ; then
      echo "Deleting a subscription requires an id (-i) value"
      usage
    fi
    invoke -X DELETE "$API_BASE/subscriptions?id=$id"
    ;;

  playlists)
    invoke -G $API_BASE/playlists \
           -d mine=true \
           -d part=snippet,status \
           -d maxResults=50 | 
      filter -r ' .items[] |  .id + "|" + .status.privacyStatus + "|" + .snippet.title'
    ;;

  playlist)
    if [ -n "$name" ] ; then
      found=$(playlist_id_by_name "$name")
      if [ -n "$found" ] ; then
        id=$found
      fi
    fi

    if [ -z "$id" ] ; then
      echo "Listing a playlist requires a playlist id"
      usage
    fi
    invoke -G $API_BASE/playlistItems \
           -d playlistId=$id \
           -d part=snippet,status,contentDetails \
           -d maxResults=50 |
      filter -r '.items[] | .id + "|" + .contentDetails.videoId + "|" + .snippet.publishedAt + "|" + .snippet.title' > $HOME/.yt.buffer
    if [ -n "$VERBOSE" ] ; then
       cat $HOME/.yt.buffer
    else
      cat $HOME/.yt.buffer | while read line ; do
        id=$(echo $line | cut -d'|' -f1)
        video_id=$(echo $line | cut -d'|' -f2)
        added=$(echo $line | cut -d'|' -f3)
        age=$(age $added)
        title=$(echo $line | cut -d'|' -f4)
        echo "$id|$video_id|$age|$title"
      done
    fi

    ;;

  playlist-update)
    if [ -n "$name" ] ; then
      id=$(playlist_id_by_name "$name")
      if [ -z "$id" ] ;  then
        echo "$name is not a valid playlist."
        exit
      fi
    fi

    if [ -z "$id" ] ; then
      echo "Updating a playlist requires an ID (-i) value"
      usage
    fi

    if [ -n "$privacy" ] ; then
      case "$privacy" in
        private|public|unlisted) ;;
        *) echo "Invalid -p privacy. Must be: public, private or unlisted" ; exit ;;
      esac
    else
      privacy=$(youtubeassist -a playlists | grep "^${id}|" | cut -d '|' -f2)
    fi

    if [ -n "$ttl" ]  ; then
      title="$ttl"
    else
      title=$(youtubeassist -a playlists | grep "^${id}|" | cut -d'|' -f3)
    fi

    invoke -X PUT "$API_BASE/playlists?part=status,snippet" \
           -H 'Content-Type: application/json' \
           -d "{ 'id': '$id', 'snippet': { 'title': '$title' }, 'status': { 'privacyStatus': '$privacy'} } " |
      filter -r '.id + "|" + .status.privacyStatus + "|" + .snippet.title'

    ;;

  playlist-item-add)
    if [ -z "$name" ] ; then
      echo "Missing -n playlist-name"
      exit
    fi

    playlist_id=$(playlist_id_by_name "$name")

    if [ -z "$playlist_id" ] ; then
      echo "No playlist found with the name $name"
    fi

    if [ -z "$id" ] ; then
       echo "Missing -i video-id"
       exit
    fi

    exists=$(youtubeassist -a playlist -i $playlist_id | grep "|$id|")
    if [ -n "$exists" ]  ; then
      echo $exists
      exit
    fi

    invoke -X POST "$API_BASE/playlistItems?part=snippet" \
           -H 'Content-Type: application/json' \
           -d "{ 'snippet': { 'playlistId': '$playlist_id', 'position': 0, 'resourceId': { 'kind': 'youtube#video', 'videoId': '$id' } } } " |
      filter -r '.id + "|" + .snippet.title'

    ;;

  playlist-items-add)
    if [ -z "$name" ] ; then
      echo "Missing -n playlist-name"
      exit
    fi

    playlist_id=$(playlist_id_by_name "$name")

    if [ -z "$playlist_id" ] ; then
      echo "No playlist found with the name $name"
    fi

    youtubeassist -a playlist -i $playlist_id > $HOME/.yt.pl
    while read id ; do
         exists=$(grep "|$id|" $HOME/.yt.pl)
         if [ -z "$exists" ]  ; then
           invoke -X POST "$API_BASE/playlistItems?part=snippet" \
                  -H 'Content-Type: application/json' \
                  -d "{ 'snippet': { 'playlistId': '$playlist_id', 'position': 0, 'resourceId': { 'kind': 'youtube#video', 'videoId': '$id' } } } " |
             filter -r '.id + "|" + .snippet.title'
         fi
    done
    ;;

  playlist-item-delete)
    if [ -z "$name" ] ; then
      echo "Missing -n playlist-name"
      exit
    fi

    playlist_id=$(playlist_id_by_name "$name")

    if [ -z "$playlist_id" ] ; then
      echo "No playlist found with the name $name"
    fi

    if [ -z "$id" ] ; then
      echo "Missing -i video-id"
      exit
    fi

    youtubeassist -a playlist -i $playlist_id | while read line ; do
      item_id=$(echo $line | cut -d'|' -f1)
      video_id=$(echo $line | cut -d'|' -f2)
      if [ "$video_id" = "$id" ] ; then
        invoke -X DELETE "$API_BASE/playlistItems?id=$item_id"
      fi
    done

    ;;

  playlist-delete)
    if [ -n "$name" ] ; then
      id=$(playlist_id_by_name "$name")
      if [ -z "$id" ] ;  then
        echo "$name is not a valid playlist."
        exit
      fi
    fi

    if [ -z "$id" ] ; then
      echo "Deleting a playlist requires an ID (-i) value"
      usage
    fi
    invoke -X DELETE "$API_BASE/playlists?id=$id"
    ;;

  playlist-rename)
    if [ -z "$id" -o -z "$name" ] ; then
      echo "Renaming a playlist requires you set the ID (-i) and name (-n)"
      usage
    fi
    invoke -X PUT "$API_BASE/playlists?part=snippet" \
           -H 'Content-Type: application/json' \
           -d "{'id': '$ID', 'snippet': {'title' : '$name'} }" |
      filter -r '.id + "|" + .snippet.title'
    ;;

  playlist-create)
    if [ -z "$name" ] ; then
      echo "Missing -n name"
      exit
    fi

    invoke -X POST "$API_BASE/playlists?part=snippet,status" \
           -H 'Content-Type: application/json' \
           -d "{'snippet': {'title' : '$name'}, 'status' : {'privacyStatus' : 'private'} }" |
      filter -r '.id + "|" + .status.privacyStatus + "|" + .snippet.title'
    ;;

  video)
    if [ -z "$id" ] ; then
      echo "Video info requires an ID (-i)"
      usage
    fi
    invoke -G $API_BASE/videos \
           -d id=$id \
           -d part=snippet,id,topicDetails,contentDetails,statistics,status,player \
           -d maxResults=50 |
      filter -r '.items[] | .id + ":" + .snippet.title'
    ;;

  playlists-snapshot)
    if [ -z "$dir" ] ; then
      echo "-d directory is missing"
      exit
    fi

    path=$(realpath -s $dir/$($(date_cmd) +%Y%m%d.%H%M))
    if [ -d "$path" -a -w "$path" ] ; then
      echo "Archive directory exists: $path"
      exit
    fi

    mkdir -p "$path"
    if [ ! -d "$path" ] ; then
      echo "Failed to make directory: $path"
      exit
    fi

    youtubeassist -a playlists | while read line ; do
      id=$(echo $line | cut -d'|' -f1)
      name=$(echo $line | cut -d'|' -f3)
      file=$(echo $name | sed -e 's/[^A-Za-z0-9]/_/g' \
                              -e 's/^_*//' \
                              -e 's/_*$//' \
                              -e 's/__*/_/g').json
      youtubeassist -a playlist -v -i $id > $path/$file
    done
    ;;

  playlists-rotate)
    archive_dir=$ROTATE_DIR
    if [ ! -d "$archive_dir" ] ; then
      echo "Need to setup archive directory: $archive_dir"
      exit
    fi

    for rule in $ROTATE_RULES ; do
      name=$(echo $rule | cut -d: -f1 | tr '_' ' ')
      ttl=$(echo $rule | cut -d: -f2)
      output=$(echo $rule | cut -d: -f3)
      directory=$(echo $rule | cut -d: -f4)

      if [ -n "$directory" ] ; then
        mkdir -p "$archive_dir/$directory"
        if [ -n "$VERBOSE" ] ; then
          echo "name=$name, ttl=$ttl, dir=$archive_dir/$directory"
        fi
        youtubeassist -a playlist-age -n "$name" -t $ttl -d "$archive_dir/$directory" -p process
      fi

      if [ -n "$output" ] ; then
        if [ -n "$VERBOSE" ] ; then
          echo "name=$name, ttl=$ttl, output=$output"
        fi
        youtubeassist -a playlist-age -n "$name" -t $ttl -o "$output" -p process
      fi
    done
    ;;



  playlist-age)
    id=$(playlist_id_by_name "$name")
    if [ -z "$id" ]  ;then
      echo "-n playlist not found"
      exit
    fi

    if [ -z "$privacy" ] ; then
      echo "Missing directive -p preview|process"
      exit
    fi

    case "$privacy" in
      preview|process) goal=$privacy ;;
      *) echo "Invalid '$privacy', must be preview or process." ; exit ;;
    esac

    output_dir=$(resolve_dir $dir)
    output_id=$(playlist_id_by_name $output)

    if [ -z "$dir" -a -z "$output" ] ; then
      echo "Must set -d dir or -o output"
      exit
    fi

    if [ -n "$dir" -a -z "$output_dir" ] ; then
      echo "-d $dir doesn't exist"
      exit
    fi

    if [ -n "$output" -a -z "$output_id" ] ; then
      echo "-o $output doesn't exist"
      exit
    fi

    if [ -z "$ttl" ] ; then
      echo "-t ttl was not set. Must be a number of days."
      exit
    fi

    youtubeassist -a playlist -i $id | while read line ; do
      id=$(echo $line | cut -d'|' -f1)
      video_id=$(echo $line | cut -d'|' -f2)
      age=$(echo $line | cut -d'|' -f3)
      title=$(echo $line | cut -d'|' -f4)
      if [ $age -gt $ttl ]  ; then
        case $goal in
          preview)
            echo "Aging '$title', $age > $ttl"
            ;;
          process)
            if [ -n "$output_id" ] ; then
              youtubeassist -a playlist-item-add -n "$output" -i "$video_id" > /dev/null
            fi

            if [ -n "$output_dir" ] ; then
              youtubeassist -a video -i "$video_id" -v  > $output_dir/$video_id.json
            fi


            youtubeassist -a playlist-item-delete -n "$name" -i $video_id > /dev/null
        esac
      fi
    done
    ;;

  search)
    if [ -z "$query" ]; then
      echo "Missing -q query"
      exit
    fi

    if [ -z "$SEARCH_HISTORY_DIR" ] ; then
      echo "You must set a SEARCH_HISTORY_DIR"
      exit
    fi

    if [ -z "$SEARCH_CACHE_DIR" ] ; then
      echo "You must set SEARCH_CACHE_DIR"
      exit
    fi

    if [ ! -d "$SEARCH_HISTORY_DIR" ] ; then
      echo "Search $SEARCH_HISTORY_DIR doesn't exist."
      exit
    fi

    if [ ! -d "$SEARCH_CACHE_DIR" ] ; then
      echo "Search $SEARCH_CACHE_DIR doesn't exist."
      exit
    fi

    if [ -z "$sort_by" ]; then
      sort_by=date
    fi

    if [ -n "$sort_by" ] ; then
      case $sort_by in
        date|rating|relevance|title|viewCount) ;;
        *) echo "Invalid -s $sort_by" ;  exit ;;
      esac
    fi

    if [ -n "$output" ] ; then
      output_id=$(playlist_id_by_name "$output")
      if [ -z "$output_id" ] ; then
        echo "-o $output does not exist"
        exit
      fi
    fi

    age=$(search_cache_age "$query" "$sort_by")
    one_hour_ago=$((60 * 60))
    if [ $age -gt $one_hour_ago ] ; then
      search_execute "$query" "$sort_by"
    fi

    search_data "$query" "$sort_by" "$full_results" > $HOME/.yt.data
    if [ -n "$output_id" ] ; then
      cat $HOME/.yt.data | jq -r '.items[] | .id.videoId' | while read video_id ; do
        youtubeassist -a playlist-item-add -n "$output_id" -i "$video_id" > /dev/null
      done
    fi

    cat $HOME/.yt.data |
      filter -r '.items[] | .id.videoId + "|" + .snippet.publishTime + "|" + .snippet.channelTitle + "|" + .snippet.title'

    ;;

  *)
    usage
    ;;
esac