Changelog updates

.gitignore

@@ -9,6 +9,7 @@ build/
 dist/
 MANIFEST
 README.txt
+CHANGELOG
 youtube-dl.1
 youtube-dl.bash-completion
 youtube-dl

CHANGELOG.md

@@ -0,0 +1,15 @@
+<!-- The section for the version in developement must be titled '@DEV@' -->
+# 2013.01.02  Codename: GIULIA
+
+* Add support for ComedyCentral clips <nto>
+* Corrected Vimeo description fetching <Nick Daniels>
+* Added the --no-post-overwrites argument <Barbu Paul - Gheorghe>
+* --verbose offers more environment info
+* New info_dict field: uploader_id
+* New updates system, with signature checking
+* New IEs: NBA, JustinTV, FunnyOrDie, TweetReel, Steam, Ustream
+* Fixed IEs: BlipTv
+* Fixed for Python 3 IEs: Xvideo, Youku, XNXX, Dailymotion, Vimeo, InfoQ
+* Simplified IEs and test code
+* Various (Python 3 and other) fixes
+* Revamped and expanded tests

Makefile

@@ -1,7 +1,7 @@
 all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
 
 clean:
-	rm -rf youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
+	rm -rf youtube-dl.1 youtube-dl.bash-completion README.txt CHANGELOG MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe
@@ -54,6 +54,9 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
+CHANGELOG: CHANGELOG.md
+	pandoc -f markdown -t plain $< -o $@
+
 youtube-dl.1: README.md
 	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
@@ -62,7 +65,7 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co
 
 bash-completion: youtube-dl.bash-completion
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+youtube-dl.tar.gz: youtube-dl README.md README.txt CHANGELOG.md CHANGELOG youtube-dl.1 youtube-dl.bash-completion
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -74,6 +77,6 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude 'testdata' \
 		-- \
 		bin devscripts test youtube_dl \
-		CHANGELOG LICENSE README.md README.txt \
+		CHANGELOG.md CHANGELOG LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
 		youtube-dl

devscripts/extract-changes.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+import sys
+import json
+
+target_version = sys.argv[1]
+
+def empty_doc():
+    return  [{"unMeta":{}}, []]
+
+doc = json.loads(input())
+body = doc[1]
+
+
+# We get the text for each version
+versions = {}
+current_version = None
+for el in body:
+    el.keys()
+    type_ = list(el.keys())[0]
+    if type_ == 'Header':
+        current_version = el['Header'][2][0]['Str']
+        versions[current_version] = empty_doc()
+    elif current_version is not None:
+        versions[current_version][1].append(el)
+
+# We get the document for the target version and create the json string for pandoc
+version_changelog = versions.get(target_version, empty_doc())
+print(json.dumps(version_changelog))
+

devscripts/gh-pages/add-version.py

@@ -5,7 +5,6 @@
 import hashlib
 import os.path
 
-
 if len(sys.argv) <= 1:
     print('Specify the version number as parameter')
     sys.exit()
@@ -35,6 +34,11 @@
     sha256sum = hashlib.sha256(data).hexdigest()
     new_version[key] = (url, sha256sum)
 
+with open(os.path.join(build_dir, 'CHANGELOG.md'), 'rt') as f:
+    changes = f.read().strip()
+    if changes:
+        new_version['changelog'] = changes
+
 versions_info['versions'][version] = new_version
 versions_info['latest'] = version
 

devscripts/gh-pages/update-feed.py

@@ -5,6 +5,7 @@
 import textwrap
 
 import json
+import subprocess
 
 atom_template=textwrap.dedent("""\
 								<?xml version='1.0' encoding='utf-8'?>
@@ -22,7 +23,8 @@
 									<atom:link href="http://rg3.github.io/youtube-dl" />
 									<atom:content type="xhtml">
 										<div xmlns="http://www.w3.org/1999/xhtml">
-											Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a>
+											<p>Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a></p>
+											<p>@CHANGES@</p>
 										</div>
 									</atom:content>
 									<atom:author>
@@ -46,6 +48,14 @@
 for v in versions:
 	entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
 	entry = entry.replace('@VERSION@',v)
+	changes = versions_info['versions'][v].get('changelog', '')
+	if changes:
+		# We only convert the changelog to html with pandoc if it's not an empty string
+		pandoc_cmd = ['pandoc', '--from', 'markdown', '--to', 'html']
+		pandoc = subprocess.Popen(pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+		changes = pandoc.communicate(input=changes.encode('utf-8'))[0].decode('utf-8')
+		changes = u'Changelog: {}'.format(changes)
+	entry = entry.replace('@CHANGES@', changes)
 	entries.append(entry)
 
 entries_str = textwrap.indent(''.join(entries), '\t')

devscripts/release.sh

@@ -23,7 +23,7 @@ fi
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+if [ ! -z "`git status --porcelain | grep -v CHANGELOG.md`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
@@ -34,12 +34,13 @@ else
     nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
 fi
 
-/bin/echo -e "\n### Changing version in version.py..."
+/bin/echo -e "\n### Changing version in version.py and CHANGELOG..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+sed -i "s/@DEV@/$version/" CHANGELOG.md
 
-/bin/echo -e "\n### Committing CHANGELOG README.md and youtube_dl/version.py..."
+/bin/echo -e "\n### Committing CHANGELOG.md README.md and youtube_dl/version.py..."
 make README.md
-git add CHANGELOG README.md youtube_dl/version.py
+git add CHANGELOG.md README.md youtube_dl/version.py
 git commit -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."
@@ -73,6 +74,9 @@ scp -r "build/$version" ytdl@yt-dl.org:html/tmp/
 ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/"
 ssh ytdl@yt-dl.org "sh html/update_latest.sh $version"
 
+echo -e "\n### Generating changelog"
+pandoc CHANGELOG.md -t json | ./devscripts/extract-changes.py ${version} | pandoc -f json -o build/$version/CHANGELOG.md
+
 /bin/echo -e "\n### Now switching to gh-pages..."
 git clone --branch gh-pages --single-branch . build/gh-pages
 ROOT=$(pwd)