Add per-platform doc-scraper support. Updated README a lot.

Now we can build API level 8 dll from jar2xml, but some API difference
blocks build completion at samples build. The build fetches old SDK docs
from sdk download repositry, which is >50MB.

Makefile

@@ -7,6 +7,7 @@ clean:
 	-rm $(TARGET)
 	-rm -rf tmpout
 	-rm -rf annotations
+	-rm -rf scraper.exe scraper.exe.mdb
 
 sources = \
 	AndroidDocScraper.java \
@@ -22,19 +23,28 @@ $(TARGET): $(sources)
 	javac -g -d obj $(sources)
 	jar cfe "$@" jar2xml.Start -C obj/ .
 
-test-8: 8.xml annotations/8.xml
-	java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-8/android.jar --out=_8.xml --docpath=$(ANDROID_SDK_PATH)/docs/reference --annotations=annotations/8.xml || exit 1
+scraper.exe : scraper.cs
+	mcs -debug scraper.cs
+
+# test API levels
+
+test-8: docs-2.2_r01-linux 8.xml annotations/8.xml
+	java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-8/android.jar --out=_8.xml --docpath=docs-08/reference --annotations=annotations/8.xml || exit 1
 	mono-xmltool --prettyprint _8.xml > _8_.xml || exit 1
 	xmlstarlet c14n _8_.xml > _8.xml || exit 1
 	rm _8_.xml
 
+docs-2.2_r01-linux:
+	wget http://dl-ssl.google.com/android/repository/docs-2.2_r01-linux.zip || exit 1
+	unzip docs-2.2_r01-linux.zip || exit 1
+
 8.xml:
 	wget --output-document=8.xml "http://android.git.kernel.org/?p=platform/frameworks/base.git;a=blob_plain;f=api/8.xml;hb=HEAD" || exit 1
 	mono-xmltool --prettyprint 8.xml > 8_.xml || exit 1
 	xmlstarlet c14n 8_.xml > 8.xml || exit 1
 	rm 8_.xml
 
-# These rules are to get diff between 10 and 13.
+# (These rules are to get diff between 10 and 13.)
 test-13: 10.xml annotations/13.xml
 	java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-13/android.jar --out=_13.xml --docpath=$(ANDROID_SDK_PATH)/docs/reference --annotations=annotations/13.xml || exit 1
 	mono-xmltool --prettyprint _13.xml > _13_.xml || exit 1
@@ -47,20 +57,20 @@ test-13: 10.xml annotations/13.xml
 	xmlstarlet c14n 10_.xml > 10.xml || exit 1
 	rm 10_.xml
 
+# annotations
+
 annotations/8.xml: scraper.exe tmpout/8-deprecated-members.xml
 	mkdir -p annotations
-	mono --debug scraper.exe tmpout/8-deprecated-members.xml > annotations/8.xml
+	mono --debug scraper.exe tmpout/8-deprecated-members.xml docs-2.2_r01-linux/reference/ > annotations/8.xml
 
 tmpout/8-deprecated-members.xml : scraper-main.sh scraper-collector.sh
 	mkdir -p tmpout
-	bash scraper-main.sh > tmpout/8-deprecated-members.xml
+	bash scraper-main.sh docs-2.2_r01-linux/reference > tmpout/8-deprecated-members.xml || exit 1
 
 annotations/13.xml: scraper.exe tmpout/13-deprecated-members.xml
 	mkdir -p annotations
 	mono --debug scraper.exe tmpout/13-deprecated-members.xml > annotations/13.xml
 
 tmpout/13-deprecated-members.xml : scraper-main.sh scraper-collector.sh
 	mkdir -p tmpout
-	bash scraper-main.sh > tmpout/13-deprecated-members.xml
-scraper.exe : scraper.cs
-	mcs -debug scraper.cs
+	bash scraper-main.sh > tmpout/13-deprecated-members.xml || exit 1

README

@@ -13,8 +13,11 @@ and associated documentation.  The information is stored in XML format.
 
 	- jar2xml : the main converter to generate the primary target xmls.
 	- scraper.exe : annotation doc scraper to generate annotations/*.xml.
+	  scraper-*.sh : shells used for scraper input. They are piped.
 
-* Easy comparison
+* <del>Easy</del> comparison
+
+well, now it's getting much more complicated than it used to be like below...
 
 	$ java -jar jar2xml.jar --jar=/home/atsushi/android-sdk-linux_x86/platforms/android-8/android.jar --out=_8.xml --docpath=/home/atsushi/android-sdk-linux_x86/docs/reference
 	$ mono-xmltool --prettyprint 8.xml > 8_.xml
@@ -23,7 +26,8 @@ and associated documentation.  The information is stored in XML format.
 	$ xmlstarlet c14n _8_.xml > _8.xml
 	$ diff -u 8.xml _8.xml
 
-It is also implemented as "make test-8" as well as "make test-13".
+It is now implemented as "make test-8" as well as "make test-13" with more
+detailed arguments. You should rather check Makefile to see what it does.
 
 
 * Difference from AOSP API XML
@@ -57,9 +61,6 @@ It is also implemented as "make test-8" as well as "make test-13".
 
 * TODO
 
-- Move methods in non-public base type to the public derived type e.g.
-  AbstractStringBuilder.append() should move to StringBuffer and StringBuilder.
-
 - Do not skip certain overriden methods. This in JavaClass.java gives
   equivalent output to AOSP xml, but it's not good:
 
@@ -88,7 +89,21 @@ It is also implemented as "make test-8" as well as "make test-13".
   in android.jar. That includes, types themselves.
   java.lang.annotation types suffer from this.
 
-- java.lang.AbstractStringBuilder exposes another issue that
+  I find it almost desparate and the only solution I can think of is
+  to build AOSP android.jar or some equivalence that does *not* strip
+  those annotations off, *for each platform version*.
+  That sounds so time-eater.
+
+  The AOSP API document is generated as out/target/common/obj/PACKAGING/android_jar_intermediaries/public_api.xml (at that state there still must be annotations, or this xml should not contain annotation information at all).
+  Also, 
+  out/target/common/obj/JAVA_LIBRARIES/framework_intermediaries and
+  out/target/common/obj/JAVA_LIBRARIES/core_intermediaries
+  contains debug jars that still seem to hold annotations.
+
+  This is from the latest AOSP build (as of Aug.17, 2011). I'm not sure
+  if this applies to older versions.
+
+- java.lang.AbstractStringBuilder exposes an issue that
   interface members on non-public types are returned by
   class#getDeclaredMethods() while they are *not* declared.
 
@@ -98,18 +113,12 @@ It is also implemented as "make test-8" as well as "make test-13".
   excluded, but this brings another check to *not* exclude methods when
   the corresponding base methods are excluded).
 
-- Some classes exists in the output xml but not in the dll:
-  - android.database.ContentObservable (derived from Observable<ContentObserver>)
-  - android.net.NetworkInfo.State (enum type)
-
 - Some fields seem to be incorrectly marked as constant. For example,
   android.os.Build.TIME is not a constant, but since the field.getLong()
   returns 0, it is set as 0 (and it does not return null for get()).
 
-- Doc scraper still misses some parameter names
-  e.g. android.text.method.ScrollingMovementMethod.onKeyUp()
-  This matters when api-fixup is involved (it matches nodes by parameter
-  names and then it dismisses the target nodes!).
+  java.io.File.pathSeparatorChar has the same problem, but it is worse;
+  it returns non-zero value so we cannot depend on the value.
 
 
 * Notes on Annotations
@@ -147,3 +156,20 @@ makefiles before confirming that it's all good).
 [*2] http://asm.ow2.org/asm33/javadoc/user/org/objectweb/asm/ClassReader.html
 
 
+* Notes on per-version doc scraping
+
+We used to scrape only reference for the latest API which was problematic.
+Now we have version-aware scraper and resolved the issue described below.
+
+<historical>
+  Doc scraper still misses some parameter names
+  e.g. android.text.method.ScrollingMovementMethod.onKeyUp()
+  This matters when api-fixup is involved (it matches nodes by parameter
+  names and then it dismisses the target nodes!).
+
+  Actually this is not going to be very good; there are some methods
+  that are removed in the latest API (the example method above is exactly
+  the case). We'll have to sort out some alternative approach.
+  A possible approach is to checkout API docs for *every* version (from
+  AOSP).
+</historical>

doc-locations.txt

@@ -0,0 +1,7 @@
+13: http://dl-ssl.google.com/android/repository/docs-3.2_r01-linux.zip
+12: http://dl-ssl.google.com/android/repository/docs-3.1_r01-linux.zip
+11: http://dl-ssl.google.com/android/repository/docs-3.0_r01-linux.zip
+10: ?
+09: http://dl-ssl.google.com/android/repository/docs-2.3_r01-linux.zip
+08: http://dl-ssl.google.com/android/repository/docs-2.2_r01-linux.zip
+07: http://dl-ssl.google.com/android/repository/docs-2.1_r01-linux.zip

scraper-collector.sh

@@ -1,8 +1,16 @@
+#!/bin/bash
 
-echo "<deprecated>"
-for f in `find $ANDROID_SDK_PATH/docs/reference -name *.html` 
+if [ "$1" ]; then
+	export docpath="$1"
+else
+	export docpath="$ANDROID_SDK_PATH/docs/reference"
+fi
+
+echo "<deprecated basepath='$docpath'>"
+for f in `find $docpath -name *.html` 
 do
-	echo "<file name='$f'>"
+	export len=`expr length "$docpath" + 1`
+	echo "<file name='${f:$len}'>"
 	echo "<fields>"
 	xmllint --html --xmlout $f 2>/dev/null | xmlstarlet sel -t -c "//p[@class='caution']/../../h4[@class='jd-details-title']/text()"
 	echo "</fields>"

scraper-main.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 echo "<deprecated>"
-bash scraper-collector.sh | xmlstarlet sel -t -c "//file[normalize-space(fields/text()) or normalize-space(methods/text())]" 
+bash scraper-collector.sh "$1" | xmlstarlet sel -t -c  "//file[normalize-space(fields/text()) or normalize-space(methods/text())]" 
 echo "</deprecated>"

scraper.cs

@@ -10,7 +10,6 @@ public class Scraper
 {
 	public static int Main (string [] args)
 	{
-		var extraPath = Path.Combine (Environment.GetEnvironmentVariable ("ANDROID_SDK_PATH"), "docs/reference") + "/";
 		var doc = new XmlDocument ();
 		doc.Load (args [0]);
 		Console.WriteLine ("<deprecated>");
@@ -37,7 +36,7 @@ public static int Main (string [] args)
 			}
 			if (output.Length == 0)
 				continue;
-			string name = file.GetAttribute ("name").Substring (extraPath.Length);
+			string name = file.GetAttribute ("name");
 			name = name.Substring (0, name.Length - 5); // .html
 			Console.WriteLine ("<file name='{0}'>\n{1}</file>", name, output);
 		}