From 731835cfa9bcd19d35425b0bdbbef7553b113a3f Mon Sep 17 00:00:00 2001 From: Atsushi Eno Date: Wed, 17 Aug 2011 18:29:57 +0900 Subject: [PATCH] Add per-platform doc-scraper support. Updated README a lot. Now we can build API level 8 dll from jar2xml, but some API difference blocks build completion at samples build. The build fetches old SDK docs from sdk download repositry, which is >50MB. --- Makefile | 26 ++++++++++++++------- README | 54 ++++++++++++++++++++++++++++++++------------ doc-locations.txt | 7 ++++++ scraper-collector.sh | 14 +++++++++--- scraper-main.sh | 3 ++- scraper.cs | 3 +-- 6 files changed, 79 insertions(+), 28 deletions(-) create mode 100644 doc-locations.txt diff --git a/Makefile b/Makefile index 2320635..55326a2 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ clean: -rm $(TARGET) -rm -rf tmpout -rm -rf annotations + -rm -rf scraper.exe scraper.exe.mdb sources = \ AndroidDocScraper.java \ @@ -22,19 +23,28 @@ $(TARGET): $(sources) javac -g -d obj $(sources) jar cfe "$@" jar2xml.Start -C obj/ . -test-8: 8.xml annotations/8.xml - java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-8/android.jar --out=_8.xml --docpath=$(ANDROID_SDK_PATH)/docs/reference --annotations=annotations/8.xml || exit 1 +scraper.exe : scraper.cs + mcs -debug scraper.cs + +# test API levels + +test-8: docs-2.2_r01-linux 8.xml annotations/8.xml + java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-8/android.jar --out=_8.xml --docpath=docs-08/reference --annotations=annotations/8.xml || exit 1 mono-xmltool --prettyprint _8.xml > _8_.xml || exit 1 xmlstarlet c14n _8_.xml > _8.xml || exit 1 rm _8_.xml +docs-2.2_r01-linux: + wget http://dl-ssl.google.com/android/repository/docs-2.2_r01-linux.zip || exit 1 + unzip docs-2.2_r01-linux.zip || exit 1 + 8.xml: wget --output-document=8.xml "http://android.git.kernel.org/?p=platform/frameworks/base.git;a=blob_plain;f=api/8.xml;hb=HEAD" || exit 1 mono-xmltool --prettyprint 8.xml > 8_.xml || exit 1 xmlstarlet c14n 8_.xml > 8.xml || exit 1 rm 8_.xml -# These rules are to get diff between 10 and 13. +# (These rules are to get diff between 10 and 13.) test-13: 10.xml annotations/13.xml java -jar jar2xml.jar --jar=$(ANDROID_SDK_PATH)/platforms/android-13/android.jar --out=_13.xml --docpath=$(ANDROID_SDK_PATH)/docs/reference --annotations=annotations/13.xml || exit 1 mono-xmltool --prettyprint _13.xml > _13_.xml || exit 1 @@ -47,13 +57,15 @@ test-13: 10.xml annotations/13.xml xmlstarlet c14n 10_.xml > 10.xml || exit 1 rm 10_.xml +# annotations + annotations/8.xml: scraper.exe tmpout/8-deprecated-members.xml mkdir -p annotations - mono --debug scraper.exe tmpout/8-deprecated-members.xml > annotations/8.xml + mono --debug scraper.exe tmpout/8-deprecated-members.xml docs-2.2_r01-linux/reference/ > annotations/8.xml tmpout/8-deprecated-members.xml : scraper-main.sh scraper-collector.sh mkdir -p tmpout - bash scraper-main.sh > tmpout/8-deprecated-members.xml + bash scraper-main.sh docs-2.2_r01-linux/reference > tmpout/8-deprecated-members.xml || exit 1 annotations/13.xml: scraper.exe tmpout/13-deprecated-members.xml mkdir -p annotations @@ -61,6 +73,4 @@ annotations/13.xml: scraper.exe tmpout/13-deprecated-members.xml tmpout/13-deprecated-members.xml : scraper-main.sh scraper-collector.sh mkdir -p tmpout - bash scraper-main.sh > tmpout/13-deprecated-members.xml -scraper.exe : scraper.cs - mcs -debug scraper.cs + bash scraper-main.sh > tmpout/13-deprecated-members.xml || exit 1 diff --git a/README b/README index 4e3b2aa..395f829 100644 --- a/README +++ b/README @@ -13,8 +13,11 @@ and associated documentation. The information is stored in XML format. - jar2xml : the main converter to generate the primary target xmls. - scraper.exe : annotation doc scraper to generate annotations/*.xml. + scraper-*.sh : shells used for scraper input. They are piped. -* Easy comparison +* Easy comparison + +well, now it's getting much more complicated than it used to be like below... $ java -jar jar2xml.jar --jar=/home/atsushi/android-sdk-linux_x86/platforms/android-8/android.jar --out=_8.xml --docpath=/home/atsushi/android-sdk-linux_x86/docs/reference $ mono-xmltool --prettyprint 8.xml > 8_.xml @@ -23,7 +26,8 @@ and associated documentation. The information is stored in XML format. $ xmlstarlet c14n _8_.xml > _8.xml $ diff -u 8.xml _8.xml -It is also implemented as "make test-8" as well as "make test-13". +It is now implemented as "make test-8" as well as "make test-13" with more +detailed arguments. You should rather check Makefile to see what it does. * Difference from AOSP API XML @@ -57,9 +61,6 @@ It is also implemented as "make test-8" as well as "make test-13". * TODO -- Move methods in non-public base type to the public derived type e.g. - AbstractStringBuilder.append() should move to StringBuffer and StringBuilder. - - Do not skip certain overriden methods. This in JavaClass.java gives equivalent output to AOSP xml, but it's not good: @@ -88,7 +89,21 @@ It is also implemented as "make test-8" as well as "make test-13". in android.jar. That includes, types themselves. java.lang.annotation types suffer from this. -- java.lang.AbstractStringBuilder exposes another issue that + I find it almost desparate and the only solution I can think of is + to build AOSP android.jar or some equivalence that does *not* strip + those annotations off, *for each platform version*. + That sounds so time-eater. + + The AOSP API document is generated as out/target/common/obj/PACKAGING/android_jar_intermediaries/public_api.xml (at that state there still must be annotations, or this xml should not contain annotation information at all). + Also, + out/target/common/obj/JAVA_LIBRARIES/framework_intermediaries and + out/target/common/obj/JAVA_LIBRARIES/core_intermediaries + contains debug jars that still seem to hold annotations. + + This is from the latest AOSP build (as of Aug.17, 2011). I'm not sure + if this applies to older versions. + +- java.lang.AbstractStringBuilder exposes an issue that interface members on non-public types are returned by class#getDeclaredMethods() while they are *not* declared. @@ -98,18 +113,12 @@ It is also implemented as "make test-8" as well as "make test-13". excluded, but this brings another check to *not* exclude methods when the corresponding base methods are excluded). -- Some classes exists in the output xml but not in the dll: - - android.database.ContentObservable (derived from Observable) - - android.net.NetworkInfo.State (enum type) - - Some fields seem to be incorrectly marked as constant. For example, android.os.Build.TIME is not a constant, but since the field.getLong() returns 0, it is set as 0 (and it does not return null for get()). -- Doc scraper still misses some parameter names - e.g. android.text.method.ScrollingMovementMethod.onKeyUp() - This matters when api-fixup is involved (it matches nodes by parameter - names and then it dismisses the target nodes!). + java.io.File.pathSeparatorChar has the same problem, but it is worse; + it returns non-zero value so we cannot depend on the value. * Notes on Annotations @@ -147,3 +156,20 @@ makefiles before confirming that it's all good). [*2] http://asm.ow2.org/asm33/javadoc/user/org/objectweb/asm/ClassReader.html +* Notes on per-version doc scraping + +We used to scrape only reference for the latest API which was problematic. +Now we have version-aware scraper and resolved the issue described below. + + + Doc scraper still misses some parameter names + e.g. android.text.method.ScrollingMovementMethod.onKeyUp() + This matters when api-fixup is involved (it matches nodes by parameter + names and then it dismisses the target nodes!). + + Actually this is not going to be very good; there are some methods + that are removed in the latest API (the example method above is exactly + the case). We'll have to sort out some alternative approach. + A possible approach is to checkout API docs for *every* version (from + AOSP). + diff --git a/doc-locations.txt b/doc-locations.txt new file mode 100644 index 0000000..4caf018 --- /dev/null +++ b/doc-locations.txt @@ -0,0 +1,7 @@ +13: http://dl-ssl.google.com/android/repository/docs-3.2_r01-linux.zip +12: http://dl-ssl.google.com/android/repository/docs-3.1_r01-linux.zip +11: http://dl-ssl.google.com/android/repository/docs-3.0_r01-linux.zip +10: ? +09: http://dl-ssl.google.com/android/repository/docs-2.3_r01-linux.zip +08: http://dl-ssl.google.com/android/repository/docs-2.2_r01-linux.zip +07: http://dl-ssl.google.com/android/repository/docs-2.1_r01-linux.zip diff --git a/scraper-collector.sh b/scraper-collector.sh index 25bc522..3b3064b 100644 --- a/scraper-collector.sh +++ b/scraper-collector.sh @@ -1,8 +1,16 @@ +#!/bin/bash -echo "" -for f in `find $ANDROID_SDK_PATH/docs/reference -name *.html` +if [ "$1" ]; then + export docpath="$1" +else + export docpath="$ANDROID_SDK_PATH/docs/reference" +fi + +echo "" +for f in `find $docpath -name *.html` do - echo "" + export len=`expr length "$docpath" + 1` + echo "" echo "" xmllint --html --xmlout $f 2>/dev/null | xmlstarlet sel -t -c "//p[@class='caution']/../../h4[@class='jd-details-title']/text()" echo "" diff --git a/scraper-main.sh b/scraper-main.sh index fe2192e..5501131 100644 --- a/scraper-main.sh +++ b/scraper-main.sh @@ -1,3 +1,4 @@ +#!/bin/bash echo "" -bash scraper-collector.sh | xmlstarlet sel -t -c "//file[normalize-space(fields/text()) or normalize-space(methods/text())]" +bash scraper-collector.sh "$1" | xmlstarlet sel -t -c "//file[normalize-space(fields/text()) or normalize-space(methods/text())]" echo "" diff --git a/scraper.cs b/scraper.cs index bde40e8..870126e 100644 --- a/scraper.cs +++ b/scraper.cs @@ -10,7 +10,6 @@ public class Scraper { public static int Main (string [] args) { - var extraPath = Path.Combine (Environment.GetEnvironmentVariable ("ANDROID_SDK_PATH"), "docs/reference") + "/"; var doc = new XmlDocument (); doc.Load (args [0]); Console.WriteLine (""); @@ -37,7 +36,7 @@ public static int Main (string [] args) } if (output.Length == 0) continue; - string name = file.GetAttribute ("name").Substring (extraPath.Length); + string name = file.GetAttribute ("name"); name = name.Substring (0, name.Length - 5); // .html Console.WriteLine ("\n{1}", name, output); }