0
@@ -14,6 +14,7 @@
0
 VALUE rb_cFastXmlAttrList;
0
 VALUE rb_sValidateDtd;
0
 VALUE rb_sForgivingParse;
0
+VALUE rb_sHtmlParse;
0
 ID s_readlines;
0
 ID s_to_s;
0
 
0
0
@@ -37,9 +38,12 @@
0
     rb_cFastXmlNodeList = rb_define_class_under( rb_mFastXml, "NodeList", rb_cObject );
0
   rb_cFastXmlAttrList = rb_define_class_under( rb_mFastXml, "AttrList", rb_cObject );
0
 
0
-    /* Doc */
0
+    /* setting symbols */
0
     rb_sValidateDtd = ID2SYM( rb_intern("validate") );
0
     rb_sForgivingParse = ID2SYM( rb_intern("forgiving") );
0
+    rb_sHtmlParse = ID2SYM( rb_intern("html") );
0
+    
0
+    /* Doc */
0
     rb_define_method( rb_cFastXmlDoc, "initialize", fastxml_doc_initialize, -1 );
0
     rb_define_method( rb_cFastXmlDoc, "search", fastxml_doc_search, 1 );
0
     rb_define_method( rb_cFastXmlDoc, "to_s", fastxml_doc_to_s, 0 );

0
@@ -18,12 +18,17 @@
0
 #include <libxml/xpathInternals.h>
0
 #include <libxslt/xslt.h>
0
 #include <libxslt/xsltInternals.h>
0
+#include <libxml/HTMLparser.h> 
0
 
0
 #ifndef XML_WITH_TREE
0
   #define XML_WITH_TREE 2
0
   #define XML_WITH_XPATH 16
0
 #endif
0
 
0
+#ifndef HTML_PARSE_RECOVER
0
+    #define HTML_PARSE_RECOVER 1
0
+#endif
0
+
0
 typedef struct {
0
     xmlDocPtr doc;
0
     xmlNodePtr node;
0
@@ -41,6 +46,7 @@
0
 
0
 RUBY_EXTERN VALUE rb_sValidateDtd;
0
 RUBY_EXTERN VALUE rb_sForgivingParse;
0
+RUBY_EXTERN VALUE rb_sHtmlParse;
0
 
0
 RUBY_EXTERN ID s_readlines;
0
 RUBY_EXTERN ID s_to_s;

0
@@ -125,6 +125,7 @@
0
     VALUE data_s, dv, lines, xml_doc_str, opts, blk;
0
     fxml_data_t *data;
0
     int parser_opts = XML_PARSE_NOERROR | XML_PARSE_NOWARNING;
0
+    short html_parser = 0;
0
 
0
     if (rb_scan_args( argc, argv, "11&", &xml_doc_str, &opts, &blk ) == 0)
0
         return Qnil; // error state
0
@@ -144,6 +145,10 @@
0
             parser_opts = parser_opts | XML_PARSE_RECOVER;
0
       rb_iv_set( self, "@forgiving", Qtrue );
0
     }
0
+
0
+        if (rb_hash_aref(opts, rb_sHtmlParse) == Qtrue) {
0
+            html_parser = 1;
0
+        }
0
     }
0
 
0
   if (rb_respond_to( xml_doc_str, s_readlines )) {
0
0
@@ -158,8 +163,13 @@
0
     data = ALLOC(fxml_data_t);
0
     memset( data, (int)NULL, sizeof(fxml_data_t) );
0
 
0
-    data->doc = xmlReadMemory( RSTRING(data_s)->ptr, RSTRING(data_s)->len, 
0
+    if (html_parser == 0)
0
+        data->doc = xmlReadMemory( RSTRING(data_s)->ptr, RSTRING(data_s)->len, 
0
                                "noname.xml", NULL, parser_opts );
0
+    else
0
+        data->doc = htmlReadMemory( RSTRING(data_s)->ptr, RSTRING(data_s)->len,
0
+                                    "noname.html", NULL, HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING );
0
+
0
     // if we're mallformed we might want to use xmlRecoverMemcory(char*, int)
0
     if (data->doc == NULL) {
0
         rb_raise( rb_eRuntimeError, "Failed to parse document" );

0
@@ -83,6 +83,12 @@
0
 
0
 
0
 def FastXml(data=nil, opts = {}, &blk)
0
-  FastXml::Doc.new data
0
+  FastXml::Doc.new( data, opts, &blk )
0
+end
0
+
0
+def FastHtml(data=nil, opts = {}, &blk)
0
+  opts ||= {}
0
+  opts[:html] = true
0
+  FastXml::Doc.new( data, opts, &blk )
0
 end

0
@@ -1 +1,57 @@
0
+# $Id$
0
+$: << '../ext'
0
+$: << './ext'
0
+
0
+require 'fastxml'
0
+
0
+describe FastXml::Doc, " doing html parsing" do
0
+  before(:all) do
0
+    @data_raw = open( "./test_data/hasno_feed.html" )
0
+    @data_ary = @data_raw.readlines
0
+    @data_str = @data_ary.join('')
0
+  end
0
+
0
+  before do
0
+    @data_raw.rewind if @data_raw
0
+  end
0
+
0
+  after(:all) do
0
+    @data_raw.close if @data_raw
0
+  end
0
+
0
+
0
+  it 'should parse string input' do
0
+    @data_str.should_not be_nil
0
+    doc = FastXml::Doc.new( @data_str, {:html=>true} )
0
+    doc.should_not be_nil
0
+    doc.to_s.should_not be_nil
0
+  end
0
+
0
+  it 'should parse array input' do
0
+    @data_ary.should_not be_nil
0
+    doc = FastXml::Doc.new( @data_ary, {:html=>true} )
0
+    doc.should_not be_nil
0
+    doc.to_s.should_not be_nil
0
+  end
0
+  
0
+  it 'should be able to parse large files' do
0
+    raw_data = open( "./test_data/twitter_public.html" ).readlines.join('')
0
+    doc = FastHtml( raw_data )
0
+    doc.should_not be_nil
0
+    doc.to_s.should_not be_nil
0
+    doc.to_s.length.should >= 30000
0
+    doc.root.should_not be_nil
0
+    (doc/"").should_not be_nil
0
+    doc.root.children.should_not be_nil
0
+  end
0
+
0
+  it 'should be able to handle the cnn site' do
0
+    raw_data = open( "./test_data/cnn_main.html" ).readlines.join('')
0
+    doc = FastHtml( raw_data )
0
+    doc.should_not be_nil
0
+    doc.to_s.should_not be_nil
0
+    doc.to_s.length.should >= 10000
0
+  end
0
+
0
+end

0
@@ -1 +1,1349 @@
0
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN""http://www.w3.org/TR/html4/loose.dtd"><html lang="en"><head><title>CNN.com - Breaking News, U.S., World, Weather, Entertainment & Video News</title>      
0
+<meta http-equiv="refresh" content="1800;url=?refresh=1">
0
+<meta name="Description" content="CNN.com delivers the latest breaking news and information on the latest top stories, weather, business, entertainment, politics, and more. For in-depth coverage, CNN.com provides special reports, video, audio, photo galleries, and interactive guides.">
0
+<meta name="Keywords" content="CNN, CNN news, CNN.com, CNN TV, news, news online, breaking news, U.S. news, world news, weather, business, CNN Money, sports, politics, law, technology, entertainment, education, travel, health, special reports, autos, developing story, news video, CNN Intl">
0
+
0
+<link rel="alternate" type="application/rss+xml" title="CNN - Top Stories [RSS]" href="http://rss.cnn.com/rss/cnn_topstories.rss">
0
+<link rel="alternate" type="application/rss+xml" title="CNN - Recent Stories [RSS]" href="http://rss.cnn.com/rss/cnn_latest.rss">
0
+
0
+<meta http-equiv="content-type" content="text/html; charset=iso-8859-1">
0
+	<style type="text/css" media="screen">
0
+		@import url(http://i.cdn.turner.com/cnn/.element/css/2.0/common.css);
0
+		@import url(http://i.cdn.turner.com/cnn/.element/css/2.0/main.css);
0
+	</style>
0
+	<script>
0
+    if(window.top!=window.self)
0
+    {
0
+        if(document.cookie.indexOf('rfrshchck=')!=-1)
0
+        {
0
+            window.top.location=window.self.location;
0
+        }
0
+        document.cookie='rfrshchck=1';
0
+    }
0
+	</script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/scripts/prototype.js" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/scripts/scriptaculous.js?load=effects" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/csiManager.js" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/StorageManager.js" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/local.js" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/main.js" type="text/javascript"></script>
0
+	<script src="http://i.cdn.turner.com/cnn/.element/js/2.0/election/primaryHome.js" type="text/javascript"></script>
0
+
0
+<link rel="apple-touch-icon" href="http://i.cdn.turner.com/cnn/apple-touch-icon.png"/>
0
+
0
+<script type="text/javascript" language="javascript">
0
+pagetypeTS='homepage';
0
+
0
+var overrideVideoAd = '/cnn_adspaces/2.0/homepage/video.postroll_emb.ad';
0
+</script>
0
+<style type="text/css">
0
+<!--
0
+.cnnElexRFoot p
0
+{position:relative;}
0
+.cnnElexRFoot p span
0
+{position:absolute;right:0;}
0
+.cnnElexPrimary
0
+{position:relative;}
0
+.cnnElexPrimary span a
0
+{position:absolute;right:0;top:5px;font-size:10px;}
0
+* html .cnnElexPrimary span a
0
+{right:20px;}
0
+* html .cnniReportBox .cnniReportMoreMain a
0
+{width:302px;}
0
+-->
0
+</style>
0
+<script type="text/javascript">
0
+var closingTimes = {"NV":"Caucuses start from 12 p.m. ET","SC":"Last polls close 7:00 p.m. ET","FL":"Last polls close 8:00 p.m. ET"};
0
+var countyLinks ={"IN":{"url":"/ELECTION/2008/primaries/results/county/#INDEMMAPPRIMARY1 ","text":"County results map"},"NC":{"url":"/ELECTION/2008/primaries/results/county/#NCDEMMAPPRIMARY1 ","text":"County results map"},"PA":{"url":"/ELECTION/2008/primaries/results/county/#PADEMMAPprimary","text":"County results map"},"MS":{"url":"/ELECTION/2008/primaries/results/county/#MSDEMMAPprimary","text":"County results map"}}
0
+</script>
0
+
0
+<script language="JavaScript" type="text/javascript">var cnnCurrTime = new Date(1210043115508); var cnnCurrHour = 23; var cnnCurrMin = 05; var cnnCurrDay='Mon';</script>
0
+<script type="text/javascript">
0
+	var cnnDocDomain = '';
0
+	if(location.hostname.indexOf('cnn.com')>0) {cnnDocDomain='cnn.com';}
0
+	if(location.hostname.indexOf('turner.com')>0) {if(document.layers){cnnDocDomain='turner.com:'+location.port;}else{cnnDocDomain='turner.com';}}
0
+	if(cnnDocDomain) {document.domain = cnnDocDomain;}
0
+	// DO NOT PUT ANYTHING BENEATH THIS!
0
+</script>
0
+
0
+
0
+<script type="text/javascript" src="http://i.cdn.turner.com/cnn/.element/js/2.0/ad_head0.js"></script>
0
+<script type="text/javascript" src="http://i.cdn.turner.com/cnn/cnn_adspaces/cnn_adspaces.js"></script>
0
+</head><body id="cnnMainPage"><div id="cnnOpacity" onclick="cnnHideMoPo();"></div>      <div id="cnnHeader">
0
+	<div class="cnnHeaderContent">
0
+		<div class="cnnHeaderCeiling">
0
+			
0
+<a href="/"><img src="http://i.cdn.turner.com/cnn/.element/img/2.0/global/nav/header/header_cnn_com_logo.gif" width="148" height="36" border="0" alt=""></a>
0
+						<div class="cnnHeadColRight">
0
+				<div class="cnnGlobalHeaderSections" id="cnnHeadSrchTypeArea"><span class="cnnSearchLabel">Web</span> | <a href="javascript:cnnUpdateSrchType('news');">CNN News</a> | <a href="javascript:cnnUpdateSrchType('video');">CNN Videos</a></div>
0
+				<div class="cnnGlobalHeaderSearch">
0
+					<form action="http://search.cnn.com/cnn/search" method="get" onsubmit="return cnnSearch(this);">
0
+						<input type="hidden" name="cnnHeadSrchType" id="cnnHeadSrchType" value="web">
0
+						<input type="text" maxlength="40" class="cnnHeaderTxtField" id="cnnHeadSrchTxt">
0
+						<input type="image" src="http://i.cdn.turner.com/cnn/.element/img/2.0/global/nav/header/header_search_btn.gif" alt="Submit" class="cnnHeaderSearchBtn">
0
+						<img src="http://i.cdn.turner.com/cnn/.element/img/2.0/global/nav/header/header_google_logo.gif" class="cnnSrchDomLogo" width="47" height="22" border="0" alt="">
0
+					</form>
0
+				</div>
0
+			</div>
0
+		</div>
0
+	</div>
0
+		<div class="cnnNavStretch">
0
+		<div class="cnnHeaderNav">
0
+			<ul class="cnnNavigation">
0
+				<li class="cnnNavLeft"></li>
0
+				<li><a class="cnnCurPage" href="/">Home</a></li>
0
+				<li><a href="/WORLD/">World</a></li>
0
+				<li><a href="/US/">U.S.</a></li>
0
+				<li><a href="/POLITICS/">Politics</a></li>
0
+				<li><a href="/CRIME/">Crime</a></li>
0
+				<li><a href="/SHOWBIZ/">Entertainment</a></li>
0
+				<li><a href="/HEALTH/">Health</a></li>