WalkSection.java

package html_parser;

import java.sql.Connection;

import java.util.ArrayList;
import html_parser.page.PageWalker;
import html_parser.page.PageWalkerAware;
import html_parser.record.Record;
import html_parser.record_processor.RecordProcessor;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import database.Connector;

import sites.autozvuk_com_ua.AvtozvukPageWalker;
import sites.autozvuk_com_ua.AvtozvukPageWalkerAware;
import sites.autozvuk_com_ua.AvtozvukSaver;
import sites.autozvuk_com_ua.AvtozvukSectionXml;

/** êëàññ, êîòîðûé íà îñíîâàíèè XML äîêóìåíòà ïîçâîëÿåò "ïðîãóëÿòüñÿ" ïî âñåì çàïèñÿì â óêàçàííîé ñåêöèè, îáÿçàòåëüíûå àòòðèáóòû êàæäîãî óçëà - "href","caption" */
public class WalkSection {
	/** ñëóøàòåëè, êîòîðûå ïðèíèìàþò ðåøåíèå ïî ïîâîäó ñêàíèðîâàíèÿ î÷åðåäíîé ñåêöèè */
	private ArrayList<ScanSectionFilter> allowListeners=new ArrayList<ScanSectionFilter>();
	
	/** äîáàâèòü ñëóøàòåëÿ, êîòîðûé ïðèíèìàåò ðåøåíèå ïî ïîâîäó ïàðñèíãà î÷åðåäíîé ñòðàíèöû */
	public void addAllowListener(ScanSectionFilter listener){
		this.allowListeners.add(listener);
	}
	/** óäàëèòü ñëóøàòåëÿ, êîòîðûé ïðèíèìàåò ðåøåíèå íà ïàðñèíã î÷åðåäíîé ñåêöèè */
	public void removeAllowListener(ScanSectionFilter listener){
		this.allowListeners.remove(listener);
	}

	/** ðåøåíèå, íóæíî ëè ïðîäîëæàòü ïàðñèòü óêàçàííûé ýëåìåíò äåðåâà */
	private boolean isProcessAllow(Element element){
		boolean returnValue=true;
		for(int counter=0;counter<this.allowListeners.size();counter++){
			if(this.allowListeners.get(counter).isFilter(element)==false){
				returnValue=false;
				break;
			}
		}
		return returnValue;
	}
	
	/** ïðîéòèñü ïî âñåì çàïèñÿì, êîòîðûå óêàçàíû â XML äîêóìåíòå 
	 * @param xmlDocument - äîêóìåíò, êîòîðûé ñîäåðæèò âñå ýëåìåíòû {@literal"<leaf caption="" href="">"} 
	 * @param urlPrefix - ïðåôèêñ, êîòîðûé íóæíî äîáàâëÿòü äëÿ ïîëó÷åíèÿ àäðåñà ñåêöèè (href)
	 * @param processParent - íóæíî ëè "ïàðñèòü" óçëû, êîòîðûå ÿâëÿþòñÿ ðîäèòåëüñêèìè, ò.å. ñîäåðæàò äðóãèå óçëû (true) èëè æå íóæíî ïàðñèòü òîëüêî "ëèñòû" áåç ðîäèòåëüñêèõ êîìïîíåíòîâ (false)
	 * @param processLeaf - íóæíî ëè "ïàðñèòü" êîíå÷íûå óçëû äåðåâà - ëèñòüÿ
	 * @param pageWalker - ïàðñåð ñòðàíèöû, êîòîðûé "âûíèìàåò" èç ñòðàíèöû íåîáõîäèìûå äàííûå
	 * @param pageWalkerAware - îáúåêò, êîòîðûé ïîëó÷àåò ññûëêè íà ñëåäóþùèå ñòðàíèöû
	 * @param delay - çàäåðæêè, êîòîðûå íåîáõîäèìî äåëàòü äëÿ íîðìàëüíîãî(ýìóëÿöèÿ ïîëüçîâàòåëÿ) ÷òåíèÿ äàííûõ èç ñåòè
	 * @param postProcessor - îáðàáîòêà ïîëó÷åííûõ çàïèñåé ïåðåä ñîõðàíåíèÿ  
	 */
	public void walk(Document xmlDocument, 
					 String urlPrefix, 
					 boolean processParent,
					 boolean processLeaf, 
					 PageWalker pageWalker,
					 PageWalkerAware pageWalkerAware,
					 Delay delay,
					 RecordProcessor preProcessor,
					 Saver saver){
		if(saver!=null)saver.begin();
		// ïåðåáðàòü âñå ëèñòû, êîòîðûå ïåðåäàíû â xmlDocument
		NodeList nodes=xmlDocument.getChildNodes().item(0).getChildNodes();
		for(int counter=0;counter<nodes.getLength();counter++){
			//System.out.println(this.getStringFromXmlDocument(nodes.item(counter)));
			walkNode(nodes.item(counter),
					 urlPrefix, 
					 processParent,
					 processLeaf, 
					 pageWalker, 
					 pageWalkerAware, 
					 delay,
					 preProcessor,
					 saver);
		}
		if(saver!=null)saver.finish();
	}
	/*
	private String getStringFromXmlDocument(Node document){
		Writer out=null;
		try{
			javax.xml.transform.TransformerFactory transformer_factory = javax.xml.transform.TransformerFactory.newInstance();  
			javax.xml.transform.Transformer transformer = transformer_factory.newTransformer();  
			javax.xml.transform.dom.DOMSource dom_source = new javax.xml.transform.dom.DOMSource(document); // Pass in your document object here  
			out=new StringWriter();
			//string_writer = new Packages.java.io.StringWriter();  
			javax.xml.transform.stream.StreamResult stream_result = new javax.xml.transform.stream.StreamResult(out);  
			transformer.transform(dom_source, stream_result);  
		}catch(Exception ex){
			System.err.println("getStringFromXmlDocument:"+ex.getMessage());
		}
		return (out==null)?"":out.toString();
	}*/

	
	/** ïðîéòèñü ïî âñåì âåòêàì, ïî âñåì çàïèñÿì  */
	private void walkNode(Node node,
						  String urlPrefix,
						  boolean processParent,
						  boolean processLeaf,
						  PageWalker pageWalker, 
						  PageWalkerAware pageWalkerAware, 
						  Delay delay,
						  RecordProcessor preProcessor,
						  Saver saver){
		if(node.hasChildNodes()){
			//System.out.println("has child");
			// åñòü äî÷åðíèå ýëåìåíòû
			NodeList list=node.getChildNodes();
			for(int counter=0;counter<list.getLength();counter++){
				this.walkNode(list.item(counter),
							  urlPrefix,
							  processParent,
							  processLeaf,
							  pageWalker,
							  pageWalkerAware, 
							  delay,
							  preProcessor,
							  saver);
			}
			if(processParent){
				this.processNode(node,urlPrefix, pageWalker,pageWalkerAware,delay,preProcessor, saver);
			};
		}else{
			//System.out.println("this is leaf");
			// äàííûé ýëåìåíò ÿâëÿåòñÿ ëèñòîì, ò.å. íå èìååò äî÷åðíèõ ýëåìåíòîâ
			// node
			if(processLeaf){
				this.processNode(node,urlPrefix,pageWalker,pageWalkerAware, delay,preProcessor, saver);
			}
		}
	}
	
	
	/** îáðàáîòàòü äàííûé ýëåìåíò èç ýëåìåíòà äåðåâà, êîòîðîå ñîäåðæèò âñå ñåêöèè
	 * @param node - óçåë, êîòîðûé áóäåò
	 * @param urlPrefix - ïðåôèêñ URL ê êîòîðîìó íóæíî äîáàâëÿòü äàííûå 
	 * @param pageWalker - îáúåêò äëÿ ÷òåíèÿ ñòðàíèö
	 * @param pageWalkerAware - îáúåêò äëÿ ïåðåìåùåíèÿ ìåæäó ñòðàíèöàìè 
	 * @param delay - çàäåðæêà äëÿ ÷òåíèÿ
	 * @param saver - îáúåêò, êîòîðûé ïðåäíàçíà÷åí äëÿ ñîõðàíåíèÿ Records 
	 */
	private void processNode(Node node,
							 String urlPrefix,
							 PageWalker pageWalker,
							 PageWalkerAware pageWalkerAware, 
							 Delay delay,
							 RecordProcessor preProcessor,
							 Saver saver){
		if(node instanceof Element){
			Element element=(Element)node;
			String href=element.getAttribute("href");
			String caption=element.getAttribute("caption");
			System.out.println(">>> Section:"+caption+"   Href: "+href);
			if(this.isProcessAllow(element)){
				pageWalker.updatePageWalkerAware(pageWalkerAware);
				pageWalkerAware.reset(urlPrefix+href);
				// ïîëó÷èòü êîä ïåðâîé ñòðàíèöû
				while(pageWalker.hasMoreElements()){
					// ïîëó÷åííûé ýëåìåíò ðàñïàðñèòü
					ArrayList<Record> block=pageWalker.nextElement();
					if(preProcessor!=null){
						preProcessor.beforeSave(block);
					}
					// ïðîáåæàòüñÿ ïî Saver-ó
					if(saver!=null){
						for(int counter=0;counter<block.size();counter++){
							//System.out.println(block.get(counter).toString());
							if(saver.save(caption, block.get(counter))==false){
								System.err.println("WalkSection save Error:");
							}
						}
					}
					if(preProcessor!=null){
						preProcessor.afterSave(block);
					}
					System.out.println(">>> NextPage ( after "+pageWalker.getLastUrl()+") ");
					// óñíóòü, åñëè íóæíî, äëÿ âîçìîæíîãî ñîêðûòèÿ ïàðñèíãà - âðåìÿ îáðàùåíèÿ ê îäíîìó è òîìó æå ñåðâåðó
					try{
						Thread.sleep(delay.getDelayReadPage());
					}catch(Exception ex){};
				}
				try{
					Thread.sleep(delay.getDelayReadSection());
				}catch(Exception ex){};
			}else{
				// îòìåíà ïàðñèíãà î÷åðåäíîãî ýëåìåíòà - çàïðåò ñëóøàòåëåé  
			}
		}else{
			// ïðîìåæóòî÷íûé ýëåìåíò
		}
	}
	
	
	public static void main(String[] args){
/*		System.out.println("begin");
		// îáúåêò, êîòîðûé ÷èòàåò ñòðàíèöû 
		PageWalker pageWalker=new AvtozvukPageWalker();
		// îáúåêò, êîòîðûé äà¸ò íîìåð ñëåäóþùåé ñòðàíèöû 
		PageWalkerAware pageWalkerAware=new AvtozvukPageWalkerAware();
		// îáúåêò, êîòîðûé âûäà¸ò XML íà îñíîâàíèè ðóáðèêàòîðà/äåðåâà ýëåìåíòîâ  
		AvtozvukSectionXml avtozvuk=new AvtozvukSectionXml();
		Document treeXml=avtozvuk.getXmlDocument();
		// îáúåêò-çàäåðæêà äëÿ ýìóëÿöèè ÷òåíèÿ äàííûõ ÷åëîâåêîì, à íå ïàðñåðîì 
		Delay delay=new Delay(5,2);
		// îáúåêò äëÿ ñîõðàíåíèÿ äàííûõ 
		Connection connection=null;
		try{
			Connector connector=new Connector("V:/eclipse_workspace/ShopList_HtmlParser/shop_list.gdb");
			connection=connector.getConnector().getConnection();
		}catch(Exception ex){
			System.out.println("WalkSection#main getConnection Exception: "+ex.getMessage());
		}
		AvtozvukSaver saver=new AvtozvukSaver(connection);
		//saver.resetAllRecord();
		
		WalkSection walk=new WalkSection();
		// óñòàíîâèòü ôèëüòð íà ïàðñèíã 
		walk.addAllowListener(new ScanSectionFilter(){
			private boolean parse=false;
			@Override
			public boolean isFilter(Element element) {
				if(element.getAttribute("caption").indexOf("Êðîññîâåðû")>=0){
					parse=true;
				}
				return this.parse;
			}
		});
		walk.walk(treeXml, 
				  "http://avtozvuk.ua", 
				  true, 
				  true,
				  pageWalker, 
				  pageWalkerAware, 
				  delay,
				  null,
				  saver
				  );
		try{
			connection.close();
		}catch(Exception ex){};
		System.out.println("end");
*/		
	}
}