PovCrawler.java Go to the documentation of this file. package org.net2map.pov.documentsManager; //import websphinx.*; import java.beans.XMLEncoder; import java.beans.XMLDecoder; import java.io.*; import java.net.URL; import java.net.MalformedURLException; /*import net.matuschek.spider.WebRobot; import net.matuschek.http.URLLogger; import net.matuschek.http.HttpDocToFile;*/ import websphinx.*; public class PovCrawler { static org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger( PovCrawler.class.getName() ); static PovUrlsSet usUrls, usNews; //PovUrlsSet htUrlsToCrawl; int nCurrentLevel = 0; PovIndexer pIndexer; String sMirrorDirectory = ""; public PovCrawler( PovIndexer indexer, String mirrordirectory ) { logger.setLevel( ( org.apache.log4j.Level )org.apache.log4j.Priority.WARN ); pIndexer = indexer; sMirrorDirectory = mirrordirectory; usUrls = new PovUrlsSet(); } public void crawl( PovUrl root, boolean create, boolean filecontinue ) { boolean bCreate = create; boolean bInit = ( usUrls.size() == 0 ); PovUrl url = root; //chargement de la liste du fichier if ( filecontinue ) { if ( usUrls.size() == 0 ) { try { usUrls = (PovUrlsSet)load( "urls.dat" ); url = usUrls.nextUrl(); } catch ( Exception e ) { logger.warn( e.getMessage() ); } } } //if ( url.getUrl() != "" ) if ( url != null ) { logger.warn( "Launch crawl with url " + url.toString() ); //création de la liste de nouvelles url sphinx( url ); //logger.warn( usNews.size() + " nouvelles url à traiter"); //logger.warn( usUrls.size() + " url courantes au total"); //merge des nouvelles url dans la liste de base PovUrlsSet usMerged = usUrls.merge( usNews ); if ( bInit ) { if ( usMerged.get( url.getUrl() ) == null ) { usMerged.put( url.getUrl(), url ); } else { ((PovUrl)usMerged.get( url.getUrl() ) ).setDepth( url.getDepth() ); } } logger.warn( usMerged.size() + " url à indexer"); logger.warn( usUrls.size() + " url courantes au total"); //logger.warn( usUrls.toString() ); //Indexation de la liste de nouvelles url try { //pIndexer.indexDocs( usNews, bCreate ); //logger.warn( usMerged.toString() ); if ( usMerged.size() > 0 ) { pIndexer.indexDocs( usMerged, bCreate ); } if ( create = true ) bCreate = false; } catch ( Exception e ) { logger.warn( e.getMessage() ); } //Mise à jour url root (a indexer => indexée) ( (PovUrl)usUrls.get( url.getUrl() ) ).setState( PovUrl.STATE_VISITED ); //Sauvegarde du la nouvelle liste d'url try { save( usUrls, "urls.dat" ); } catch ( Exception e ) { logger.warn( e.getMessage() ); } //Lancement de la prochaine indexation //PovUrl next = usUrls.nextUrl(); usMerged = null; System.gc(); /*if ( Runtime.getRuntime().freeMemory() > 1000000 ) crawl( usUrls.nextUrl(), bCreate, true );*/ org.net2map.pov.indexer.IndexerTask.getInstance().setCurrentUrl(usUrls.nextUrl()); } else { logger.warn( "Url is empty" ); } } public void addCachedUrl( PovUrl url ) { //logger.warn( " add cached " + url.toString() ); usNews.put( url.getUrl(), url ); } private void sphinx( PovUrl url ) { Crawler theCrawler = new Crawler(); DownloadParameters theDownloadParameters = new DownloadParameters(); theDownloadParameters.changeDownloadTimeout(5); theDownloadParameters.changeMaxThreads(10); theDownloadParameters.changeObeyRobotExclusion(true); try { theCrawler.setRoot ( new Link( new URL( url.getUrl() ) ) ); } catch ( Exception e ) { logger.warn( e.getMessage() ); } theCrawler.setDownloadParameters( theDownloadParameters ); theCrawler.setDepthFirst( false ); theCrawler.setMaxDepth( 1 ); //theCrawler.setAction( new SphinxMirrorAction( mirrordir, true, this, create ) ); //PovUrlsSet usUrls = new PovUrlsSet(); //SphinxMirrorAction theSMA = new SphinxMirrorAction( sMirrorDirectory, this, url ); theCrawler.setAction( new SphinxMirrorAction( sMirrorDirectory, this, url ) ); //SphinxEventLog theEventLog = new SphinxEventLog(); //theCrawler.addCrawlListener( theEventLog ); //theCrawler.addLinkListener( theEventLog ); usNews = new PovUrlsSet(); theCrawler.run(); while ( theCrawler.getState() != CrawlEvent.STOPPED ) { } theCrawler.clear(); theCrawler = null; logger.warn( "Finished url " + url.getUrl() ); url.setState( url.STATE_VISITED ); //System.out.println( usUrls ); //System.out.println( theSMA.getUrls().toString() ); //pIndexer.stop(); //return usUrls; /*WebRobot robby = new WebRobot(); try { robby.setStartURL( new URL( url.getUrl() ) ); robby.setMaxDepth( 1 ); robby.setSleepTime( 0 ); robby.setAllowWholeHost( true ); robby.setWalkToOtherHosts( true ); robby.setAllowCaching( true ); FileWriter logfile = new FileWriter( "urls.txt" ); URLLogger log = new URLLogger( logfile ); //HttpDocToFile theHttpdtf = new HttpDocToFile( "/home/alain/mirror/" ); //robby.setDocManager(theHttpdtf); robby.setDocManager( log ); robby.run(); logfile.close(); } catch ( Exception e ) { System.out.println( e ); }*/ } public void save( PovUrlsSet us, String filename ) throws Exception { /*try { FileWriter file = new FileWriter( filename ); for ( Enumeration e = elements() ; e.hasMoreElements() ;) { file.write( (PovUrl)e.nextElement() + "\n" ); } file.close(); } catch ( Exception e ) { logger.warn( e.getMessage() ); }*/ try { ObjectOutputStream out = new ObjectOutputStream( new FileOutputStream( filename ) ); out.writeObject( us ); out.close(); logger.warn( "File " + filename + " saved" ); } catch ( Exception e ) { logger.warn( e.getMessage() ); throw e; } } public PovUrlsSet load( String filename ) throws Exception { PovUrlsSet result; try { ObjectInputStream in = new ObjectInputStream( new FileInputStream( filename ) ); result = (PovUrlsSet)in.readObject(); in.close(); logger.warn( "File " + filename + " load" ); } catch ( Exception e ) { logger.warn( e.getMessage() ); throw e; } return result; } public PovUrl nextUrl() { if ( usUrls != null ) { return usUrls.nextUrl(); } return null; } }
Accueil | Téléchargement | Manuel | Doc. technique | Sources CVS | Faq | Nous contacter ©2003 - All Rights Reserved