SphinxMirrorAction.java Go to the documentation of this file. /* * WebSphinx web-crawling toolkit * * Copyright (c) 1998-2002 Carnegie Mellon University. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ package org.net2map.pov.documentsManager; import websphinx.workbench.*; import websphinx.*; import java.io.File; import java.io.IOException; import java.net.URL; import java.net.MalformedURLException; public class SphinxMirrorAction extends MirrorAction { transient File dir; transient Mirror mirror; static org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger( SphinxMirrorAction.class.getName() ); //private static PovIndexer pIndexer; //private static boolean bCreate = true; //Runtime theRuntime = Runtime.getRuntime(); /* static int garbageCollectorCycle = 30; static int garbageCollectorCycleCounter=0; static long count = 0; */ static PovUrl uRoot; //static PovUrlsSet usUrls; static PovCrawler pCrawler; public SphinxMirrorAction( String directory, PovCrawler crawler, PovUrl rooturl ) { super( directory, false ); logger.setLevel( ( org.apache.log4j.Level )org.apache.log4j.Priority.WARN ); //pIndexer = indexer; //bCreate = create; uRoot = rooturl; //usUrls = urls; //usUrls = new PovUrlsSet(); pCrawler = crawler; } public boolean equals ( Object object ) { if ( !( object instanceof MirrorAction ) ) return false; MirrorAction a = ( MirrorAction )object; return same ( a.getDirectory(), getDirectory() ) && a.getUseBrowser() == getUseBrowser(); } private boolean same ( String s1, String s2 ) { if ( s1 == null || s2 == null ) return s1 == s2; else return s1.equals ( s2 ); } private void showit() { Browser browser = Context.getBrowser(); if ( browser != null ) try { browser.show ( Link.FileToURL( dir ) ); } catch ( MalformedURLException e) { } } /*public PovUrlsSet getUrls() { return usUrls; }*/ public synchronized void visit (Page page) { try { mirror.writePage( page ); logger.warn( "Lookup " + mirror.lookup( new URL( "file:" ), page.getURL() ) ); //PovMappedFile file = new PovMappedFile( mirror.lookup( new URL( "file:" ), page.getURL() ), page.getURL() ); PovUrl url = new PovUrl( page.getURL().toString(), mirror.lookup( new URL( "file:" ), page.getURL() ), uRoot.getDepth() - 1 ); //pIndexer.indexDocs( file, bCreate ); //file = null; if ( url.getUrl() != uRoot.getUrl() ) { pCrawler.addCachedUrl( url ); } //usUrls.put( url.getUrl(), url ); page.discardContent(); /*count++; if( garbageCollectorCycleCounter >= garbageCollectorCycle ) { logger.warn("Garbage collection!"); long memoryBeforeGarbageCollection = theRuntime.freeMemory(); theRuntime.gc(); long memoryAfterGarbageCollection = theRuntime.freeMemory(); logger.warn( "Raised free memory from " + memoryBeforeGarbageCollection + " to " + memoryAfterGarbageCollection + " freeing " + (memoryAfterGarbageCollection - memoryBeforeGarbageCollection) + " bytes"); logger.warn( "Documents count = " + count ); garbageCollectorCycleCounter=0; } else { garbageCollectorCycleCounter++; }*/ } catch ( IOException e ) { throw new RuntimeException ( e.toString() ); } catch ( Exception f ) { logger.warn( f.getMessage() ); } } public void connected ( Crawler crawler ) { crawler.addCrawlListener( this ); } public void disconnected( Crawler crawler ) { crawler.removeCrawlListener( this ); } public void started( CrawlEvent event ){ if ( mirror == null ) { try { dir = ( getDirectory() != null ) ? new File ( getDirectory() ) : Access.getAccess().makeTemporaryFile( "mirror", "" ); mirror = new Mirror( dir.toString() ); Crawler crawler = event.getCrawler (); Link[] roots = crawler.getRoots (); for ( int i=0; i<roots.length; ++i ) mirror.mapDir( roots[i].getURL(), dir.toString() ); } catch ( IOException e ) { logger.warn( e ); // FIX: use GUI when available } } //pIndexer.init( bCreate ); logger.warn( "Sphinx mirror manager started..." ); } public void stopped( CrawlEvent event ) { try { if ( mirror != null ) { mirror.close(); mirror = null; if ( getUseBrowser() ) showit(); } } catch ( IOException e ) { System.err.println( e ); // FIX: use GUI when available } //pIndexer.stop(); logger.warn( "Sphinx mirror manager stoped..." ); } public void cleared( CrawlEvent event ) { try { if ( mirror != null ) { mirror.close(); mirror = null; if ( getUseBrowser() ) showit(); } } catch ( IOException e ) { logger.warn( e ); // FIX: use GUI when available } } public void timedOut( CrawlEvent event ) { try { if ( mirror != null ) { mirror.close(); mirror = null; if ( getUseBrowser() ) showit(); } } catch ( IOException e ) { logger.warn( e ); // FIX: use GUI when available } } public void paused( CrawlEvent event ) { try { if ( mirror != null ) { mirror.rewrite(); if ( getUseBrowser() ) showit(); } } catch ( IOException e ) { logger.warn( e ); // FIX: use GUI when available } } }
Accueil | Téléchargement | Manuel | Doc. technique | Sources CVS | Faq | Nous contacter ©2003 - All Rights Reserved