head	1.1;
access;
symbols;
locks; strict;
comment	@# @;


1.1
date	2001.07.19.20.22.44;	author dlr;	state Exp;
branches;
next	;


desc
@@


1.1
log
@Ported Joist searching classes into Helm.
@
text
@/* ================================================================
 * Copyright (c) 2000 Collab.Net.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * 3. The end-user documentation included with the redistribution, if
 * any, must include the following acknowlegement: "This product includes
 * software developed by Collab.Net (http://www.Collab.Net/)."
 * Alternately, this acknowlegement may appear in the software itself, if
 * and wherever such third-party acknowlegements normally appear.
 *
 * 4. The hosted project names must not be used to endorse or promote
 * products derived from this software without prior written
 * permission. For written permission, please contact info@@collab.net.
 *
 * 5. Products derived from this software may not use the "Tigris" name
 * nor may "Tigris" appear in their names without prior written
 * permission of Collab.Net.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL COLLAB.NET OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of Collab.Net.
 */
 package org.tigris.helm.search;


/** Swish-e Search Engine query interface implementation.
* Does a java.lang.Runtime.exec( ) on swish-e.
* <p>
* This class can parse '# SWISH format 1.3'.  Will throw
* an exception if run against anything else.
* </p>
* <p>
* Swish-e 'ranking' can be from 1-1000.  Depends on a number of
* factors such as how many times your search word appears in the
* file, how many words are in the file, and if the word appears
* in a title or header tag (if it's an HTML file), etc.
* </p>
* @@author <a href="mailto:stack@@collab.net">St.Ack</a>
* @@version $Id: SearchSwishe.java,v 1.7 2000/11/06 00:08:21 stack Exp $
* @@see <a href="http://sunsite.berkeley.edu/SWISH-E/Manual">Swish-e Documentation</a>
*/
public class SearchSwishe
    implements SearchInterface
{
    /** Class version string
    */
    private static final String versionID
        = "$Id: SearchSwishe.java,v 1.7 2000/11/06 00:08:21 stack Exp $";


    /** Define key for pulling system property DEBUG.
    */
    private static final String DEBUG_KEY
        = "DEBUG";


    /** True if we're in debugging mode
    * Leave it protected rather than private since it's
    * read-only anyways.
    */
    protected boolean DEBUG
        = ( new Boolean( System.getProperty( DEBUG_KEY ) ) ).booleanValue( );


    /** Use this key to pull query from system properties.
    * Leave it protected rather than private since it's
    * read-only anyways.
    */
    protected static final String QUERY_PARAM_KEY
        = "query";


    /** The document Root for this collection.
     */
    private String docRoot = null;


    /** Search result format version string.
    * This class will only work against search results
    * that match this regular expression.
    * Leave it protected rather than private since it's
    * read-only anyways.
    */
    protected static final String FORMAT_VERSION_RE
        = "# SWISH format 1.3";


    /** The swish-e binary.
    */
    private java.io.File swisheBinary
        = null;


    /** The Swish-e Result Format Version string we work with.
    * Written as a regular expression in case we do more than
    * one explicit version.  Search across multilines.
    */
    private gnu.regexp.RE reFormatVersion
        = null;


    /** Regular expression to find header lines.
    * They have format: '# Key: Value'.  Search
    * across multi-lines so we jump over preamble
    * to get to first header line.
    */
    private gnu.regexp.RE reHeaderKeyValues
        = null;


    /** Search result error line Regular Expression.
    */
    private gnu.regexp.RE reErrorLine
        = null;


    /** The 'No Results' regular expression.
    */
    private gnu.regexp.RE reNoResults
        = null;



    /** Regular Expression to cut up a search result item.
    * Format of line is:
    * <pre>
    *   &lt;rank> &lt;location> &lt;title> &lt;size>
    *<pre>
    * <p>
    * I put a '"' into the title of a document,
    * the third field of the regular expression,
    * to see how swish-e returned it. Swish-e strips all
    * '"' characters from the title
    */
    private gnu.regexp.RE reSearchResultLine
        = null;


    /** Search result header key for index name
    */
    private final static String INDEX_NAME
        = "Name";


    /** Search result header key for index date
    */
    private final static String INDEX_MOD_DATE
        = "Indexed on";


    /** Search result header key for index stats
    */
    private final static String INDEX_STATS
        = "Counts";


    /** Search result header key for number of hits
    */
    private final static String NUMBER_OF_HITS
        = "Number of hits";


    /** Search result header key for index path.
    * May be an url or full path.
    */
    private final static String INDEX_PATH
        = "Pointer";


    /** Search result header key for description.
    */
    private final static String INDEX_DESCRIPTION
        = "Description";


    /** Search result header key for search engine search words.
    */
    private final static String SEARCH_WORDS
        = "Search words";
        
        
    /** Domain to use.
    * Needed to manufacture search result url 
    * according to receipe described at head of getSourceCastURL
    * in searchresultitem.
    * @@see SearchResultItem#getSourceCastURL
    */
    private String domain = null;
    
    
    /** Port to use.
    * Needed to manufacture search result url 
    * according to receipe described at head of getSourceCastURL
    * in searchresultitem.
    * @@see SearchResultItem#getSourceCastURL
    */    
    private String port = null;
    
    
    /** Protocol to use.
    * Needed to manufacture search result url 
    * according to receipe described at head of getSourceCastURL
    * in searchresultitem.
    * @@see SearchResultItem#getSourceCastURL
    */
    private String protocol = null;


    /** Constructor.
    *
    * @@param inBinary Path to the swish-e binary
    * @@param inDocRoot May be null.  Document root for search.
    * @@param inDomain   Pass me a domain.  I need this to make up a
    * search result url according to receipe described at head of 
    * getSourceCastURL in searchresultitem.  Cannot be null.  
    * Will throw exception if it is.
    * @@param inPort Port to use making up search result url 
    * according to receipe described at head of getSourceCastURL.
    * in searchresultitem.  Cannot be null.  
    * Will throw exception if it is.
    * @@param inProtocol Protocol to use making up search result url 
    * according to receipe described at head of getSourceCastURL.
    * in searchresultitem.  Cannot be null. 
    *
    * @@exception NullPointerException If passed in params
    * are null or empty.
    * @@exception java.io.FileNotFoundException If swish-e
    * binary does not exist.
    * @@exception gnu.regexp.REException Should never happen.
    * Let caller figure what to do w/ it.
    *
    * @@see SearchResultItem#getSourceCastURL
    */
    public SearchSwishe( String inBinary, 
                            String inDocRoot, 
                            String inDomain,
                            String inPort,
                            String inProtocol )
        throws java.io.FileNotFoundException,
                gnu.regexp.REException,
                NullPointerException
    {
        if( inBinary == null )
            throw new NullPointerException( "inBinary is null" );
        swisheBinary = new java.io.File( inBinary );
        if( !swisheBinary.exists( ) )
            throw new java.io.FileNotFoundException( "inBinary swish-e at location <"
                + inBinary
                + "> does not exist" );

        if( inDocRoot == null )
            throw new NullPointerException( "inDocRoot is null" );
        docRoot = inDocRoot.trim( );
        
        if( inDomain == null )
            throw new NullPointerException( "inDomain is null" );
        domain = inDomain.trim( );
        
        port = inPort;
        protocol = inProtocol;

        // Make all the regular expressions i'll need
        //
        reFormatVersion
            = new gnu.regexp.RE( FORMAT_VERSION_RE, gnu.regexp.RE.REG_MULTILINE );

        reHeaderKeyValues
            = new gnu.regexp.RE( "# ([^:\n\r]*): (.*)",
                gnu.regexp.RE.REG_MULTILINE );

        reErrorLine
            = new gnu.regexp.RE( "err: (.*)" ); // Error line. Won't work if i put a '^' at start, even w/ REG_ANCHORINDEX

        reNoResults
            = new gnu.regexp.RE( "no results" );

        reSearchResultLine
            = new gnu.regexp.RE( "(\\d*) ([^\"]*) \\\"(.*)\\\" (\\d*)" );
    }



   /** Run a search w/ passed in Query.
    *
    * @@param inQuery Query object to run search with.
    * @@return A search result object.
    * @@exception java.io.IOException Happens when we can't
    * get to swish-e or we're having trouble reading from
    * Runtime.exec( ).  This is pretty serious.  Let
    * it bubble up and let the caller deal w/ it.
    * @@exception SearchResultException If query causes
    * the search-engine to return an error -- 'Could
    * not open the index', or 'Please enter at least
    * one search word', or 'One of your search terms
    * is too common', etc., we throw a SearchResultException.
    * @@see #parseSearchResult( Query, String ) Conditions that
    * will throw a SearchResultException
    */
    public SearchResult search( Query inQuery )
        throws SearchResultException,
                java.io.IOException
    {
        SearchResult searchresult = null;

        try
        {
            String swisheSearchResultStr
                = runSwishe( inQuery );
            searchresult
                = parseSearchResult( inQuery,
                    swisheSearchResultStr );
        }

        // Exceptions caught and converted to generic
        // SearchResultException.  Do this because
        // exceptions below are particular to swishe
        // search implementation.
        //
        catch( InterruptedException interruptedException )
        {
            throw new SearchResultException( interruptedException.toString( ) );
        }


        catch( NumberFormatException numberformatexception )
        {
            throw new SearchResultException( numberformatexception.toString( ) );
        }

        catch( java.net.MalformedURLException malformedURLexception )
        {
            throw new SearchResultException( malformedURLexception.toString( ) );
        }

        return searchresult;
    }


    /** Parse the results returned out of swish-e.
    * Parse up the output from swish-e and write it as
    * a SearchResult object.
    * <p>
    * Here is sample successful search output:
    * <pre>
    *   # SWISH format 1.3
    *   # Swish-e format 1.3
    *   #
    *   # Name: (no name)
    *   # Saved as: swish.idx
    *   # Counts: 19 words, 2 files
    *   # Indexed on: 18/10/00 10:56:45 PDT
    *   # Description: (no description)
    *   # Pointer: (no pointer)
    *   # Maintained by: (no maintainer)
    *   # DocumentProperties: Enabled
    *   # Stemming Applied: 0
    *   # Search words: tigris or web
    *   # Number of hits: 1
    *   1000 /home/stack/cvsroot/sandbox/data/helm/htdoc/index.html "Tigris sandbox at /home/stack/cvsroot/sandbox on port 8500" 464
    * </pre>
    * </p>
    * Here is a sample error search result:
    * <pre>
    *   # SWISH format 1.3
    *   # Name: unknown index
    *   err: could not open index file
    * </pre>
    * <p> Possible errors according to http://sunsite.berkeley.edu/SWISH-E/Manual/searching.html
    * are:
    * <pre>
    *   err: no results
    *       There were no results of the search.
    *       Return an empty SearchResult object.
    *       (I noticed that there is no 'Number of
    *       hits' header item when this is returned)
    *
    *   err: could not open index file
    *       Either the index file could not be found or it couldn't be opened.
    *       Throw a SearchResultException
    *
    *   err: no search words specified
    *       No words were specified for searching.
    *       Throw a SearchResultException
    *
    *   err: a word is too common
    *       A search word was used that was too common to give any meaningful feedback.
    *       Throw a SearchResultException
    *
    *   err: the index file is empty
    *       No words are in the index file.
    *       Throw a SearchResultException
    *
    *   err: the index file format is unknown
    *       SWISH-E can't read the particular format of the file.
    *       Throw a SearchResultException
    *
    *   err: the Metaname <name> does not exist in the user configuration file.
    *       The option OKNOMETA is set to NO and a metaname found in one of the files is not in the user configuration file.
    *       Throw a SearchResultException
    * </pre>
    * </p>
    * <p>Implemenation tactic:
    * <pre>
    *   Tactic is to first find our search results format version
    *   string.  If no match, throw an exception.
    *
    *   Next look for headers.  Put all i find into a header hash.
    *
    *   Then there is the search result body.  Switch off what's on
    *   the first line. If it begins w/  'err:', then its an error of
    *   some kind.  Go into error processing either throwing an
    *   exception or just reporting the 'No Results' that Swish-e
    *   reports as an 'Error'. If no error, make a ResultSetItem
    *   per line found and keep a running list in searchResultItems.
    *
    *   Keep an index.  Use this to walk through the passed in
    *   string of search results.
    * </pre>
    * </p>
    *
    * @@param inSearchResults String of swish-e search results.
    * @@param inQuery The query object that generated the search results.
    *
    * @@return Results converted to a SearchResult object.
    *
    * @@exception SearchResultException If we are presented
    * w/ a Swish-e result set format other than what we understand.
    * @@exception NumberFormatException Thrown if difficulty converting
    * a string representation of an integer to an int.  Let caller
    * figure what to do w/ it.  Should never happen.
    */
    public SearchResult parseSearchResult( Query inQuery, String inSearchResults )
        throws SearchResultException,
                NumberFormatException,
                java.net.MalformedURLException
    {
        java.util.List searchResultItems = null;
        java.util.Hashtable hashOfHeaders = new java.util.Hashtable( );
        gnu.regexp.REMatch match = null; // General match pointer. Reused.
        String strPtr = null; // General string pointer.  Reused.
        int index = 0; // Index to walk through inSearchResults with

        // Look for the Search Result Format String.  Search if no match.
        //
        match = reFormatVersion.getMatch( inSearchResults );
        if( match == null )
            throw new SearchResultException( "Search Result Format "
                + "not found: "
                + FORMAT_VERSION_RE );
        index = match.getEndIndex( ); // Move on our index

        // Run through the search result headers.
        //
        while( ( match = reHeaderKeyValues.getMatch( inSearchResults, index ) )
                != null )
        {
            if( DEBUG )
                log( match.toString( ) );
            index = match.getEndIndex( ); // Move on our index.
            hashOfHeaders.put(
                inSearchResults.substring( match.getSubStartIndex( 1 ),
                    match.getSubEndIndex( 1 ) ),
                inSearchResults.substring( match.getSubStartIndex( 2 ),
                    match.getSubEndIndex( 2 ) ) );
        }

        if( DEBUG )
            log( "HEADERS: " + hashOfHeaders.toString( ) );


        // Is the first line of the search result body an error
        // or a set or result items?  If error is other than the
        // 'No match' error, throw exception.  If 'No match',
        // fall out bottom returning empty SearchResult.  If
        // set of results, make up a SearchItemResult.
        //
        label_results: if( ( match = reErrorLine.getMatch( inSearchResults, index ) )
                        != null )
        {
            strPtr = match.toString( );
            if( ( match = reNoResults.getMatch( strPtr ) ) == null )
                throw new SearchResultException( "Search Result Error: "
                                                    + strPtr );
        }
        else
        {
            java.net.URL url = null;
            int ranking = -1;
            int size = -1;
            String title = null;
            searchResultItems
                = new java.util.ArrayList( );

            while( ( match = reSearchResultLine.getMatch( inSearchResults, index ) )
                    != null )
            {
                index = match.getEndIndex( ); // Move on our index.

                strPtr = inSearchResults.substring( match.getSubStartIndex( 1 ),
                            match.getSubEndIndex( 1 ) );
                ranking = Integer.parseInt( strPtr.trim( ) );

                strPtr = inSearchResults.substring( match.getSubStartIndex( 2 ),
                             match.getSubEndIndex( 2 ) );
                strPtr = strPtr.trim( );
                if( strPtr.startsWith( "/" ) )
                    url = new java.net.URL( "file", null, strPtr );
                else
                    throw new java.net.MalformedURLException(
                        "'file' is only protocol handled" );

                strPtr = inSearchResults.substring( match.getSubStartIndex( 3 ),
                            match.getSubEndIndex( 3 ) );
                title = strPtr.trim( );

                strPtr = inSearchResults.substring( match.getSubStartIndex( 4 ),
                             match.getSubEndIndex( 4 ) );
                size = Integer.parseInt( strPtr.trim( ) );

                searchResultItems.add(
                    new SearchResultItem( ranking,
                        size,
                        url,
                        title,
                        docRoot, 
                        domain,
                        port, 
                        protocol ) );
            }
        }   // End of label_results if..else



        // Get number of hits as int.
        //
        int numberOfHits = 0;
        if( ( strPtr = ( String )hashOfHeaders.get( NUMBER_OF_HITS ) )
                != null )
        {
            numberOfHits = Integer.parseInt( strPtr );
        }

        return new SearchResult( inQuery,
                                searchResultItems,
                                ( String )hashOfHeaders.get( INDEX_NAME ),
                                numberOfHits,
                                ( String )hashOfHeaders.get( INDEX_MOD_DATE ),
                                ( String )hashOfHeaders.get( INDEX_STATS ),
                                ( String )hashOfHeaders.get( INDEX_PATH ),
                                ( String )hashOfHeaders.get( INDEX_DESCRIPTION ),
                                ( String )hashOfHeaders.get( SEARCH_WORDS ) );
    }


    /** Do an execute of swish-e using Runtime.exec( ).
    * @@param inQuery Holds the query.
    * @@return String of the swish-e output.  Can be null if
    * we failed our command.
    * @@exception java.io.IOException When we fail to read
    * the output from our Runtime.exec( ) or when the
    * process exits w/ a non-zero value.
    * @@exception InterruptedException Thrown if abnormal
    * termination to process produced by call to Runtime.exec( ).
    */
    protected String runSwishe( Query inQuery )
        throws java.io.IOException,
                InterruptedException
    {
        java.io.StringWriter stringwriter = null;

        // Make up the query to Swish-e. It looks like this:
        //
        //  /usr/bin/swish-e -m MAX_RESULT_COUNT -f PATH_TO_INDEX -w QUERY
        //
        StringBuffer strBufferCmd
            = new StringBuffer( swisheBinary.getName( ) );
        strBufferCmd.append( " -m " );
        strBufferCmd.append( Integer.toString( inQuery.getResultsPerQuery( ) ) );
        strBufferCmd.append( " -f " );
        strBufferCmd.append( inQuery.getSearchIndex( ) );
        strBufferCmd.append( " -w ");
        strBufferCmd.append( inQuery.getQueryString( ) );
        String strCommand = strBufferCmd.toString( );
        if( DEBUG )
            log( "Command to run: " + strCommand );

        // Run the swish-e command.  Wait on its completion.
        //
        Runtime runtime = Runtime.getRuntime( );
        Process process = runtime.exec( strCommand );
        int processResult = process.waitFor( );
        if( processResult != 0 )
            throw new java.io.IOException( "process.waitFor( ) returned non-zero: "
                + Integer.toString( processResult ) );
        processResult = process.exitValue( );
        if( processResult != 0 )
            throw new java.io.IOException( "process.exitValue( ) is non-zero: "
                + Integer.toString( processResult ) );

        // Read the process output.  Ingore the error output.
        // In http://java.sun.com/j2se/1.3/docs/api/java/lang/Process.html#getInputStream()
        // it suggests using a buffered stream.
        //
        java.io.BufferedReader bufferedreader
            = new java.io.BufferedReader(
                new java.io.InputStreamReader( process.getInputStream( ) ) );

        try
        {
           stringwriter = new java.io.StringWriter( );
           int character;
           while( ( character = bufferedreader.read( ) ) != -1  )
                stringwriter.write( character );
        }

        finally
        {
            bufferedreader.close( );
            if( stringwriter != null )
                stringwriter.close( );
        }

        return stringwriter.toString( );
    }



    /** Log passed string.
    * Dumb logging.  Override w/ something better.
    * Used for standalone debugging of this class.
    * @@parm inString2Log String to log
    */
    public void log( String inString2Log )
    {
        System.out.println( ( new java.util.Date( ) ).toString( )
                                + ": "
                                + inString2Log );
    }


    /** Implement main for testing Swish-e searching.
    * Pass at least a 'query' system property using
    * the '-Dkey=value' facility.
    * <p> Here's an example:
    * <pre>
    *   % java -nojit -classpath ./classes:${CLASSPATH} -DDEBUG=true -Dquery="sandbox" org.tigris.helm.search.SearchSwishe
    * </pre>
    *
    *
    * @@param args List of command line arguments. Ignored.
    *
    * @@exception java.io.FileNotFoundException
    * @@exception NullPointerException
    * @@exception SearchResultException
    * @@exception java.io.IOException
    * @@exception gnu.regexp.REException
    */
    public static void main( String[] args )
        throws java.io.FileNotFoundException,
                NullPointerException,
                SearchResultException,
                java.io.IOException,
                gnu.regexp.REException
    {
        // Test that i've been passed a query to run.
        //
        String strQuery = System.getProperty( QUERY_PARAM_KEY );

        if( strQuery == null )
            throw new NullPointerException( "The 'query' system property is "
                + "null. Set it on the command-line "
                + "w/ -Dquery=\"SOME_QUERY\"" );

        SearchSwishe swishe = new SearchSwishe( "/usr/bin/swish-e",
            "/home/stack/cvsroot/sandbox/data/helm/htdoc", 
            "marx.collab.net",
            "8500",
             "http" );
        Query query
            = new Query( strQuery,
                "/home/stack/cvsroot/sandbox/search/swish.idx",
                20 );
        SearchResult searchresult = swishe.search( query );
        System.out.println( searchresult.toString( ) );
        return;
    }

}
@
