<?PHP

#
#   FILE:  SearchEngine.php
#
#   Open Source Metadata Archive Search Engine (OSMASE)
#   Copyright 2002-2011 Edward Almasy and Internet Scout
#   http://scout.wisc.edu
#

class SearchEngine {

    # ---- PUBLIC INTERFACE --------------------------------------------------

    # possible types of logical operators
    const LOGIC_AND = 1;
    const LOGIC_OR = 2;

    # flags used for indicating field types
    const FIELDTYPE_TEXT = 1;
    const FIELDTYPE_NUMERIC = 2;
    const FIELDTYPE_DATE = 3;
    const FIELDTYPE_DATERANGE = 4;

    # object constructor
    function SearchEngine(&$DB, $ItemTableName, $ItemIdFieldName)
    {
        # save database object for our use
        $this->DB = $DB;

        # save item access parameters
        $this->ItemTableName = $ItemTableName;
        $this->ItemIdFieldName = $ItemIdFieldName;

        # define flags used for indicating word states
        if (!defined("WORD_PRESENT"))  {  define("WORD_PRESENT", 1);  }
        if (!defined("WORD_EXCLUDED")) {  define("WORD_EXCLUDED", 2);  }
        if (!defined("WORD_REQUIRED")) {  define("WORD_REQUIRED", 4);  }

        # set default debug state
        $this->DebugLevel = 0;
    }

    # add field to be searched
    function AddField(
            $FieldName, $DBFieldName, $FieldType, $Weight, $UsedInKeywordSearch)
    {
        # save values
        $this->FieldInfo[$FieldName]["DBFieldName"] = $DBFieldName;
        $this->FieldInfo[$FieldName]["FieldType"] = $FieldType;
        $this->FieldInfo[$FieldName]["Weight"] = $Weight;
        $this->FieldInfo[$FieldName]["InKeywordSearch"] = $UsedInKeywordSearch;
    }

    # retrieve info about tables and fields (useful for child objects)
    function ItemTableName() {  return $this->ItemTableName;  }
    function ItemIdFieldName() {  return $this->ItemIdFieldName;  }
    function DBFieldName($FieldName)
            {  return $this->FieldInfo[$FieldName]["DBFieldName"];  }
    function FieldType($FieldName)
            {  return $this->FieldInfo[$FieldName]["FieldType"];  }
    function FieldWeight($FieldName)
            {  return $this->FieldInfo[$FieldName]["Weight"];  }
    function FieldInKeywordSearch($FieldName)
            {  return $this->FieldInfo[$FieldName]["InKeywordSearch"];  }

    # set debug level
    function DebugLevel($Setting)
    {
        $this->DebugLevel = $Setting;
    }


    # ---- search functions

    # perform keyword search
    function Search($SearchString, $StartingResult = 0, $NumberOfResults = 10,
            $SortByField = NULL, $SortDescending = TRUE)
    {
        $SearchString = $this->SetDebugLevel($SearchString);
        $this->DMsg(0, "In Search() with search string \"".$SearchString."\"");

        # save start time to use in calculating search time
        $StartTime = microtime(TRUE);

        # clear word counts
        $this->InclusiveTermCount = 0;
        $this->RequiredTermCount = 0;
        $this->ExcludedTermCount = 0;

        # parse search string into terms
        $Words = $this->ParseSearchStringForWords($SearchString);
        $this->DMsg(1, "Found ".count($Words)." words");

        # parse search string for phrases
        $Phrases = $this->ParseSearchStringForPhrases($SearchString);
        $this->DMsg(1, "Found ".count($Phrases)." phrases");

        # if only excluded terms specified
        if ($this->ExcludedTermCount && !$this->InclusiveTermCount)
        {
            # load all records
            $this->DMsg(1, "Loading all records");
            $Scores = $this->LoadScoresForAllRecords();
        }
        else
        {
            # perform searches
            $Scores = $this->SearchForWords($Words);
            $this->DMsg(1, "Found ".count($Scores)." results after word search");
            $Scores = $this->SearchForPhrases($Phrases, $Scores);
            $this->DMsg(1, "Found ".count($Scores)." results after phrase search");
        }

        # if search results found
        if (count($Scores) > 0)
        {
            # handle any excluded words
            $Scores = $this->FilterOnExcludedWords($Words, $Scores);

            # strip off any results that don't contain required words
            $Scores = $this->FilterOnRequiredWords($Scores);
        }

        # count, sort, and trim search result scores list
        $Scores = $this->CleanScores($Scores, $StartingResult, $NumberOfResults,
                $SortByField, $SortDescending);

        # record search time
        $this->LastSearchTime = microtime(TRUE) - $StartTime;

        # return list of items to caller
        $this->DMsg(0, "Ended up with ".$this->NumberOfResultsAvailable." results");
        return $Scores;
    }

    # perform search across multiple fields and return trimmed results to caller
    function FieldedSearch($SearchStrings, $StartingResult = 0, $NumberOfResults = 10,
            $SortByField = NULL, $SortDescending = TRUE)
    {
        $SearchStrings = $this->SetDebugLevel($SearchStrings);
        $this->DMsg(0, "In FieldedSearch() with "
                .count($SearchStrings)." search strings");

        # save start time to use in calculating search time
        $StartTime = microtime(TRUE);

        # perform search
        $Scores = $this->SearchAcrossFields($SearchStrings);
        $Scores = ($Scores === NULL) ? array() : $Scores;

        # count, sort, and trim search result scores list
        $Scores = $this->CleanScores($Scores, $StartingResult, $NumberOfResults,
                $SortByField, $SortDescending);

        # record search time
        $this->LastSearchTime = microtime(TRUE) - $StartTime;

        # return list of items to caller
        $this->DMsg(0, "Ended up with ".$this->NumberOfResultsAvailable." results");
        return $Scores;
    }

    # perform search with logical groups of fielded searches
    function GroupedSearch($SearchGroups, $StartingResult = 0, $NumberOfResults = 10,
            $SortByField = NULL, $SortDescending = TRUE)
    {
        foreach ($SearchGroups as $Index => $Groups)
        {
            if (isset($SearchGroups[$Index]["SearchStrings"]))
            {
                $SearchGroups[$Index]["SearchStrings"] =
                        $this->SetDebugLevel($SearchGroups[$Index]["SearchStrings"]);
            }
        }
        $this->DMsg(0, "In GroupedSearch() with "
                .count($SearchGroups)." search groups");

        # save start time to use in calculating search time
        $StartTime = microtime(TRUE);

        # start with no results
        $Scores = array();

        # save AND/OR search setting
        $SavedSearchLogic = $this->DefaultSearchLogic;

        # for each search group
        $FirstSearch = TRUE;
        foreach ($SearchGroups as $Group)
        {
            $this->DMsg(0, "----- GROUP ---------------------------");

            # if group has AND/OR setting specified
            if (isset($Group["Logic"]))
            {
                # use specified AND/OR setting
                $this->DefaultSearchLogic = $Group["Logic"];
            }
            else
            {
                # use saved AND/OR setting
                $this->DefaultSearchLogic = $SavedSearchLogic;
            }
            $this->DMsg(2, "Logic is "
                    .(($this->DefaultSearchLogic == self::LOGIC_AND) ? "AND" : "OR"));

            # if we have search strings for this group
            if (isset($Group["SearchStrings"]))
            {
                # perform search
                $GroupScores = $this->SearchAcrossFields($Group["SearchStrings"]);

                # if search was conducted
                if ($GroupScores !== NULL)
                {
                    # if saved AND/OR setting is OR or this is first search
                    if (($SavedSearchLogic == self::LOGIC_OR) || $FirstSearch)
                    {
                        # add search results to result list
                        foreach ($GroupScores as $ItemId => $Score)
                        {
                            if (isset($Scores[$ItemId]))
                            {
                                $Scores[$ItemId] += $Score;
                            }
                            else
                            {
                                $Scores[$ItemId] = $Score;
                            }
                        }

                        # (reset flag indicating first search)
                        $FirstSearch = FALSE;
                    }
                    else
                    {
                        # AND search results with previous results
                        $OldScores = $Scores;
                        $Scores = array();
                        foreach ($GroupScores as $ItemId => $Score)
                        {
                            if (isset($OldScores[$ItemId]))
                            {
                                $Scores[$ItemId] = $OldScores[$ItemId] + $Score;
                            }
                        }
                    }
                }
            }
        }

        # restore AND/OR search setting
        $this->DefaultSearchLogic = $SavedSearchLogic;

        # count, sort, and trim search result scores list
        $Scores = $this->CleanScores($Scores, $StartingResult, $NumberOfResults,
                $SortByField, $SortDescending);

        # record search time
        $this->LastSearchTime = microtime(TRUE) - $StartTime;

        # return search results to caller
        $this->DMsg(0, "Ended up with ".$this->NumberOfResultsAvailable." results");
        return $Scores;
    }

    # add function that will be called to filter search results
    function AddResultFilterFunction($FunctionName)
    {
        # save filter function name
        $this->FilterFuncs[] = $FunctionName;
    }

    # get or set default search logic (AND or OR)
    function DefaultSearchLogic($NewSetting = NULL)
    {
        if ($NewSetting != NULL)
        {
            $this->DefaultSearchLogic = $NewSetting;
        }
        return $this->DefaultSearchLogic;
    }

    function SearchTermsRequiredByDefault($NewSetting = TRUE)
    {
        if ($NewSetting)
        {
            $this->DefaultSearchLogic = self::LOGIC_AND;
        }
        else
        {
            $this->DefaultSearchLogic = self::LOGIC_OR;
        }
    }

    function NumberOfResults()
    {
        return $this->NumberOfResultsAvailable;
    }

    function SearchTerms()
    {
        return $this->SearchTermList;
    }

    function SearchTime()
    {
        return $this->LastSearchTime;
    }

    # report total weight for all fields involved in search
    function FieldedSearchWeightScale($SearchStrings)
    {
        $Weight = 0;
        $IncludedKeywordSearch = FALSE;
        foreach ($SearchStrings as $FieldName => $SearchStringArray)
        {
            if ($FieldName == "XXXKeywordXXX")
            {
                $IncludedKeywordSearch = TRUE;
            }
            else
            {
                $Weight += $this->FieldInfo[$FieldName]["Weight"];
            }
        }
        if ($IncludedKeywordSearch)
        {
            foreach ($this->FieldInfo as $FieldName => $Info)
            {
                if ($Info["InKeywordSearch"])
                {
                    $Weight += $Info["Weight"];
                }
            }
        }
        return $Weight;
    }


    # ---- search database update functions

    # update search DB for the specified item
    function UpdateForItem($ItemId)
    {
        # bail out if item ID is negative (indicating a temporary record)
        if ($ItemId < 0) {  return;  }

        # clear word count added flags for this item
        unset($this->WordCountAdded);

        # delete any existing info for this item
        $this->DB->Query("DELETE FROM SearchWordCounts WHERE ItemId = ".$ItemId);

        # for each metadata field
        foreach ($this->FieldInfo as $FieldName => $Info)
        {
            # if search weight for field is positive
            if ($Info["Weight"] > 0)
            {
                # retrieve text for field
                $Text = $this->GetFieldContent($ItemId, $FieldName);

                # if text is array
                if (is_array($Text))
                {
                    # for each text string in array
                    foreach ($Text as $String)
                    {
                        # record search info for text
                        $this->RecordSearchInfoForText($ItemId, $FieldName,
                                                       $Info["Weight"], $String,
                                                       $Info["InKeywordSearch"]);
                    }
                }
                else
                {
                    # record search info for text
                    $this->RecordSearchInfoForText($ItemId, $FieldName,
                                                   $Info["Weight"], $Text,
                                                   $Info["InKeywordSearch"]);
                }
            }
        }
    }

    # update search DB for the specified range of items
    function UpdateForItems($StartingItemId, $NumberOfItems)
    {
        # retrieve IDs for specified number of items starting at specified ID
        $this->DB->Query("SELECT ".$this->ItemIdFieldName." FROM ".$this->ItemTableName
                ." WHERE ".$this->ItemIdFieldName." >= ".$StartingItemId
                ." ORDER BY ".$this->ItemIdFieldName." LIMIT ".$NumberOfItems);
        $ItemIds = $this->DB->FetchColumn($this->ItemIdFieldName);

        # for each retrieved item ID
        foreach ($ItemIds as $ItemId)
        {
            # update search info for item
            $this->UpdateForItem($ItemId);
        }

        # return ID of last item updated to caller
        return $ItemId;
    }

    # drop all data pertaining to item from search DB
    function DropItem($ItemId)
    {
        # drop all entries pertaining to item from word count table
        $this->DB->Query("DELETE FROM SearchWordCounts WHERE ItemId = ".$ItemId);
    }

    # drop all data pertaining to field from search DB
    function DropField($FieldName)
    {
        # retrieve our ID for field
        $FieldId = $this->DB->Query("SELECT FieldId FROM SearchFields "
                ."WHERE FieldName = '".addslashes($FieldName)."'", "FieldId");

        # drop all entries pertaining to field from word counts table
        $this->DB->Query("DELETE FROM SearchWordCounts WHERE FieldId = \'".$FieldId."\'");

        # drop field from our fields table
        $this->DB->Query("DELETE FROM SearchFields WHERE FieldId = \'".$FieldId."\'");
    }

    # return total number of terms indexed by search engine
    function SearchTermCount()
    {
        return $this->DB->Query("SELECT COUNT(*) AS TermCount"
                ." FROM SearchWords", "TermCount");
    }

    # return total number of items indexed by search engine
    function ItemCount()
    {
        return $this->DB->Query("SELECT COUNT(DISTINCT ItemId) AS ItemCount"
                ." FROM SearchWordCounts", "ItemCount");
    }

    /**
    * Add synonyms.
    * @param Word Word for which synonyms should apply.
    * @param Synonyms Array of synonyms.
    * @return Count of new synonyms added.
    */
    function AddSynonyms($Word, $Synonyms)
    {
        # asssume no synonyms will be added
        $AddCount = 0;

        # get ID for word
        $WordId = $this->GetWordId($Word, TRUE);

        # for each synonym passed in
        foreach ($Synonyms as $Synonym)
        {
            # get ID for synonym
            $SynonymId = $this->GetWordId($Synonym, TRUE);

            # if synonym is not already in database
            $this->DB->Query("SELECT * FROM SearchWordSynonyms"
                    ." WHERE (WordIdA = ".$WordId
                        ." AND WordIdB = ".$SynonymId.")"
                    ." OR (WordIdB = ".$WordId
                        ." AND WordIdA = ".$SynonymId.")");
            if ($this->DB->NumRowsSelected() == 0)
            {
                # add synonym entry to database
                $this->DB->Query("INSERT INTO SearchWordSynonyms"
                        ." (WordIdA, WordIdB)"
                        ." VALUES (".$WordId.", ".$SynonymId.")");
                $AddCount++;
            }
        }

        # report to caller number of new synonyms added
        return $AddCount;
    }

    # remove synonym(s)
    function RemoveSynonyms($Word, $Synonyms = NULL)
    {
        # find ID for word
        $WordId = $this->GetWordId($Word);

        # if ID found
        if ($WordId !== NULL)
        {
            # if no specific synonyms provided
            if ($Synonyms === NULL)
            {
                # remove all synonyms for word
                $this->DB->Query("DELETE FROM SearchWordSynonyms"
                        ." WHERE WordIdA = '".$WordId."'"
                        ." OR WordIdB = '".$WordId."'");
            }
            else
            {
                # for each specified synonym
                foreach ($Synonyms as $Synonym)
                {
                    # look up ID for synonym
                    $SynonymId = $this->GetWordId($Synonym);

                    # if synonym ID was found
                    if ($SynonymId !== NULL)
                    {
                        # delete synonym entry
                        $this->DB->Query("DELETE FROM SearchWordSynonyms"
                                ." WHERE (WordIdA = '".$WordId."'"
                                    ." AND WordIdB = '".$SynonymId."')"
                                ." OR (WordIdB = '".$WordId."'"
                                    ." AND WordIdA = '".$SynonymId."')");
                    }
                }
            }
        }
    }

    # remove all synonyms
    function RemoveAllSynonyms()
    {
        $this->DB->Query("DELETE FROM SearchWordSynonyms");
    }

    # get synonyms for word (returns array of synonyms)
    function GetSynonyms($Word)
    {
        # assume no synonyms will be found
        $Synonyms = array();

        # look up ID for word
        $WordId = $this->GetWordId($Word);

        # if word ID was found
        if ($WordId !== NULL)
        {
            # look up IDs of all synonyms for this word
            $this->DB->Query("SELECT WordIdA, WordIdB FROM SearchWordSynonyms"
                    ." WHERE WordIdA = ".$WordId
                    ." OR WordIdB = ".$WordId);
            $SynonymIds = array();
            while ($Record = $this->DB->FetchRow)
            {
                $SynonymIds[] = ($Record["WordIdA"] == $WordId)
                        ? $Record["WordIdB"] : $Record["WordIdA"];
            }

            # for each synonym ID
            foreach ($SynonymIds as $SynonymId)
            {
                # look up synonym word and add to synonym list
                $Synonyms[] = $this->GetWord($SynonymId);
            }
        }

        # return synonyms to caller
        return $Synonyms;
    }

    # get all synonyms (returns 2D array w/ words as first index)
    function GetAllSynonyms()
    {
        # assume no synonyms will be found
        $SynonymList = array();

        # for each synonym ID pair
        $OurDB = new SPTDatabase();
        $OurDB->Query("SELECT WordIdA, WordIdB FROM SearchWordSynonyms");
        while ($Record = $OurDB->FetchRow())
        {
            # look up words
            $Word = $this->GetWord($Record["WordIdA"]);
            $Synonym = $this->GetWord($Record["WordIdB"]);

            # if we do not already have an entry for the word
            #       or synonym is not listed for this word
            if (!isset($SynonymList[$Word])
                    || !in_array($Synonym, $SynonymList[$Word]))
            {
                # add entry for synonym
                $SynonymList[$Word][] = $Synonym;
            }

            # if we do not already have an entry for the synonym
            #       or word is not listed for this synonym
            if (!isset($SynonymList[$Synonym])
                    || !in_array($Word, $SynonymList[$Synonym]))
            {
                # add entry for word
                $SynonymList[$Synonym][] = $Word;
            }
        }

        # for each word
        # (this loop removes reciprocal duplicates)
        foreach ($SynonymList as $Word => $Synonyms)
        {
            # for each synonym for that word
            foreach ($Synonyms as $Synonym)
            {
                # if synonym has synonyms and word is one of them
                if (isset($SynonymList[$Synonym])
                        && isset($SynonymList[$Word])
                        && in_array($Word, $SynonymList[$Synonym])
                        && in_array($Synonym, $SynonymList[$Word]))
                {
                    # if word has less synonyms than synonym
                    if (count($SynonymList[$Word])
                            < count($SynonymList[$Synonym]))
                    {
                        # remove synonym from synonym list for word
                        $SynonymList[$Word] = array_diff(
                                $SynonymList[$Word], array($Synonym));

                        # if no synonyms left for word
                        if (!count($SynonymList[$Word]))
                        {
                            # remove empty synonym list for word
                            unset($SynonymList[$Word]);
                        }
                    }
                    else
                    {
                        # remove word from synonym list for synonym
                        $SynonymList[$Synonym] = array_diff(
                                $SynonymList[$Synonym], array($Word));

                        # if no synonyms left for word
                        if (!count($SynonymList[$Synonym]))
                        {
                            # remove empty synonym list for word
                            unset($SynonymList[$Synonym]);
                        }
                    }
                }
            }
        }

        # sort array alphabetically (just for convenience)
        foreach ($SynonymList as $Word => $Synonyms)
        {
            asort($SynonymList[$Word]);
        }
        ksort($SynonymList);

        # return 2D array of synonyms to caller
        return $SynonymList;
    }

    # set all synonyms (accepts 2D array w/ words as first index)
    function SetAllSynonyms($SynonymList)
    {
        # remove all existing synonyms
        $this->RemoveAllSynonyms();

        # for each synonym entry passed in
        foreach ($SynonymList as $Word => $Synonyms)
        {
            # add synonyms for word
            $this->AddSynonyms($Word, $Synonyms);
        }
    }

    /**
    * Load synonyms from a file.  Each line of file should contain one word
    *   at the beginning of the line, followed by one or more synonyms
    *   separated by spaces or commas.  Blank lines or lines beginning with
    *   "#" (i.e. comments) will be ignored.
    * @param FileName Name of file containing synonyms (with path if needed).
    * @return Number of new synonyms added.
    */
    function LoadSynonymsFromFile($FileName)
    {
        # asssume no synonyms will be added
        $AddCount = 0;

        # read in contents of file
        $Lines = file($FileName, FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES);

        # if file contained lines
        if (count($Lines))
        {
            # for each line of file
            foreach ($Lines as $Line)
            {
                # if line is not a comment
                if (!preg_match("/[\s]*#/", $Line))
                {
                    # split line into words
                    $Words = preg_split("/[\s,]+/", $Line);

                    # if synonyms found
                    if (count($Words) > 1)
                    {
                        # separate out word and synonyms
                        $Word = array_shift($Words);

                        # add synonyms
                        $AddCount += $this->AddSynonyms($Word, $Words);
                    }
                }
            }
        }

        # return count of synonyms added to caller
        return $AddCount;
    }

    # suggest alternatives
    function SuggestAlternateSearches($SearchString)
    {
        #
    }


    # ---- PRIVATE INTERFACE -------------------------------------------------

    protected $DB;
    protected $DebugLevel;
    protected $ItemTableName;
    protected $ItemIdFieldName;
    protected $NumberOfResultsAvailable;
    protected $LastSearchTime;
    protected $FilterFuncs;
    protected $DefaultSearchLogic = self::LOGIC_AND;
    protected $StemmingEnabled = TRUE;
    protected $SynonymsEnabled = TRUE;

    private $WordCountAdded;
    private $FieldIds;
    private $FieldInfo;
    private $RequiredTermCount;
    private $RequiredTermCounts;
    private $InclusiveTermCount;
    private $ExcludedTermCount;
    private $SearchTermList;

    const STEM_ID_OFFSET = 1000000;


    # ---- common private functions (used in both searching and DB build)

    # normalize and parse search string into list of search terms
    private function ParseSearchStringForWords($SearchString, $IgnorePhrases = FALSE)
    {
        # strip off any surrounding whitespace
        $Text = trim($SearchString);

        # set up normalization replacement strings
        $Patterns = array(
                "/'s[^a-z0-9\\-+~]+/i", # get rid of possessive plurals
                "/'/",                  # get rid of single quotes / apostrophes
                "/\"[^\"]*\"/",         # get rid of phrases  (NOTE: HARD-CODED INDEX BELOW!!!)  "
                "/\\([^)]*\\)/",        # get rid of groups  (NOTE: HARD-CODED INDEX BELOW!!!)
                "/[^a-z0-9\\-+~]+/i",   # convert non-alphanumerics / non-minus/plus to a space
                "/([^\\s])-+/i",        # convert minus preceded by anything but whitespace to a space
                "/([^\\s])\\++/i",      # convert plus preceded by anything but whitespace to a space
                "/-\\s/i",              # convert minus followed by whitespace to a space
                "/\\+\\s/i",            # convert plus followed by whitespace to a space
                "/~\\s/i",              # convert tilde followed by whitespace to a space
                "/[ ]+/"                # convert multiple spaces to one space
                );
        $Replacements = array(
                " ",
                "",
                " ",
                " ",
                "\\1 ",
                "\\1 ",
                " ",
                " ",
                " ",
                " ",
                " "
                );

        # if we are supposed to ignore phrases and groups (series of words in quotes or surrounded by parens)
        if ($IgnorePhrases)
        {
            # switch phrase removal to double quote removal  (HARD-CODED INDEX INTO PATTERN LIST!!)
            $Patterns[2] = "/\"/";

            # switch group removal to paren removal  (HARD-CODED INDEX INTO PATTERN LIST!!)
            $Patterns[3] = "/[\(\)]+/";
        }

        # remove punctuation from text and normalize whitespace
        $Text = preg_replace($Patterns, $Replacements, $Text);
        $this->DMsg(2, "Normalized search string is '".$Text."'");

        # convert text to lower case
        $Text = strtolower($Text);

        # strip off any extraneous whitespace
        $Text = trim($Text);

        # start with an empty array
        $Words = array();

        # if we have no words left after parsing
        if (strlen($Text) != 0)
        {
            # for each word
            foreach (explode(" ", $Text) as $Word)
            {
                # grab first character of word
                $FirstChar = substr($Word, 0, 1);

                # strip off option characters and set flags appropriately
                $Flags = WORD_PRESENT;
                if ($FirstChar == "-")
                {
                    $Word = substr($Word, 1);
                    $Flags |= WORD_EXCLUDED;
                    if (!isset($Words[$Word]))
                    {
                        $this->ExcludedTermCount++;
                    }
                }
                else
                {
                    if ($FirstChar == "~")
                    {
                        $Word = substr($Word, 1);
                    }
                    elseif (($this->DefaultSearchLogic == self::LOGIC_AND)
                            || ($FirstChar == "+"))
                    {
                        if ($FirstChar == "+")
                        {
                            $Word = substr($Word, 1);
                        }
                        $Flags |= WORD_REQUIRED;
                        if (!isset($Words[$Word]))
                        {
                            $this->RequiredTermCount++;
                        }
                    }
                    if (!isset($Words[$Word]))
                    {
                        $this->InclusiveTermCount++;
                        $this->SearchTermList[] = $Word;
                    }
                }

                # store flags to indicate word found
                $Words[$Word] = $Flags;
                $this->DMsg(3, "Word identified (".$Word.")");
            }
        }

        # return normalized words to caller
        return $Words;
    }

    protected function GetFieldId($FieldName)
    {
        # if field ID is not in cache
        if (!isset($this->FieldIds[$FieldName]))
        {
            # look up field info in database
            $this->DB->Query("SELECT FieldId FROM SearchFields "
                    ."WHERE FieldName = '".addslashes($FieldName)."'");

            # if field was found
            if ($Record = $this->DB->FetchRow())
            {
                # load info from DB record
                $FieldId = $Record["FieldId"];
            }
            else
            {
                # add field to database
                $this->DB->Query("INSERT INTO SearchFields (FieldName) "
                        ."VALUES ('".addslashes($FieldName)."')");

                  # retrieve ID for newly added field
                $FieldId = $this->DB->LastInsertId("SearchFields");
            }

            # cache field info
            $this->FieldIds[$FieldName] = $FieldId;
        }

        # return cached ID to caller
        return $this->FieldIds[$FieldName];
    }

    # retrieve ID for specified word (returns NULL if no ID found)
    private function GetWordId($Word, $AddIfNotFound = FALSE)
    {
        static $WordIdCache;

        # if word was in ID cache
        if (isset($WordIdCache[$Word]))
        {
            # use ID from cache
            $WordId = $WordIdCache[$Word];
        }
        else
        {
            # look up ID in database
            $WordId = $this->DB->Query("SELECT WordId"
                    ." FROM SearchWords"
                    ." WHERE WordText='".addslashes($Word)."'",
                    "WordId");

            # if ID was not found and caller requested it be added
            if (($WordId === NULL) && $AddIfNotFound)
            {
                # add word to database
                $this->DB->Query("INSERT INTO SearchWords (WordText)"
                        ." VALUES ('".addslashes(strtolower($Word))."')");

                # get ID for newly added word
                $WordId = $this->DB->LastInsertId("SearchWords");
            }

            # save ID to cache
            $WordIdCache[$Word] = $WordId;
        }

        # return ID to caller
        return $WordId;
    }

    # retrieve ID for specified word stem (returns NULL if no ID found)
    private function GetStemId($Stem, $AddIfNotFound = FALSE)
    {
        static $StemIdCache;

        # if stem was in ID cache
        if (isset($StemIdCache[$Stem]))
        {
            # use ID from cache
            $StemId = $StemIdCache[$Stem];
        }
        else
        {
            # look up ID in database
            $StemId = $this->DB->Query("SELECT WordId"
                    ." FROM SearchStems"
                    ." WHERE WordText='".addslashes($Stem)."'",
                    "WordId");

            # if ID was not found and caller requested it be added
            if (($StemId === NULL) && $AddIfNotFound)
            {
                # add stem to database
                $this->DB->Query("INSERT INTO SearchStems (WordText)"
                        ." VALUES ('".addslashes(strtolower($Stem))."')");

                # get ID for newly added stem
                $StemId = $this->DB->LastInsertId("SearchStems");
            }

            # adjust from DB ID value to stem ID value
            $StemId += self::STEM_ID_OFFSET;

            # save ID to cache
            $StemIdCache[$Stem] = $StemId;
        }

        # return ID to caller
        return $StemId;
    }

    # retrieve word for specified word ID (returns FALSE if no word found)
    private function GetWord($WordId)
    {
        static $WordCache;

        # if word was in cache
        if (isset($WordCache[$WordId]))
        {
            # use word from cache
            $Word = $WordCache[$WordId];
        }
        else
        {
            # adjust search location and word ID if word is stem
            $TableName = "SearchWords";
            if ($WordId >= self::STEM_ID_OFFSET)
            {
                $TableName = "SearchStems";
                $WordId -= self::STEM_ID_OFFSET;
            }

            # look up word in database
            $Word = $this->DB->Query("SELECT WordText"
                    ." FROM ".$TableName
                    ." WHERE WordId='".$WordId."'",
                    "WordText");

            # save word to cache
            $WordCache[$WordId] = $Word;
        }

        # return word to caller
        return $Word;
    }


    # ---- private functions used in searching

    # perform search across multiple fields and return raw results to caller
    private function SearchAcrossFields($SearchStrings)
    {
        # start by assuming no search will be done
        $Scores = NULL;

        # clear word counts
        $this->InclusiveTermCount = 0;
        $this->RequiredTermCount = 0;
        $this->ExcludedTermCount = 0;

        # for each field
        $NeedComparisonSearch = FALSE;
        foreach ($SearchStrings as $FieldName => $SearchStringArray)
        {
            # convert search string to array if needed
            if (!is_array($SearchStringArray))
            {
                $SearchStringArray = array($SearchStringArray);
            }

            # for each search string for this field
            foreach ($SearchStringArray as $SearchString)
            {
                # if field is keyword or field is text and does not look like comparison match
                if (($FieldName == "XXXKeywordXXX")
                    || (isset($this->FieldInfo[$FieldName])
                        && ($this->FieldInfo[$FieldName]["FieldType"] == self::FIELDTYPE_TEXT)
                        && !preg_match("/^[><!]=./", $SearchString)
                        && !preg_match("/^[><=]./", $SearchString)))
                {
                    $this->DMsg(0, "Searching text field \""
                            .$FieldName."\" for string \"$SearchString\"");

                    # normalize text and split into words
                    $Words[$FieldName] =
                            $this->ParseSearchStringForWords($SearchString);

                    # calculate scores for matching items
                    if (count($Words[$FieldName]))
                    {
                        $Scores = $this->SearchForWords(
                                $Words[$FieldName], $FieldName, $Scores);
                        $this->DMsg(3, "Have "
                                .count($Scores)." results after word search");
                    }

                    # split into phrases
                    $Phrases[$FieldName] =
                            $this->ParseSearchStringForPhrases($SearchString);

                    # handle any phrases
                    if (count($Phrases[$FieldName]))
                    {
                        $Scores = $this->SearchForPhrases(
                                $Phrases[$FieldName], $Scores, $FieldName, TRUE, FALSE);
                        $this->DMsg(3, "Have "
                                .count($Scores)." results after phrase search");
                    }
                }
                else
                {
                    # set flag to indicate possible comparison search candidate found
                    $NeedComparisonSearch = TRUE;
                }
            }
        }

        # perform comparison searches
        if ($NeedComparisonSearch)
        {
            $Scores = $this->SearchForComparisonMatches($SearchStrings, $Scores);
            $this->DMsg(3, "Have ".count($Scores)." results after comparison search");
        }

        # if no results found and exclusions specified
        if (!count($Scores) && $this->ExcludedTermCount)
        {
            # load all records
            $Scores = $this->LoadScoresForAllRecords();
        }

        # if search results found
        if (count($Scores))
        {
            # for each search text string
            foreach ($SearchStrings as $FieldName => $SearchStringArray)
            {
                # convert search string to array if needed
                if (!is_array($SearchStringArray))
                {
                    $SearchStringArray = array($SearchStringArray);
                }

                # for each search string for this field
                foreach ($SearchStringArray as $SearchString)
                {
                    # if field is text
                    if (($FieldName == "XXXKeywordXXX")
                            || (isset($this->FieldInfo[$FieldName])
                                && ($this->FieldInfo[$FieldName]["FieldType"]
                                        == self::FIELDTYPE_TEXT)))
                    {
                        # if there are words in search text
                        if (isset($Words[$FieldName]))
                        {
                            # handle any excluded words
                            $Scores = $this->FilterOnExcludedWords($Words[$FieldName], $Scores, $FieldName);
                        }

                        # handle any excluded phrases
                        if (isset($Phrases[$FieldName]))
                        {
                            $Scores = $this->SearchForPhrases(
                                    $Phrases[$FieldName], $Scores, $FieldName, FALSE, TRUE);
                        }
                    }
                }
            }

            # strip off any results that don't contain required words
            $Scores = $this->FilterOnRequiredWords($Scores);
        }

        # return search result scores to caller
        return $Scores;
    }

    # search for words in specified field
    private function SearchForWords(
            $Words, $FieldName = "XXXKeywordXXX", $Scores = NULL)
    {
        $DB = $this->DB;

        # start with empty search result scores list if none passed in
        if ($Scores == NULL)
        {
            $Scores = array();
        }

        # grab field ID
        $FieldId = $this->GetFieldId($FieldName);

        # for each word
        foreach ($Words as $Word => $Flags)
        {
            $this->DMsg(2, "Searching for word '${Word}' in field ".$FieldName);

            # if word is not excluded
            if (!($Flags & WORD_EXCLUDED))
            {
                # look up record ID for word
                $this->DMsg(2, "Looking up word \"".$Word."\"");
                $WordId = $this->GetWordId($Word);

                # if word is in DB
                if ($WordId !== NULL)
                {
                    # look up counts for word
                    $DB->Query("SELECT ItemId,Count FROM SearchWordCounts "
                            ."WHERE WordId = ".$WordId
                            ." AND FieldId = ".$FieldId);
                    $Counts = $DB->FetchColumn("Count", "ItemId");

                    # if synonym support is enabled
                    if ($this->SynonymsEnabled)
                    {
                        # look for any synonyms
                        $DB->Query("SELECT WordIdA, WordIdB"
                                ." FROM SearchWordSynonyms"
                                ." WHERE WordIdA = ".$WordId
                                ." OR WordIdB = ".$WordId);

                        # if synonyms were found
                        if ($DB->NumRowsSelected())
                        {
                            # retrieve synonym IDs
                            $SynonymIds = array();
                            while ($Record = $DB->FetchRow())
                            {
                                $SynonymIds[] = ($Record["WordIdA"] == $WordId)
                                        ? $Record["WordIdB"]
                                        : $Record["WordIdA"];
                            }

                            # for each synonym
                            foreach ($SynonymIds as $SynonymId)
                            {
                                # retrieve counts for synonym
                                $DB->Query("SELECT ItemId,Count"
                                        ." FROM SearchWordCounts"
                                        ." WHERE WordId = ".$SynonymId
                                        ." AND FieldId = ".$FieldId);
                                $SynonymCounts = $DB->FetchColumn("Count", "ItemId");

                                # for each count
                                foreach ($SynonymCounts as $ItemId => $Count)
                                {
                                    # adjust count because it's a synonym
                                    $AdjustedCount = ceil($Count / 2);

                                    # add count to existing counts
                                    if (isset($Counts[$ItemId]))
                                    {
                                        $Counts[$ItemId] += $AdjustedCount;
                                    }
                                    else
                                    {
                                        $Counts[$ItemId] = $AdjustedCount;
                                    }
                                }
                            }
                        }
                    }
                }

                # if stemming is enabled
                if ($this->StemmingEnabled)
                {
                    # retrieve stem ID
                    $Stem = PorterStemmer::Stem($Word);
                    $this->DMsg(2, "Looking up stem \"".$Stem."\"");
                    $StemId = $this->GetStemId($Stem);

                    # if ID found for stem
                    if ($StemId !== NULL)
                    {
                        # retrieve counts for stem
                        $DB->Query("SELECT ItemId,Count"
                                ." FROM SearchWordCounts"
                                ." WHERE WordId = ".$StemId
                                ." AND FieldId = ".$FieldId);
                        $StemCounts = $DB->FetchColumn("Count", "ItemId");

                        # for each count
                        foreach ($StemCounts as $ItemId => $Count)
                        {
                            # adjust count because it's a stem
                            $AdjustedCount = ceil($Count / 2);

                            # add count to existing counts
                            if (isset($Counts[$ItemId]))
                            {
                                $Counts[$ItemId] += $AdjustedCount;
                            }
                            else
                            {
                                $Counts[$ItemId] = $AdjustedCount;
                            }
                        }
                    }
                }

                # if counts were found
                if (isset($Counts))
                {
                    # for each count
                    foreach ($Counts as $ItemId => $Count)
                    {
                        # if word flagged as required
                        if ($Flags & WORD_REQUIRED)
                        {
                            # increment required word count for record
                            if (isset($this->RequiredTermCounts[$ItemId]))
                            {
                                $this->RequiredTermCounts[$ItemId]++;
                            }
                            else
                            {
                                $this->RequiredTermCounts[$ItemId] = 1;
                            }
                        }

                        # add to item record score
                        if (isset($Scores[$ItemId]))
                        {
                            $Scores[$ItemId] += $Count;
                        }
                        else
                        {
                            $Scores[$ItemId] = $Count;
                        }
                    }
                }
            }
        }

        # return basic scores to caller
        return $Scores;
    }

    # extract phrases (terms surrounded by quotes) from search string
    private function ParseSearchStringForPhrases($SearchString)
    {
        # split into chunks delimited by double quote marks
        $Pieces = explode("\"", $SearchString);   # "

        # for each pair of chunks
        $Index = 2;
        $Phrases = array();
        while ($Index < count($Pieces))
        {
            # grab phrase from chunk
            $Phrase = trim(addslashes($Pieces[$Index - 1]));
            $Flags = WORD_PRESENT;

            # grab first character of phrase
            $FirstChar = substr($Pieces[$Index - 2], -1);

            # set flags to reflect any option characters
            if ($FirstChar == "-")
            {
                $Flags |= WORD_EXCLUDED;
                if (!isset($Phrases[$Phrase]))
                {
                    $this->ExcludedTermCount++;
                }
            }
            else
            {
                if ((($this->DefaultSearchLogic == self::LOGIC_AND) && ($FirstChar != "~"))
                        || ($FirstChar == "+"))
                {
                    $Flags |= WORD_REQUIRED;
                    if (!isset($Phrases[$Phrase]))
                    {
                        $this->RequiredTermCount++;
                    }
                }
                if (!isset($Phrases[$Phrase]))
                {
                    $this->InclusiveTermCount++;
                    $this->SearchTermList[] = $Phrase;
                }
            }
            $Phrases[$Phrase] = $Flags;

            # move to next pair of chunks
            $Index += 2;
        }

        # return phrases to caller
        return $Phrases;
    }

    # extract groups (terms surrounded by parens) from search string
    # (NOTE: NOT YET IMPLEMENTED!!!)
    private function ParseSearchStringForGroups($SearchString)
    {
        # split into chunks delimited by open paren
        $Pieces = explode("(", $SearchString);

        # for each chunk
        $Index = 2;
        while ($Index < count($Pieces))
        {
            # grab phrase from chunk
            $Group = trim(addslashes($Pieces[$Index - 1]));
            $Groups[] = $Group;

            # move to next pair of chunks
            $Index += 2;
        }

        # return phrases to caller
        return $Groups;
    }

    protected function SearchFieldForPhrases($FieldName, $Phrase)
    {
        # error out
        exit("<br>SE - ERROR:  SearchFieldForPhrases() not implemented<br>\n");
    }

    private function SearchForPhrases($Phrases, $Scores, $FieldName = "XXXKeywordXXX",
            $ProcessNonExcluded = TRUE, $ProcessExcluded = TRUE)
    {
        # if phrases are found
        if (count($Phrases) > 0)
        {
            # if this is a keyword search
            if ($FieldName == "XXXKeywordXXX")
            {
                # for each field
                foreach ($this->FieldInfo as $KFieldName => $Info)
                {
                    # if field is marked to be included in keyword searches
                    if ($Info["InKeywordSearch"])
                    {
                        # call ourself with that field
                        $Scores = $this->SearchForPhrases($Phrases, $Scores, $KFieldName,
                                                          $ProcessNonExcluded, $ProcessExcluded);
                    }
                }
            }
            else
            {
                # for each phrase
                foreach ($Phrases as $Phrase => $Flags)
                {
                    $this->DMsg(2, "Searching for phrase '".$Phrase
                            ."' in field ".$FieldName);

                    # if phrase flagged as excluded and we are doing excluded phrases
                    #         or phrase flagged as non-excluded and we are doing non-excluded phrases
                    if (($ProcessExcluded && ($Flags & WORD_EXCLUDED))
                            || ($ProcessNonExcluded && !($Flags & WORD_EXCLUDED)))
                    {
                        # initialize score list if necessary
                        if ($Scores === NULL) {  $Scores = array();  }

                        # retrieve list of items that contain phrase
                        $ItemIds = $this->SearchFieldForPhrases(
                                $FieldName, $Phrase);

                        # for each item that contains phrase
                        foreach ($ItemIds as $ItemId)
                        {
                            # if we are doing excluded phrases and phrase flagged as excluded
                            if ($ProcessExcluded && ($Flags & WORD_EXCLUDED))
                            {
                                # knock item off of list
                                unset($Scores[$ItemId]);
                            }
                            elseif ($ProcessNonExcluded)
                            {
                                # calculate phrase value based on number of words and field weight
                                $PhraseScore = count(preg_split("/[\s]+/", $Phrase, -1, PREG_SPLIT_NO_EMPTY))
                                                       * $this->FieldInfo[$FieldName]["Weight"];
                                $this->DMsg(2, "Phrase score is ".$PhraseScore);

                                # bump up item record score
                                if (isset($Scores[$ItemId]))
                                {
                                    $Scores[$ItemId] += $PhraseScore;
                                }
                                else
                                {
                                    $Scores[$ItemId] = $PhraseScore;
                                }

                                # if phrase flagged as required
                                if ($Flags & WORD_REQUIRED)
                                {
                                    # increment required word count for record
                                    if (isset($this->RequiredTermCounts[$ItemId]))
                                    {
                                        $this->RequiredTermCounts[$ItemId]++;
                                    }
                                    else
                                    {
                                        $this->RequiredTermCounts[$ItemId] = 1;
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }

        # return updated scores to caller
        return $Scores;
    }

    private function FilterOnExcludedWords($Words, $Scores, $FieldName = "XXXKeywordXXX")
    {
        $DB = $this->DB;

        # grab field ID
        $FieldId = $this->GetFieldId($FieldName);

        # for each word
        foreach ($Words as $Word => $Flags)
        {
            # if word flagged as excluded
            if ($Flags & WORD_EXCLUDED)
            {
                # look up record ID for word
                $WordId = $this->GetWordId($Word);

                # if word is in DB
                if ($WordId !== NULL)
                {
                    # look up counts for word
                    $DB->Query("SELECT ItemId FROM SearchWordCounts "
                            ."WHERE WordId=${WordId} AND FieldId=${FieldId}");

                    # for each count
                    while ($Record = $DB->FetchRow())
                    {
                        # if item record is in score list
                        $ItemId = $Record["ItemId"];
                        if (isset($Scores[$ItemId]))
                        {
                            # remove item record from score list
                            $this->DMsg(3, "Filtering out item ".$ItemId
                                    ." because it contained word \"".$Word."\"");
                            unset($Scores[$ItemId]);
                        }
                    }
                }
            }
        }

        # returned filtered score list to caller
        return $Scores;
    }

    private function FilterOnRequiredWords($Scores)
    {
        # if there were required words
        if ($this->RequiredTermCount > 0)
        {
            # for each item
            foreach ($Scores as $ItemId => $Score)
            {
                # if item does not meet required word count
                if (!isset($this->RequiredTermCounts[$ItemId])
                        || ($this->RequiredTermCounts[$ItemId] < $this->RequiredTermCount))
                {
                    # filter out item
                    $this->DMsg(4, "Filtering out item ".$ItemId
                            ." because it didn't have required word count of "
                            .$this->RequiredTermCount
                            .(isset($this->RequiredTermCounts[$ItemId])
                                    ? " (only had "
                                    .$this->RequiredTermCounts[$ItemId]
                                    : " (had none")
                            .")");
                    unset($Scores[$ItemId]);
                }
            }
        }

        # return filtered list to caller
        return $Scores;
    }

    # count, sort, and trim search result scores list
    private function CleanScores($Scores, $StartingResult, $NumberOfResults,
            $SortByField, $SortDescending)
    {
        # perform any requested filtering
        $this->DMsg(0, "Have ".count($Scores)." results before filter callbacks");
        $Scores = $this->FilterOnSuppliedFunctions($Scores);

        # save total number of results available
        $this->NumberOfResultsAvailable = count($Scores);

        # if no sorting field specified
        if ($SortByField === NULL)
        {
            # sort result list by score
            if ($SortDescending)
                arsort($Scores, SORT_NUMERIC);
            else
                asort($Scores, SORT_NUMERIC);
        }
        else
        {
            # get list of item IDs in sorted order
            $SortedIds = $this->GetItemIdsSortedByField(
                    $SortByField, $SortDescending);

            # if we have sorted item IDs
            if (count($SortedIds) && count($Scores))
            {
                # strip sorted ID list down to those that appear in search results
                $SortedIds = array_intersect($SortedIds, array_keys($Scores));

                # rebuild score list in sorted order
                foreach ($SortedIds as $Id)
                {
                    $NewScores[$Id] = $Scores[$Id];
                }
                $Scores = $NewScores;
            }
            else
            {
                # sort result list by score
                arsort($Scores, SORT_NUMERIC);
            }
        }

        # trim result list to match range requested by caller
        $ScoresKeys = array_slice(
                array_keys($Scores), $StartingResult, $NumberOfResults);
        $TrimmedScores = array();
        foreach ($ScoresKeys as $Key) {  $TrimmedScores[$Key] = $Scores[$Key];  }

        # returned cleaned search result scores list to caller
        return $TrimmedScores;
    }

    protected function FilterOnSuppliedFunctions($Scores)
    {
        # if filter functions have been set
        if (isset($this->FilterFuncs))
        {
            # for each result
            foreach ($Scores as $ItemId => $Score)
            {
                # for each filter function
                foreach ($this->FilterFuncs as $FuncName)
                {
                    # if filter function return TRUE for item
                    if ($FuncName($ItemId))
                    {
                        # discard result
                        $this->DMsg(2, "Filter callback <i>".$FuncName
                                ."</i> rejected item ".$ItemId);
                        unset($Scores[$ItemId]);

                        # bail out of filter func loop
                        continue 2;
                    }
                }
            }
        }

        # return filtered list to caller
        return $Scores;
    }

    private function SearchForComparisonMatches($SearchStrings, $Scores)
    {
        # for each field
        $Index = 0;
        foreach ($SearchStrings as $SearchFieldName => $SearchStringArray)
        {
            # if field is not keyword
            if ($SearchFieldName != "XXXKeywordXXX")
            {
                # convert search string to array if needed
                if (!is_array($SearchStringArray))
                {
                    $SearchStringArray = array($SearchStringArray);
                }

                # for each search string for this field
                foreach ($SearchStringArray as $SearchString)
                {
                    # if search string looks like comparison search
                    $FoundOperator = preg_match("/^[><!]=./", $SearchString)
                            || preg_match("/^[><=]./", $SearchString);
                    if ($FoundOperator
                            || (isset($this->FieldInfo[$SearchFieldName]["FieldType"])
                            && ($this->FieldInfo[$SearchFieldName]["FieldType"]
                                    != self::FIELDTYPE_TEXT)))
                    {
                        # determine value
                        $Patterns = array("/^[><!]=/", "/^[><=]/");
                        $Replacements = array("", "");
                        $Value = trim(preg_replace($Patterns, $Replacements, $SearchString));

                        # determine and save operator
                        if (!$FoundOperator)
                        {
                            $Operators[$Index] = "=";
                        }
                        else
                        {
                            $Term = trim($SearchString);
                            $FirstChar = $Term{0};
                            $FirstTwoChars = $FirstChar.$Term{1};
                            if ($FirstTwoChars == ">=")     {  $Operators[$Index] = ">=";  }
                            elseif ($FirstTwoChars == "<=") {  $Operators[$Index] = "<=";  }
                            elseif ($FirstTwoChars == "!=") {  $Operators[$Index] = "!=";  }
                            elseif ($FirstChar == ">")      {  $Operators[$Index] = ">";  }
                            elseif ($FirstChar == "<")      {  $Operators[$Index] = "<";  }
                            elseif ($FirstChar == "=")      {  $Operators[$Index] = "=";  }
                        }

                        # if operator was found
                        if (isset($Operators[$Index]))
                        {
                            # save value
                            $Values[$Index] = $Value;

                            # save field name
                            $FieldNames[$Index] = $SearchFieldName;
                            $this->DMsg(3, "Added comparison (field = <i>"
                                    .$FieldNames[$Index]."</i>  op = <i>"
                                    .$Operators[$Index]."</i>  val = <i>"
                                    .$Values[$Index]."</i>)");

                            # move to next comparison array entry
                            $Index++;
                        }
                    }
                }
            }
        }

        # if comparisons found
        if (isset($Operators))
        {
            # perform comparisons on fields and gather results
            $Results = $this->SearchFieldsForComparisonMatches($FieldNames, $Operators, $Values);

            # if search logic is set to AND
            if ($this->DefaultSearchLogic == self::LOGIC_AND)
            {
                # if results were found
                if (count($Results))
                {
                    # if there were no prior results and no terms for keyword search
                    if ((count($Scores) == 0) && ($this->InclusiveTermCount == 0))
                    {
                        # add all results to scores
                        foreach ($Results as $ItemId)
                        {
                            $Scores[$ItemId] = 1;
                        }
                    }
                    else
                    {
                        # remove anything from scores that is not part of results
                        foreach ($Scores as $ItemId => $Score)
                        {
                            if (in_array($ItemId, $Results) == FALSE)
                            {
                                unset($Scores[$ItemId]);
                            }
                        }
                    }
                }
                else
                {
                    # clear scores
                    $Scores = array();
                }
            }
            else
            {
                # add result items to scores
                if ($Scores === NULL) {  $Scores = array();  }
                foreach ($Results as $ItemId)
                {
                    if (isset($Scores[$ItemId]))
                    {
                        $Scores[$ItemId] += 1;
                    }
                    else
                    {
                        $Scores[$ItemId] = 1;
                    }
                }
            }
        }

        # return results to caller
        return $Scores;
    }

    private function SetDebugLevel($SearchStrings)
    {
        # if search info is an array
        if (is_array($SearchStrings))
        {
            # for each array element
            foreach ($SearchStrings as $FieldName => $SearchStringArray)
            {
                # if element is an array
                if (is_array($SearchStringArray))
                {
                    # for each array element
                    foreach ($SearchStringArray as $Index => $SearchString)
                    {
                        # pull out search string if present
                        $SearchStrings[$FieldName][$Index] = $this->ExtractDebugLevel($SearchString);
                    }
                }
                else
                {
                    # pull out search string if present
                    $SearchStrings[$FieldName] = $this->ExtractDebugLevel($SearchStringArray);
                }
            }
        }
        else
        {
            # pull out search string if present
            $SearchStrings = $this->ExtractDebugLevel($SearchStrings);
        }

        # return new search info to caller
        return $SearchStrings;
    }

    private function ExtractDebugLevel($SearchString)
    {
        # if search string contains debug level indicator
        if (strstr($SearchString, "DBUGLVL="))
        {
            # remove indicator and set debug level
            $Level = preg_replace("/^\\s*DBUGLVL=([1-9]{1,2}).*/", "\\1", $SearchString);
            if ($Level > 0)
            {
                $this->DebugLevel = $Level;
                $this->DMsg(0, "Setting debug level to ".$Level);
                $SearchString = preg_replace("/DBUGLVL=${Level}/", "", $SearchString);
            }
        }

        # return (possibly) modified search string to caller
        return $SearchString;
    }

    # load and return search result scores array containing all possible records
    private function LoadScoresForAllRecords()
    {
        # start with empty list
        $Scores = array();

        # for every item
        $this->DB->Query("SELECT ".$this->ItemIdFieldName
                         ." FROM ".$this->ItemTableName);
        while ($Record = $this->DB->FetchRow())
        {
            # set score for item to 1
            $Scores[$Record[$this->ItemIdFieldName]] = 1;
        }

        # return array with all scores to caller
        return $Scores;
    }


    # ---- private functions used in building search database

    /**
    * Update weighted count for term/item/field combination in DB.
    * @param Word Word for which count should be updated.
    * @param ItemId ID of item for which count applies.
    * @param FieldId ID of field for which count applies.
    * @param Weight Numeric weight to apply to count.  (OPTIONAL - defaults to 1)
    */
    private function UpdateWordCount($Word, $ItemId, $FieldId, $Weight = 1)
    {
        # retrieve ID for word
        $WordIds[] = $this->GetWordId($Word, TRUE);

        # if stemming is enabled
        if ($this->StemmingEnabled)
        {
            # retrieve ID for stem of word
            $Stem = PorterStemmer::Stem($Word, TRUE);
            $WordIds[] = $this->GetStemId($Stem, TRUE);
        }

        # for word and stem of word
        foreach ($WordIds as $WordId)
        {
            # if word count already added to database
            if (isset($this->WordCountAdded[$WordId][$FieldId]))
            {
                # update word count
                $this->DB->Query("UPDATE SearchWordCounts SET Count=Count+".$Weight
                        ." WHERE WordId=".$WordId
                                ." AND ItemId=".$ItemId
                                ." AND FieldId=".$FieldId);
            }
            else
            {
                # add word count to DB
                $this->DB->Query("INSERT INTO SearchWordCounts"
                        ." (WordId, ItemId, FieldId, Count) VALUES"
                        ." (".$WordId.", ".$ItemId.", ".$FieldId.", ".$Weight.")");

                # remember that we added count for this word
                $this->WordCountAdded[$WordId][$FieldId] = TRUE;
            }

            # decrease weight for stem
            $Weight = ceil($Weight / 2);
        }
    }

    protected function GetFieldContent($ItemId, $FieldName)
    {
        # error out
        exit("<br>SE - ERROR: GetFieldContent() not implemented<br>\n");
    }

    private function RecordSearchInfoForText(
            $ItemId, $FieldName, $Weight, $Text, $IncludeInKeyword)
    {
        # normalize text
        $Words = $this->ParseSearchStringForWords($Text, TRUE);

        # if there was text left after parsing
        if (count($Words) > 0)
        {
            # get ID for field
            $FieldId = $this->GetFieldId($FieldName);

            # if text should be included in keyword searches
            if ($IncludeInKeyword)
            {
                # get ID for keyword field
                $KeywordFieldId = $this->GetFieldId("XXXKeywordXXX");
            }

            # for each word
            foreach ($Words as $Word => $Flags)
            {
                # update count for word
                $this->UpdateWordCount($Word, $ItemId, $FieldId);

                # if text should be included in keyword searches
                if ($IncludeInKeyword)
                {
                    # update keyword field count for word
                    $this->UpdateWordCount(
                            $Word, $ItemId, $KeywordFieldId, $Weight);
                }
            }
        }
    }

    # print debug message if level set high enough
    protected function DMsg($Level, $Msg)
    {
        if ($this->DebugLevel > $Level)
        {
            print("SE:  ".$Msg."<br>\n");
        }
    }

    # ---- BACKWARD COMPATIBILITY --------------------------------------------

    # possible types of logical operators
    const SEARCHLOGIC_AND = 1;
    const SEARCHLOGIC_OR = 2;
}

?>
