<?PHP

#
#   FILE:  SPT--Recommender.php
#
#   METHODS PROVIDED:
#       Recommender()
#           - constructor
#       SomeMethod($SomeParameter, $AnotherParameter)
#           - short description of method
#
#   AUTHOR:  Edward Almasy
#
#   Part of the Scout Portal Toolkit
#   Copyright 2002-2004 Internet Scout Project
#   http://scout.wisc.edu
#

class Recommender {

    # ---- PUBLIC INTERFACE --------------------------------------------------
    # define content field types
    const CONTENTFIELDTYPE_TEXT =  1;
    const CONTENTFIELDTYPE_NUMERIC =  2;
    const CONTENTFIELDTYPE_CONTROLLEDNAME =  3;
    const CONTENTFIELDTYPE_DATE =  4;
    const CONTENTFIELDTYPE_DATERAMGE =  5;

    # object constructor
    function Recommender(&$DB, $ItemTableName, $RatingTableName, 
            $ItemIdFieldName, $UserIdFieldName, $RatingFieldName,
            $ContentFields)
    {
        # set default parameters
        $this->ContentCorrelationThreshold = 1;

        # save database object
        $this->DB =& $DB;

        # save new configuration values
        $this->ItemTableName = $ItemTableName;
        $this->RatingTableName = $RatingTableName;
        $this->ItemIdFieldName = $ItemIdFieldName;
        $this->UserIdFieldName = $UserIdFieldName;
        $this->RatingFieldName = $RatingFieldName;
        $this->ContentFields = $ContentFields;

        # set default debug state
        $this->DebugLevel = 0;
    }

    # set level for debugging output
    function DebugLevel($Setting)
    {
        $this->DebugLevel = $Setting;
    }


    # ---- recommendation methods

    # recommend items for specified user
    function Recommend($UserId, $StartingResult = 0, $NumberOfResults = 10)
    {
        if ($this->DebugLevel > 0) {  print("REC:  Recommend(${UserId}, ${StartingResult}, ${NumberOfResults})<br>\n");  }

        # load in user ratings
        $Ratings = array();
        $DB =& $this->DB;
        $DB->Query("SELECT ".$this->ItemIdFieldName.", ".$this->RatingFieldName
                ." FROM ".$this->RatingTableName
                ." WHERE ".$this->UserIdFieldName." = ${UserId}");
        while ($Row = $DB->FetchRow())
        {
            $Ratings[$Row[$this->ItemIdFieldName]] = 
                    $Row[$this->RatingFieldName];
        }
        if ($this->DebugLevel > 1) {  print("REC:  user has rated ".count($Ratings)." items<br>\n");  }

        # for each item that user has rated
        $RecVals = array();
        foreach ($Ratings as $ItemId => $ItemRating)
        {
            # for each content correlation available for that item
            $DB->Query("SELECT Correlation, ItemIdB "
                    ."FROM RecContentCorrelations "
                    ."WHERE ItemIdA = ${ItemId}");
            while ($Row = $DB->FetchRow())
            {
                # multiply that correlation by normalized rating and add
                #       resulting value to recommendation value for that item
                if (isset($RecVals[$Row["ItemIdB"]]))
                {
                    $RecVals[$Row["ItemIdB"]] +=
                            $Row["Correlation"] * ($ItemRating - 50);
                }
                else
                {
                    $RecVals[$Row["ItemIdB"]] =
                            $Row["Correlation"] * ($ItemRating - 50);
                }
                if ($this->DebugLevel > 9) {  print("REC:  RecVal[".$Row["ItemIdB"]."] = ".$RecVals[$Row["ItemIdB"]]."<br>\n");  }
            }
        }
        if ($this->DebugLevel > 1) {  print("REC:  found ".count($RecVals)." total recommendations<br>\n");  }

        # calculate average correlation between items
        $ResultThreshold = $DB->Query("SELECT AVG(Correlation) "
                ."AS Average FROM RecContentCorrelations", "Average");
        $ResultThreshold = round($ResultThreshold) * 2;

        # for each recommended item
        foreach ($RecVals as $ItemId => $RecVal)
        {
            # remove item from list if user already rated it
            if (isset($Ratings[$ItemId]))
            {
                unset($RecVals[$ItemId]);  
            }
            else
            {
                # scale recommendation value back to match thresholds
                $RecVals[$ItemId] = round($RecVal / 50);

                # remove item from recommendation list if value is below threshold
                if ($RecVals[$ItemId] < $ResultThreshold)
                {  
                    unset($RecVals[$ItemId]);  
                }
            }
        }
        if ($this->DebugLevel > 1) {  print("REC:  found ".count($RecVals)." positive recommendations<br>\n");  }

        # sort recommendation list by value
        if (isset($RecVals)) {  arsort($RecVals, SORT_NUMERIC);  }

        # save total number of results available
        $this->NumberOfResultsAvailable = count($RecVals);

        # trim result list to match range requested by caller
        $RecValKeys = array_slice(
                array_keys($RecVals), $StartingResult, $NumberOfResults);
        $RecValSegment = array();
        foreach ($RecValKeys as $Key) 
        {  
            $RecValSegment[$Key] = $RecVals[$Key];  
        }

        # return recommendation list to caller
        return $RecValSegment;
    }

    # add function to be called to filter returned recommendation list
    function AddResultFilterFunction($FunctionName)
    {
        # save filter function name
        $this->FilterFuncs[] = $FunctionName;
    }

    # return number of recommendations generated
    function NumberOfResults()
    {
        return $this->NumberOfResultsAvailable;
    }

    # return recommendation generation time
    function SearchTime()
    {
        return $this->LastSearchTime;
    }

    # return list of items used to generate recommendation of specified item
    function GetSourceList($UserId, $RecommendedItemId)
    {
        # pull list of correlations from DB
        $this->DB->Query("SELECT * FROM RecContentCorrelations, ".$this->RatingTableName
                ." WHERE (ItemIdA = ${RecommendedItemId}"
                        ." OR ItemIdB = ${RecommendedItemId})"
                        ." AND ".$this->UserIdFieldName." = ".$UserId
                        ." AND (RecContentCorrelations.ItemIdA = ".$this->RatingTableName.".".$this->ItemIdFieldName
                        ." OR RecContentCorrelations.ItemIdB = ".$this->RatingTableName.".".$this->ItemIdFieldName.")"
                        ." AND Rating >= 50 "
                ." ORDER BY Correlation DESC");

        # for each correlation
        $SourceList = array();
        while ($Row = $this->DB->FetchRow())
        {
            # pick out appropriate item ID
            if ($Row["ItemIdA"] == $RecommendedItemId)
            {
                $ItemId = $Row["ItemIdB"];
            }
            else
            {
                $ItemId = $Row["ItemIdA"];
            }

            # add item to recommendation source list
            $SourceList[$ItemId] = $Row["Correlation"];
        }

        # return recommendation source list to caller
        return $SourceList;
    }

    # dynamically generate and return list of items similar to specified item
    function FindSimilarItems($ItemId, $FieldList = NULL)
    {
        if ($this->DebugLevel > 1) {  print("REC:  searching for items similar to item \"".$ItemId."\"<br>\n");  }

        # make sure we have item IDs available
        $this->LoadItemIds();

        # start with empty array
        $SimilarItems = array();

        # for every item
        foreach ($this->ItemIds as $Id)
        {
            # if item is not specified item
            if ($Id != $ItemId)
            {
                # calculate correlation of item to specified item
                $Correlation = $this->CalculateContentCorrelation($ItemId, $Id, $FieldList);

                # if correlation is above threshold
                if ($Correlation > $this->ContentCorrelationThreshold)
                {
                    # add item to list of similar items
                    $SimilarItems[$Id] = $Correlation;
                }
            }
        }
        if ($this->DebugLevel > 3) {  print("REC:  ".count($SimilarItems)." similar items to item \"".$ItemId."\" found<br>\n");  }

        # filter list of similar items (if any)
        if (count($SimilarItems) > 0)
        {
            $SimilarItems = $this->FilterOnSuppliedFunctions($SimilarItems);
            if ($this->DebugLevel > 4) {  print("REC:  ".count($SimilarItems)." similar items to item \"".$ItemId."\" left after filtering<br>\n");  }
        }
        
        # if any similar items left
        if (count($SimilarItems) > 0)
        {
            # sort list of similar items in order of most to least similar
            arsort($SimilarItems, SORT_NUMERIC);
        }

        # return list of similar items to caller
        return $SimilarItems;
    }

    # dynamically generate and return list of recommended field values for item
    function RecommendFieldValues($ItemId, $FieldList = NULL)
    {
        if ($this->DebugLevel > 1) {  print("REC:  generating field value recommendations for item \"".$ItemId."\"<br>\n");  }

        # start with empty array of values
        $RecVals = array();

        # generate list of similar items
        $SimilarItems = $this->FindSimilarItems($ItemId, $FieldList);
        
        # if similar items found
        if (count($SimilarItems) > 0)
        {
            # prune list of similar items to only top third of better-than-average
            $AverageCorr = intval(array_sum($SimilarItems) / count($SimilarItems));
            reset($SimilarItems);
            $HighestCorr = current($SimilarItems);
            $CorrThreshold = intval($HighestCorr - (($HighestCorr - $AverageCorr) / 3));
            if ($this->DebugLevel > 8) {  print("REC:  <i>Average Correlation: $AverageCorr &nbsp;&nbsp;&nbsp;&nbsp; Highest Correlation: $HighestCorr &nbsp;&nbsp;&nbsp;&nbsp; Correlation Threshold: $CorrThreshold </i><br>\n");  }
            foreach ($SimilarItems as $ItemId => $ItemCorr)
            {
                if ($ItemCorr < $CorrThreshold)
                {
                    unset($SimilarItems[$ItemId]);
                }
            }
            if ($this->DebugLevel > 6) {  print("REC:  ".count($SimilarItems)." similar items left after threshold pruning<br>\n");  }

            # for each item
            foreach ($SimilarItems as $SimItemId => $SimItemCorr)
            {
                # for each field
                foreach ($this->ContentFields as $FieldName => $FieldAttributes)
                {
                    # load field data for this item
                    $FieldData = $this->GetFieldValue($SimItemId, $FieldName);

                    # if field data is array
                    if (is_array($FieldData))
                    {
                        # for each field data value
                        foreach ($FieldData as $FieldDataVal)
                        {
                            # if data value is not empty
                            $FieldDataVal = trim($FieldDataVal);
                            if (strlen($FieldDataVal) > 0)
                            {
                                # increment count for data value
                                $RecVals[$FieldName][$FieldDataVal]++;
                            }
                        }
                    }
                    else
                    {
                        # if data value is not empty
                        $FieldData = trim($FieldData);
                        if (strlen($FieldData) > 0)
                        {
                            # increment count for data value
                            $RecVals[$FieldName][$FieldData]++;
                        }
                    }
                }
            }

            # for each field
            $MatchingCountThreshold = 3;
            foreach ($RecVals as $FieldName => $FieldVals)
            {
                # determine cutoff threshold
                arsort($FieldVals, SORT_NUMERIC);
                reset($FieldVals);
                $HighestCount = current($FieldVals);
                $AverageCount = intval(array_sum($FieldVals) / count($FieldVals));
                $CountThreshold = intval($AverageCount + (($HighestCount - $AverageCount) / 2));
                if ($CountThreshold < $MatchingCountThreshold) {  $CountThreshold = $MatchingCountThreshold;  }
                if ($this->DebugLevel > 8) {  print("REC:  <i>Field: $FieldName &nbsp;&nbsp;&nbsp;&nbsp;  Average Count: $AverageCount &nbsp;&nbsp;&nbsp;&nbsp; Highest Count: $HighestCount &nbsp;&nbsp;&nbsp;&nbsp; Count Threshold: $CountThreshold </i><br>\n");  }

                # for each field data value
                foreach ($FieldVals as $FieldVal => $FieldValCount)
                {
                    # if value count is below threshold
                    if ($FieldValCount < $CountThreshold)
                    {
                        # unset value
                        unset($RecVals[$FieldName][$FieldVal]);
                    }
                }

                if ($this->DebugLevel > 3) {  print("REC:  found ".count($RecVals[$FieldName])." recommended values for field \"".$FieldName."\" after threshold pruning<br>\n");  }
            }
        }

        # return recommended values to caller
        return $RecVals;
    }


    # ---- database update methods

    function UpdateForItems($StartingItemId, $NumberOfItems)
    {
        if ($this->DebugLevel > 0) {  print("REC:  UpdateForItems(${StartingItemId}, ${NumberOfItems})<br>\n");  }
        # make sure we have item IDs available
        $this->LoadItemIds();

        # for every item
        $ItemsUpdated = 0;
        $ItemId = NULL;
        foreach ($this->ItemIds as $ItemId)
        {
            # if item ID is within requested range
            if ($ItemId >= $StartingItemId)
            {
                # update recommender info for item
                if ($this->DebugLevel > 1) {  print("REC:  doing item ${ItemId}<br>\n");  }
                $this->UpdateForItem($ItemId, TRUE);
                $ItemsUpdated++;

                # if we have done requested number of items
                if ($ItemsUpdated >= $NumberOfItems)
                {
                    # bail out
                    if ($this->DebugLevel > 1) {  print("REC:  bailing out with item ${ItemId}<br>\n");  }
                    return $ItemId;
                }
            }
        }

        # return ID of last resource updated to caller
        return $ItemId;
    }

    function UpdateForItem($ItemId, $FullPass = FALSE)
    {   
        if ($this->DebugLevel > 1) {  print("REC:  updating for item \"".$ItemId."\"<br>\n");  }

        # make sure we have item IDs available
        $this->LoadItemIds();

        # clear existing correlations for this item
        $this->DB->Query("DELETE FROM RecContentCorrelations "
                ."WHERE ItemIdA = ${ItemId}");

        # for every item
        foreach ($this->ItemIds as $Id)
        {
            # if full pass and item is later in list than current item
            if (($FullPass == FALSE) || ($Id > $ItemId))
            {
                # update correlation value for item and target item
                $this->UpdateContentCorrelation($ItemId, $Id);
            }
        }
    }

    function DropItem($ItemId)
    {
        # drop all correlation entries referring to item
        $this->DB->Query("DELETE FROM RecContentCorrelations "
                         ."WHERE ItemIdA = ".$ItemId." "
                            ."OR ItemIdB = ".$ItemId);
    }

    function PruneCorrelations()
    {
        # get average correlation
        $AverageCorrelation = $this->DB->Query("SELECT AVG(Correlation) "
                ."AS Average FROM RecContentCorrelations", "Average");

        # dump all below-average correlations
        if ($AverageCorrelation > 0)
        {
            $this->DB->Query("DELETE FROM RecContentCorrelations "
                    ."WHERE Correlation <= ${AverageCorrelation}");
        }
    }

    /**
    * Retrieve all item IDs.
    * @return Array of item IDs.
    */
    function GetItemIds()
    {
        static $ItemIds;
        if (!isset($ItemIds))
        {
            $this->DB->Query("SELECT ".$this->ItemIdFieldName." AS Id FROM "
                    .$this->ItemTableName." ORDER BY ".$this->ItemIdFieldName);
            $ItemIds = $this->DB->FetchColumn("Id");
        }
        return $ItemIds;
    }


    # ---- PRIVATE INTERFACE -------------------------------------------------

    var $ContentCorrelationThreshold;
    var $ContentFields;
    var $ItemTableName;
    var $RatingTableName;
    var $ItemIdFieldName;
    var $UserIdFieldName;
    var $RatingFieldName;
    var $ItemIds;
    var $DB;
    var $FilterFuncs;
    var $LastSearchTime;
    var $NumberOfResultsAvailable;
    var $DebugLevel;


    function LoadItemIds()
    {
        # if item IDs not already loaded
        if (!isset($this->ItemIds))
        {
            # load item IDs from DB
            $this->DB->Query("SELECT ".$this->ItemIdFieldName." AS Id FROM "
                    .$this->ItemTableName." ORDER BY ".$this->ItemIdFieldName);
            $this->ItemIds = array();
            while ($Item = $this->DB->FetchRow())
            {
                $this->ItemIds[] = $Item["Id"];
            }
        }
    }

    function GetFieldData($ItemId, $FieldName)
    {
        static $ItemData;
        static $CachedItemList;

        # if data not already loaded
        if (!isset($ItemData[$ItemId][$FieldName]))
        {
            # load field value from DB
            $FieldValue = $this->GetFieldValue($ItemId, $FieldName);

            # if field value is array
            if (is_array($FieldValue))
            {
                # concatenate together text from array elements
                $FieldValue = implode(" ", $FieldValue);
            }

            # normalize text and break into word array
            $ItemData[$ItemId][$FieldName] = $this->NormalizeAndParseText($FieldValue);

            # if more items than cache limit
            if (count($ItemData) > 1000)
            {
                # dump oldest item
                reset($ItemData);
                list($DumpedItemId, $DumpedItemData) = each($ItemData);
                unset($ItemData[$DumpedItemId]);
            }
        }

        # return cached data to caller
        return $ItemData[$ItemId][$FieldName];
    }

    # calculate content correlation between two items and return value to caller
    function CalculateContentCorrelation($ItemIdA, $ItemIdB, $FieldList = NULL)
    {
        static $CorrelationCache;
        
        if ($this->DebugLevel > 10) {  print("REC:  calculating correlation between items $ItemIdA and $ItemIdB<br>\n");  }
        
        # order item ID numbers
        if ($ItemIdA > $ItemIdB)
        {
            $Temp = $ItemIdA;
            $ItemIdA = $ItemIdB;
            $ItemIdB = $Temp;
        }
        
        # if we already have the correlation
        if (isset($CorrelationCache[$ItemIdA][$ItemIdB]))
        {
            # retrieve correlation from cache
            $TotalCorrelation = $CorrelationCache[$ItemIdA][$ItemIdB];
        }
        else
        {
            # if list of fields to correlate specified
            if ($FieldList != NULL)
            {
                # create list with only specified fields
                foreach ($FieldList as $FieldName)
                {
                    $ContentFields[$FieldName] = $this->ContentFields[$FieldName];
                }
            }
            else
            {
                # use all fields
                $ContentFields = $this->ContentFields;
            }

            # for each content field
            $TotalCorrelation = 0;
            foreach ($ContentFields as $FieldName => $FieldAttributes)
            {
                # if field is of a type that we use for correlation
                $FieldType = intval($FieldAttributes["FieldType"]);
                if (($FieldType == Recommender::CONTENTFIELDTYPE_TEXT)
                        || ($FieldType == Recommender::CONTENTFIELDTYPE_CONTROLLEDNAME))
                {
                    # load data
                    $ItemAData = $this->GetFieldData($ItemIdA, $FieldName);
                    $ItemBData = $this->GetFieldData($ItemIdB, $FieldName);
                    if ($this->DebugLevel > 15) {  print("REC:  loaded ".count($ItemAData)." terms for item #".$ItemIdA." and ".count($ItemBData)." terms for item #".$ItemIdB." for field \"".$FieldName."\"<br>\n");  }

                    # call appropriate routine to get correlation
                    switch ($FieldType)
                    {
                        case Recommender::CONTENTFIELDTYPE_TEXT:
                        case Recommender::CONTENTFIELDTYPE_CONTROLLEDNAME:
                            $Correlation = $this->CalcTextCorrelation(
                                    $ItemAData, $ItemBData);
                            break;
                    }

                    # add correlation multiplied by weight to total
                    $TotalCorrelation += $Correlation * $FieldAttributes["Weight"];
                }
            }
            
            # store correlation to cache
            $CorrelationCache[$ItemIdA][$ItemIdB] = $TotalCorrelation;
        }

        # return correlation value to caller
        if ($this->DebugLevel > 9) {  print("REC:  correlation between items $ItemIdA and $ItemIdB found to be $TotalCorrelation<br>\n");  }
        return $TotalCorrelation;
    }

    # calculate content correlation between two items and update in DB
    function UpdateContentCorrelation($ItemIdA, $ItemIdB)
    {
        if ($this->DebugLevel > 6) {  print("REC:  updating correlation between items $ItemIdA and $ItemIdB<br>\n");  }

        # bail out if two items are the same
        if ($ItemIdA == $ItemIdB) {  return;  }

        # calculate correlation
        $Correlation = $this->CalculateContentCorrelation($ItemIdA, $ItemIdB);

        # save new correlation
        $this->ContentCorrelation($ItemIdA, $ItemIdB, $Correlation);
    }

    function NormalizeAndParseText($Text)
    {
        $StopWords = array(
                "a",
                "about",
                "also",
                "an",
                "and",
                "are",
                "as",
                "at",
                "be",
                "but",
                "by",
                "can",
                "each",
                "either",
                "for",
                "from",
                "has",
                "he",
                "her",
                "here",
                "hers",
                "him",
                "his",
                "how",
                "i",
                "if",
                "in",
                "include",
                "into",
                "is",
                "it",
                "its",
                "me",
                "neither",
                "no",
                "nor",
                "not",
                "of",
                "on",
                "or",
                "so",
                "she",
                "than",
                "that",
                "the",
                "their",
                "them",
                "then",
                "there",
                "these",
                "they",
                "this",
                "those",
                "through",
                "to",
                "too",
                "very",
                "what",
                "when",
                "where",
                "while",
                "who",
                "why",
                "will",
                "you",
                "");

        # strip any HTML tags
        $Text = strip_tags($Text);

        # strip any punctuation
        $Text = preg_replace("/,\\.\\?-\\(\\)\\[\\]\"/", " ", $Text);   # "

        # normalize whitespace
        $Text = trim(preg_replace("/[\\s]+/", " ", $Text));

        # convert to all lower case
        $Text = strtolower($Text);

        # split text into arrays of words
        $Words = explode(" ", $Text);

        # filter out all stop words
        $Words = array_diff($Words, $StopWords);

        # return word array to caller
        return $Words;
    }

    function CalcTextCorrelation($WordsA, $WordsB)
    {
        # get array containing intersection of two word arrays
        $IntersectWords = array_intersect($WordsA, $WordsB);

        # return number of words remaining as score
        return count($IntersectWords);
    }

    function ContentCorrelation($ItemIdA, $ItemIdB, $NewCorrelation = -1)
    {
        # if item ID A is greater than item ID B
        if ($ItemIdA > $ItemIdB)
        {
            # swap item IDs
            $Temp = $ItemIdA;
            $ItemIdA = $ItemIdB;
            $ItemIdB = $Temp;
        }

        # if new correlation value provided
        if ($NewCorrelation != -1)
        {
            # if new value is above threshold
            if ($NewCorrelation >= $this->ContentCorrelationThreshold)
            {
                # insert new correlation value in DB
                $this->DB->Query("INSERT INTO RecContentCorrelations "
                        ."(ItemIdA, ItemIdB, Correlation) "
                        ."VALUES (${ItemIdA}, ${ItemIdB}, ${NewCorrelation})");

                # return correlation value is new value
                $Correlation = $NewCorrelation;
            }
            # else
            else
            {
                # return value is zero
                $Correlation = 0;
            }
        }
        else
        {
            # retrieve correlation value from DB
            $Correlation = $this->DB->Query(
                    "SELECT Correlation FROM RecContentCorrelations "
                            ."WHERE ItemIdA = ${ItemIdA} AND ItemIdB = ${ItemIdB}",
                    "Correlation");

            # if no value found in DB
            if ($Correlation == FALSE)
            {
                # return value is zero
                $Correlation = 0;
            }
        }

        # return correlation value to caller
        return $Correlation;
    }

    function FilterOnSuppliedFunctions($Results)
    {
        # if filter functions have been set
        if (count($this->FilterFuncs) > 0)
        {
            # for each result
            foreach ($Results as $ResourceId => $Result)
            {
                # for each filter function
                foreach ($this->FilterFuncs as $FuncName)
                {
                    # if filter function return TRUE for result resource
                    if ($FuncName($ResourceId))
                    {
                        # discard result
                        if ($this->DebugLevel > 2) {  print("REC:      filter callback rejected resource ${ResourceId}<br>\n");  }
                        unset($Results[$ResourceId]);

                        # bail out of filter func loop
                        continue 2;
                    }
                }
            }
        }

        # return filtered list to caller
        return $Results;
    }
}

?>
