<?php
/**
 * @category 	Atmail
 * @package 	Models
 * @subpackage	Search Lucene
 * @copyright 	Copyright (c) 2009-2011 ATMAIL. All rights reserved
 * @license 	See http://atmail.com/license.php for license agreement
 * @author		Allan Wrethman allan@staff.atmail.com
 * @author		Atmail (http://atmail.com)
 */
class searchLucene
{
	private $_messageStore;
	//private $_rootIndexFolder = APP_ROOT . DIRECTORY_SEPARATOR . 'archive' . DIRECTORY_SEPARATOR . 'lucene' . DIRECTORY_SEPARATOR;
	private $_indexSubFolder = '';
	private $_indexName = 'luceneIndex';
	
	/*
		RATIONALE: generic (portable, reusable) set of functions 
		depends on standard Zend Framework libraries so must be set up properly
		add an item into a lucene index (create index if does not exist)
		search an index for a match
		delete stale
		optimize 
		
		
		(authorization must be handled outside, i.e. no auth done in here)
        
	*/
	
	public function construct(Atmail_Mail_Storage_Writable_Maildir $messageStore, $rootIndexFolder = null, $indexSubfolder = null, $indexName = null)
	{
		$this->_messageStore = $messageStore;
		
		if( isset($rootIndexFolder) )
			$this->_rootIndexFolder = $rootIndexFolder;
		if( !file_exists($this->_rootIndexFolder) )
			throw new Atmail_Exception('unable to find root index folder: ' . $this->_rootIndexFolder);
		
		if( isset($indexSubfolder) )
			$this->_indexSubFolder = $indexSubfolder;
		
		if( isset($indexName) )
			$this->_indexName = $indexName;
	}



	private function deleteStale()
	{
		$startTime = microtime(true);
		$log = Zend_Registry::get('log');
		$indexPath = $this->_rootIndexFolder . $this->_indexSubFolder . $this->_indexName; 
        try {
			$index = Zend_Search_Lucene::open($indexPath); 
		} catch (Zend_Search_Lucene_Exception $e) {
		    $index = Zend_Search_Lucene::create($indexPath);
			$log->info('Index did non exist so created: ' . $indexPath);
		}
        $log->info('Removing stale entries out of index: ' . $indexPath);

		//go through Index looking for documents that no longer exist, continue if found else delete from index
		for ($indexId = 0; $indexId < $index->maxDoc(); $indexId++) { 
			if($this->_messageStore->hasUniquiId($index->getDocument($indexId)->getFieldValue('uniqueId')))
				continue;
			else
				$index->delete($indexId);
		}
	}

	private function optimizeCurrentIndex()
	{
		$indexPath = $this->_rootIndexFolder . $this->_indexSubFolder . $this->_indexName; 
		$log = Zend_Registry::get('log');
		$log->info('beginning index optimization.');
		try {
			$index = Zend_Search_Lucene::open($indexPath); 
		} catch (Zend_Search_Lucene_Exception $e) {
		    $index = Zend_Search_Lucene::create($indexPath);
			$log->info('Index did exist so created: ' . $indexPath);
			return;
		}
		$index->optimize();
	}   

	public function indexCurrentFolder()
	{
		$resetTimeLimit = 300;
		$messagesToIndexPerPass = 200;
		$cleanupTimeNeeded = 20;
		$startTime = microtime(true);
		$log = Zend_Registry::get('log');
		$indexPath = $this->_rootIndexFolder . $this->_indexSubFolder . $this->_indexName; 
		try {
			$index = Zend_Search_Lucene::open($indexPath); 
		} catch (Zend_Search_Lucene_Exception $e) {
		    $index = Zend_Search_Lucene::create($indexPath);
			$log->info('Index did exist so created: ' . $indexPath);
		}
		$log->info('Pre-optimizing: ' . $indexPath);
		$index->optimize(); //optomize index before processing to attempt to avoid corrupted index
        $log->info('Skipping already indexed messages');
		$messagesIndexed = 0;
		$messagesC = $this->countMessages();
		for( $a=0 ; $a<$messagesC ; $a++ )
		{
			$currentMessageNumber = $a;
			$message = $this->getMessage($this->_messageStore->_files[$a]['uniq']);
			set_time_limit($resetTimeLimit);
			$timerStart = microtime(true);
			if( $messagesIndexed > $messagesToIndexPerPass )
			{
				$log->info($messagesIndexed . ' messages indexed into memory, beginning commit');
				$index->optimize();
				$log->info('Commit complete');
				break;
			}
			if( (microtime(true) - $startTime) > ($resetTimeLimit - $messagesToIndexPerPass))
			{
				$log->info('About to run out of time so cleaning up and processing index');
				break;
			}
			$uniqueId = $this->_messageStore->_files[$a]['uniq'];
			
			if($index->termDocs(new Zend_Search_Lucene_Index_Term($uniqueId, 'uniqueId')))
				continue;

   			$log->info('About to index: ' . $this->getCurrentFolder() . ': ' . $uniqueId);
			$headers = $message->processHeaders();
			// if message larger than available time to process then commit ready to start with beg message next pass
			//$viableSecondsLeft = 60 - (microtime(true) - $startTime);
			//if(($headers['sizeraw']/1000) > $viableSecondsLeft*100)
			//{
				//TODO: consider skipping messages too large to index, or store them with cruncated content;
			//	$log->info('Message too big (' . $headers['size'] . ') to handle in time left (' . round($viableSecondsLeft) . ') so deferring to next pass and comitting now.');
			//	break;
			//}
			$bodyContents = '';
			$attachmentContents = '';
			$attachmentNames = '';
			$partsList = $message->recurseParts(null,0);
			foreach($partsList AS $part)
			{
				$isInline = true;
				$partHeaders = $part['part']->getHeaders();
				if(
					isset($partHeaders['content-type']) 
					&& 
					( 
						!(
							stripos($partHeaders['content-type'],'filename') === false 
							&& 
							stripos($partHeaders['content-type'],'name') === false
						) 
					) 
				)
				{
					$isInline = false; // inline body parts will not have a file name, only attachments have a filename
					if(preg_match('/ame="(.+?)"/', $partHeaders['content-type'], $matches))
					{
						$filename = $matches[1];
						$attachmentNames .= $filename . ' ';
					}
				}
				//defined part name can be called name, Name, Filename
				//TODO: indexing of certain types of attachments not just html and text
				$fileNameExtension = '.notindexable';
				if(isset($partHeaders['content-type']))
				{
					if(strpos($partHeaders['content-type'], 'text/html') === 0)
					{
						include_once('class.html2text.inc');
						//TODO: consider converting character type to utf-8 before processing
						$converter = new html2text($part['part']->getContent());
						$content = quoted_printable_decode($converter->get_text());
						//$content = $converter->get_text();
						if($isInline)
							$bodyContents .= $content;
						else
							$attachmentContents .= $content;
					}
					elseif(strpos($partHeaders['content-type'], 'text/plain') === 0)
					{	
						$content = quoted_printable_decode($part['part']->getContent());
						if($isInline)
							$bodyContents .= $content;
						else
							$attachmentContents .= $content;
					}
					elseif(stripos($partHeaders['content-type'], 'text/pdf') !== false || $fileNameExtension == 'pdf')
					{
						//shell of elseif for handling other content type indexing
					}
				}	
				else
				{
					//normally if content-type not set it is because it is a multipart container for other parts
					//possible very old email format with not multipart, then inline
					//echo '<font color="red">content-type: <b>Not Set</b></font><br />';
					//$bodyContents .= $content;
					//echo $part['part']->getContent();
				}

			}
			$doc = new Zend_Search_Lucene_Document(); 
			$doc->addField(Zend_Search_Lucene_Field::Keyword('uniqueId', $uniqueId));        
			$doc->addField(Zend_Search_Lucene_Field::Text('from', utf8_encode($headers['from']), 'utf-8')); 
			$doc->addField(Zend_Search_Lucene_Field::Text('to', utf8_encode($headers['to']), 'utf-8')); 
			$doc->addField(Zend_Search_Lucene_Field::Text('subject', utf8_encode($headers['subject']), 'utf-8')); 
			$doc->addField(Zend_Search_Lucene_Field::Keyword('epoch', $headers['epoch'])); 
			$doc->addField(Zend_Search_Lucene_Field::Keyword('sizeraw', $headers['sizeraw'])); 
			$doc->addField(Zend_Search_Lucene_Field::Unstored('bodyContents', utf8_encode($bodyContents), 'utf-8')); 
			$doc->addField(Zend_Search_Lucene_Field::Unstored('attachmentContents', utf8_encode($attachmentContents), 'utf-8' )); 
    		$doc->addField(Zend_Search_Lucene_Field::Text('attachmentNames', utf8_encode($attachmentNames), 'utf-8' )); //Is this really useful or should we drop 
			//$log->info(round((microtime(true)-$timerStart),0) . ' seconds to prep document for index: ' . $uniqueId);
			//$timerStart = microtime(true);
			$index->addDocument($doc);
			//$log->info(round((microtime(true)-$timerStart),0) . ' seconds to add document to index: ' . $uniqueId);
			$log->info('Added to index: ' . $this->getCurrentFolder() . ': ' . $uniqueId);
        	$messagesIndexed++;
		}
		//$indexSize = $index->count(); 
		//$documents = $index->numDocs();
		$log->info('Finished indexing: ' . $this->getCurrentFolder());
		//echo "Index currently contains: " . $indexSize . ' documents, ' . $documents . ' not deleted<br />';
	}

	public function search( $searchArray )
	{
		// folder/(possible any resource in future) (only first record folder used)
		// field = list of accepted fields (currently: from, to, subject, epoch, sizeraw, flag, bodyContent)
		// modifier = exactly, contains, beginswith, endswith, greaterthan, lessthan, not (default = contains) (modifiers may be field type specific)
		// compoundWithNext = and, or (default or, and only if there is a next record)
	    // value = value 
		$indexPath = $this->_rootIndexFolder . $this->_indexSubFolder . $this->_indexName; 
		
		$results = array();
		$resultsIndex = 0;
			try {
				$index = Zend_Search_Lucene::open($indexPath); 
				$query = new Zend_Search_Lucene_Search_Query_Boolean(); 
			 	foreach($searchArray as $searchTerm) {
					$term = new Zend_Search_Lucene_Index_Term('*' . strtolower($searchTerm['value']) . '*', $searchTerm['field']); 
					$subquery = new Zend_Search_Lucene_Search_Query_Wildcard($term); 
					$query->addSubquery($subquery, true); 
				} 
				try {
					$hits  = $index->find($query);
				} catch (Zend_Search_Lucene_Search_QueryParserException $e) { 
				    echo "Query syntax error: " . $e->getMessage() . "\n"; 
				}
				foreach($hits AS $hit) {
					$results['score'][$resultsIndex] = $hit->score;
					$results['uniqueId'][$resultsIndex] = $hit->uniqueId;
					$results['to'][$resultsIndex] = $hit->to;
					$results['from'][$resultsIndex] = $hit->from;
					$results['subject'][$resultsIndex] = $hit->subject;
					$results['epoch'][$resultsIndex] = $hit->epoch;
					$results['sizeraw'][$resultsIndex++] = $hit->sizeraw;
				}  
			} catch (Zend_Search_Lucene_Exception $e) {
			    Zend_Registry::get('log')->info('failed searching index: ' . $indexPath);
			}

		//sort results
		if( isset($results['epoch']) && count($results['epoch']) > 1)
		array_multisort(
			$results['epoch'], SORT_NUMERIC, SORT_DESC, 
			$results['score'], 
			$results['uniqueId'], 
			$results['from'], 
			$results['to'], 
			$results['subject'], 
			$results['sizeraw']
		);
		//normalize array for fluent access
		$searchResults = array();
		foreach( $results AS $key => $valueArray )
			for( $a=0 ; $a<$resultsIndex ; $a++ )
				$searchResults[$a][$key] = $results[$key][$a];
	    return $searchResults;  
	}
   
	/* other code snippets we will prob need as we flesh out the app */
	//$query = Zend_Search_Lucene_Search_QueryParser::parse($queryString); // string search syntax     
	//$hits  = $index->find($query, 'epoch', SORT_NUMERIC, SORT_DESC); //sort syntax
	//$highlightedHTML = $query->highlightMatches($sourceHTML);
	//$query = new Zend_Search_Lucene_Search_Query_MultiTerm(); 
	//new Zend_Search_Lucene_Search_Query_Wildcard($pattern);
	//$subquery1 = new Zend_Search_Lucene_Search_Query_Wildcard($pattern); 
	//$subquery2 = new Zend_Search_Lucene_Search_Query_MultiTerm(); 
	//$subquery2->addTerm(new Zend_Search_Lucene_Index_Term('word4', 'author')); 
	//$subquery2->addTerm(new Zend_Search_Lucene_Index_Term('word5', 'author')); 
	//$subquery3 = new Zend_Search_Lucene_Search_Query_Term(new Zend_Search_Lucene_Index_Term('word6')); 
	//$query->addSubquery($subquery1, true  /* required */); 
	//$query->addSubquery($subquery2, null  /* optional */); 
	//$query->addSubquery($subquery3, false /* prohibited */); 
	//$hits  = $index->find($query); 
	//$term = new Zend_Search_Lucene_Index_Term('' . $value . '', $requestParams['field'][$key]);
	//$query->addTerm($term, true);
	//$query->addTerm(new Zend_Search_Lucene_Index_Term('word1'), true); 
	//$query->addTerm(new Zend_Search_Lucene_Index_Term('word2', 'author'), null); 
	//$query->addTerm(new Zend_Search_Lucene_Index_Term('word3'), false); 
	//limit result set
	//$currentResultSetLimit = Zend_Search_Lucene::getResultSetLimit(); 
	//echo '$currentResultSetLimit: ' . $currentResultSetLimit;
	//Zend_Search_Lucene::setResultSetLimit(0);
}                                               

