#!/s/std/bin/perl -w 
#
#  My WebInfo SearchEngine
#
#

package main;

require "ReadParse.pl";
use CGI qw(:standard);
use LWP::Simple;

# glob $x_min $y_min $x_max $y_max %searchparam
# glob 
##
my($search_num,$search_keyword);
@SearchEngines = 
    ( "EXCITE", "INFOSEEK");
$SearchEngineNum = 2;

print STDERR "-----------------------------------------------------------------------\n";
$|=1;

print  header, start_html("MetaSearchEngine"),"\n";
print "<center>";
print "</center>";
%searchparam = &ReadParse(*input);

$inTime=int time;
$dir=$searchparam{"uid"};
$searchAnew=0;

$x_clik = $y_clik = 0;
$x_clik = $searchparam{"map.x"};
$y_clik = $searchparam{"map.y"};

if($dir == 0){
    $searchAnew=1;
}
else {
    if(`ls /p/devise/demo2/MetaSearch/db/$dir/synch_infomgr` =~ /synch_infomgr/){
#	print STDERR "matched";
	open(synchFile,"/p/devise/demo2/MetaSearch/db/$dir/synch_infomgr");
	print synchFile "1\n";
	close synchFile;
    }
    else {
	$searchAnew=1;
    }
}

# buildSearchTables does CustomiseSessionFile
if($searchAnew){
    buildSearchTables(); 
}			  
else {    
    $x_min=$searchparam{"x_min"};
    $y_min=$searchparam{"y_min"};
    $x_max=$searchparam{"x_max"};
    $y_max=$searchparam{"y_max"};

    $SE1=$searchparam{"SE1"};
    $SE2=$searchparam{"SE2"};
    if(($x_clik!=0) || ($y_clik!=0)){
	ReturnURLInfo($x_clik,$y_clik);
	exit 0;
    }

    RecalculateXY();	
    
    #only item in precleanup
    `/bin/cp /p/devise/demo2/MetaSearch/db/url_shows_rank.ds /p/devise/demo2/MetaSearch/db/$dir/session/url_shows_rank.ds`;
    $stringFile="+< /p/devise/demo2/MetaSearch/db/$dir/session/url_shows_rank.ds";
    open dsFile,$stringFile;
    
    ReplaceOverFile("MS_DB_DIRECTORY",$dir);
    
    $string=sprintf("%-8.0f %-8.0f %-8.0f %-8.0f",($x_min, $x_max, $y_min, $y_max));
    ReplaceOverFile("MS_XMINVAL MS_XMAXVAL MS_YMINVAL MS_YMAXVAL",$string);
    ReplaceOverFile("MS_SEARCH_ENGINE1",$SE1);
    ReplaceOverFile("MS_SEARCH_ENGINE2",$SE2);
    close dsFile;
    
}
#print STDERR "-----------------------\n";

#if(open(dat1File,"+< /p/devise/demo2/MetaSearch/db/$dir/$SE1.dat")!=0){
#    if(open(dat2File,"+< /p/devise/demo2/MetaSearch/db/$dir/$SE2.dat")!=0){
#	if(open(stringsFile,"+< /p/devise/demo2/MetaSearch/db/$dir/devise.strings")!=0){
#	    if(open(stringsFile,"+< /p/devise/demo2/MetaSearch/db/$dir/devise.strings")!=0){
#	if(
#	close dat2File;

`run/devisec -p 6100 -h monty\.cs\.wisc\.edu \/p\/devise\/demo2\/MetaSearch\/db\/$dir\/session\/url_shows_rank\.ds`;
#print STDERR "%%%%%%%%%%%%%%%%%%%%%%%%\n";
$date_string = `date \'+_%d%m_%H%M%S\'`;
chop($date_string);
$date_string=$dir . $date_string . ".gif";

`./MSpostcleanup $dir $date_string`;
#print STDERR "End of Clean up\n";
# Customise Reply Page
open dsFile, "+< /p/devise/demo2/MetaSearch/db/$dir/dat/replypage.html";
$string=sprintf("%-8.0f",$x_min);	ReplaceOverFile("MS_XMINVAL",$string );
$string=sprintf("%-8.0f",$x_max);	ReplaceOverFile("MS_XMAXVAL",$string );
$string=sprintf("%-8.0f",$y_min);	ReplaceOverFile("MS_YMINVAL",$string );
$string=sprintf("%-8.0f",$y_max);	ReplaceOverFile("MS_YMAXVAL",$string );
ReplaceOverFile("MS_DB_DIRECTORY", $dir);
ReplaceOverFile("DATE_STRING",$date_string);

ReplaceOverFile("MS_SEARCH_ENGINE1",$SE1);
ReplaceOverFile("MS_SEARCH_ENGINE2",$SE2);
$search_num=$searchparam{"resultnum"};
$search_keyword =  lc join('+',$searchparam{"searchname"});
ReplaceOverFile("                                                                 MS_SEARCHNAME", $search_keyword);
ReplaceOverFile("MS_RESULTNUM",$search_num);
#	ReplaceOverFile("SEARCH KEYWORDS :",$tempString);
seek dsFile,0,0;
close dsFile;

$seconds=((int time) - $inTime);

#print STDERR "End of Clean up\n";

if($searchAnew==1){
    open(synchFile,"> /p/devise/demo2/MetaSearch/db/$dir/synch_infomgr")
	or die "Could not open synch_infomgr\n";
    print synchFile "0\n$inTime\n1\n$inTime\n$seconds\n";    
}
else {
    open(synchFile,"+< /p/devise/demo2/MetaSearch/db/$dir/synch_infomgr") 
    or die "Could not open synch_infomgr\n";
    <synchFile>;
    $firstRequest=<synchFile>;
    $numberOfRequestsMade=<synchFile>+1;
    <synchFile>;
    $secondsOfService=<synchFile>+$seconds;
    seek synchFile,0,0;
    print synchFile 
	"0\n$firstRequest$numberOfRequestsMade\n$inTime\n$secondsOfService\n";
    
}
close synchFile;
   


if(($x_clik!=0) || ($y_clik!=0)){
    ReturnURLInfo($x_clik,$y_clik);
    exit 0;
}

# send it back
open dsFile, "+< /p/devise/demo2/MetaSearch/db/$dir/dat/replypage.html";

while($string=<dsFile>){
    print $string;
};
close dsFile;

exit 0;
	


sub buildSearchTables {
 my($engine_names);

 $search_num = $searchparam{"resultnum"};
 
 $fetchsize=$search_num;
# this is the best strategy
 if($search_num <=10){
     $timeout = 10;
     @SEbits = (1,1,1,0,1,0);
 }
 elsif($search_num<=20){
     $timeout = 20; 
# works for $timeout = 5
     @SEbits = (1,1,1,0,1,0);
 }
 elsif($search_num<=100){
     $timeout = 20;
# works for $timeout =10
     @SEbits = (1,1,1,0,1,0);
 }     
#@SEbits = (0,1,1,0,0,0);
 else
 {
     if($search_num > 400){
	 $search_num = 400;
     }
     $timeout = 20;
     @SEbits = (1,1,1,0,1,0);
 }

 for($i=0;$i<$SearchEngineNum;$i++){
     if($SEbits[$i]==1){
	 push @SEList, $SearchEngines[$i];
	 $SENumList++;
     }
 }

 if ( $search_num =~ /D/)
    {
    print "The searchnum MUST BE a positive INT !\n";
    exit; 
    }

 #put the search string together
 $search_keyword = lc join('+',$searchparam{"searchname"});
 $engine_names = $searchparam{"DataSource"};

# for a new Connection, ($dir == 0) and the loop below will be entered
# for an old connection, it will never be entered.

 $MAXINT=2**32 ;
# $dir=7;
 while(($dir < 10)){
     $dir=rand $MAXINT; 
 }
# $dir = SynchWithCGIs();

#clean up
 `./MSprecleanup $dir`;

 DoOmniSearch($engine_names,$search_num,$dir,$search_keyword );
 `./MScheckpoint $dir`;
 create_devise_strings();
 $x_max = $search_num * 1.2;
 $y_max = 100;
 $x_min = 0.00;
 $y_min = 0.00;

# CustomiseSessionFile

 $stringFile="+< /p/devise/demo2/MetaSearch/db/$dir/session/url_shows_rank.ds";
 open dsFile,$stringFile;
  
 ReplaceOverFile("MS_DB_DIRECTORY",$dir); 
 #print STDERR "ok\n";
 $string=sprintf("%-8.0f %-8.0f %-8.0f %-8.0f",($x_min,$x_max,$y_min,$y_max));
 ReplaceOverFile("MS_XMINVAL MS_XMAXVAL MS_YMINVAL MS_YMAXVAL",$string);

 ReplaceOverFile("MS_SEARCH_ENGINE1", $SE1);

 ReplaceOverFile("MS_SEARCH_ENGINE2", $SE2);
 #print STDERR "still ok\n";
 close dsFile;
 
 return;
}


# added sorting features
sub get_url_site_dom {
    my($url) = shift;
    my($site);

    if( $url =~ /:\/\/([^:\/]+)/ )
    {
       $site = $1;
       if( $site =~ /\.$/ )
       {
          chop($site);
       }

       #change to lower case
       $site =~ tr/A-Z/a-z/;

       if ( $site =~ /\.([^.]+)$/ )
       {
          my($dom) = $1;
          $dom =~ tr/A-Z/a-z/;
          ($dom, $site);
       }
       else
       {
#         die "no domain in this site\n";
	   (0,0);
       }
    }
    else
    {
 #      die "no site in this $url\n";
	(0,0);
    }
}

sub url_recursive_like {
    # subroutine that serves as the comparison function for sorting
    # the function compares the two site names (www.cs.wisc.edu and
    # www.wisc.edu), and does it component by component ( starting from the
    # last, i.e., starting with domain name (edu) ), and return whichever is
    # smaller. If they are identical, return the lexical order of the strings.

    if ( $a =~ /\d+: (.*)/){
	$a_url = $1;
    }
    if ( $b =~ /\d+: (.*)/){
	$b_url = $1;
    }

    return ($a_url cmp $b_url);

}


sub cleanList {
    my(@rawList, $valid, @newList, $search_num, $length);
    if(open(inUrlFile, "</p/devise/demo2/MetaSearch/db/$dir/dat/$_[0].dat-urls")) {
        return (0,0) unless (@rawList=<inUrlFile>);

        close inUrlFile;

        $valid = shift @rawList;
	$length = @rawList;


	for($search_num = $searchparam{"resultnum"};
	    $search_num < $length;--$length){
	    pop @rawList;
	}
	return (1,@rawList);

        if($valid == 0){
            pop @rawList;
            return (0,@rawList);
              }
        else {
            return (1,@rawList);
         }
    }
    else {
        return (0,0);
    }
}


sub cleanDatFiles {
        my($max_length)=$_[0];
        $lower_case = lc $SEList[$max_length];
        open(inDatFile, "+< /p/devise/ext11/demo2/MetaSearch/db/$dir/dat/$lower_case\.dat"); 

        unless($in_str=<inDatFile>){
	    printHeader();
            print h2("Error :\n");
            print "No results : no search engine was able to match your keywords.\n</td> </tr> </table> </center>\n </BODY> </HTML>";
            die "Increase time out period";
	    exit 0;
        }

        while($newin_str = <inDatFile>)
        {
                $in_str=$newin_str;
        }
        $in_str_length = length $in_str; 
        seek inDatFile, - $in_str_length,1;

        chomp $in_str;
        $in_str =~ s/./ /;
        print inDatFile $in_str;
        close inDatFile;

}



sub create_devise_strings {

    my($null, $i);
    my($prev);
    my($num_strings) = 0;
    my($search_num) = $searchparam{"resultnum"};
    my($in_str_length);

    open(stringFile, "> /p/devise/ext11/demo2/MetaSearch/db/$dir/dat/devise\.strings");

    ($valid[0],@List1)= cleanList(lc $SEList[0]);
    $length[0]=@List1;


    ($valid[1],@List2)= cleanList(lc $SEList[1]);
    $length[1]=@List2;

    #print STDERR "2. @length \n",$search_num,"\n";
    @rawLists = (@List1,@List2);

    #print STDERR $max_length1, "\n",$max_length2,"\n";

#    if($valid[$max_length1]==0) {
#        cleanDatFiles $max_length1;
#    }
#    if($valid[$max_length2]==0) {
#        cleanDatFiles $max_length2;
#    }


    $SE1=lc $SEList[0];
    $SE2=lc $SEList[1];

    @sortedLists = sort url_recursive_like @rawLists;
    $prev=0;
    for($null=shift @sortedLists;@sortedLists ne 0; $null = shift @sortedLists)
    {
	if($prev ne $null){
	    if($null =~ /\d+: (.*)/){
		print stringFile "$1\n";
		$num_strings++;
	    }
	}
	$prev=$null;
    }
    if($prev ne $null){
	if($null =~ /\d+: (.*)/){
	    print stringFile "$1\n";
	}
    }
    
    close(stringFile);
#    print STDERR "done with strings();\n";
    return $num_strings;
}



#
sub DoOmniSearch {
	my($engine_names, $search_num, $dir, $search_keyword, $search_file );

        $engine_names = $_[0];
        $search_num = $_[1];
        $dir = $_[2];
        $search_keyword = $_[3];
	open(dieFile, "> die");
	print dieFile "0\n";
	close dieFile;
eval {
	local $SIG{ALRM} = sub { die "alarm\n"; };

        @child_pid_list = 0;

        # fork off the child processes
	
        for ($i=0;$i<$SENumList;$i++){
	    $search_file="search/explore" . $SEList[$i] . " $search_num $dir $timeout $fetchsize $search_keyword  2\>\/dev\/null"; 
	    #print STDERR $search_file;
	    if(($pid = fork) == 0){
		`$search_file`;
		exit 0;
	    }
	    $child_pid_list[$i] = $pid;
	    
        }
        alarm $timeout;

        sleep;
};
	open(dieFile, "+< die");
	seek dieFile,0,0;
	print dieFile "1\n";
	close dieFile;
return;
 	
}



sub ReplaceOverFile {
    my ($fileName, $string1, $string2, $string3, $dsFile);
    my ($in_str, $in_strlength, $newstrlength);
    $string1=$_[0];
    $string2=$_[1];

    seek dsFile,0,0;
    while($in_str = <dsFile>){
	if( $in_str =~ /$string1/){
	    $in_strlength=length $in_str;
	    seek dsFile,-$in_strlength,1;
	    $in_str =~ s/$string1/$string2/;
	
	    $newstrlength = length $in_str;
	    chomp($in_str);
	    while($newstrlength<$in_strlength){
	    	$in_str .= " " ;
		++$newstrlength;
	    }

	    $in_str .= "\n";
	    print dsFile $in_str;
	    seek dsFile,0,1;
	    seek dsFile,-$in_strlength,1;
	}
    }

    return;
}



# Recalculate x_min, x_max, y_min, y_max after rubberband. $query_string
# can contain [n,s,e,w,ne,nw,se,sw].x . Does not rubberband if 
# $query_string contains none of the above.
sub RecalculateXY {
    my ($newx_min)=$x_min;
    my ($newy_min)=$y_min;
    my ($newx_max)=$x_max;
    my ($newy_max)=$y_max;

    if($query_string =~ /n.?\.x/){
		     $newy_min = $y_max;
		     $newy_max = 2*$y_max - $y_min;
		     }
    if($query_string =~ /s.?\.x/){
		     $newy_max = $y_min;
		     $newy_min = 2*$y_min - $y_max;
		     }
    if($query_string =~ /w\.x/){
		     $newx_max = $x_min;
		     $newx_min = 2*$x_min - $x_max;
		     }
    if($query_string =~ /e\.x/){
		     $newx_min = $x_max;
		     $newx_max = 2*$x_max - $x_min;
		     }

    $x_min=$newx_min;
    $y_min=$newy_min;
    $x_max=$newx_max;
    $y_max=$newy_max;
    return;
}


sub ReturnURLInfo{

my($i,$j, $match_url, $match_engine);
my($LMARG) = 48;
my($RMARG) = 3;
my($TMARG) = 25;
my($BMARG) = 15;
my($RIGHT) = 370;
my($BOTTOM) = 260;
my($string);

$x_clik_min = $_[0]-7;
$y_clik_min = $_[1]+7;
$x_clik_max = $_[0]+7;
$y_clik_max = $_[1]-7;
#restore sanity
if($x_clik_min < $LMARG){
    $x_clik_min = $LMARG;
}
if(($BOTTOM  - $BMARG)<$y_clik_min){
    $y_clik_min = $BOTTOM - $BMARG;
}
if($x_clik_max > ($RIGHT - $RMARG)){
    $x_clik_max = ($RIGHT - $RMARG);
}
if($y_clik_max < $TMARG){
    $y_clik_max = $TMARG;
}


$x_click_min_val = ( ( ($x_clik_min-$LMARG)*1.0 )/($RIGHT - $LMARG - $RMARG) )*($x_max - $x_min) + $x_min;
$y_click_min_val = ( ( ($BOTTOM - $y_clik_min-$BMARG)*1.0 )/($BOTTOM - $TMARG - $BMARG) )*($y_max - $y_min) + $y_min;
$x_click_max_val = ( ( ($x_clik_max-$LMARG)*1.0 )/($RIGHT - $LMARG - $RMARG) )*($x_max - $x_min) + $x_min;
$y_click_max_val = ( ( ($BOTTOM - $y_clik_max-$BMARG)*1.0 )/($BOTTOM - $TMARG - $BMARG) )*($y_max - $y_min) + $y_min;

$temp_url_min_val = int($x_click_min_val);
$temp_url_max_val = int($x_click_max_val)+1;
$rank_min_val = roundoff($y_click_min_val);
$rank_max_val = roundoff($y_click_max_val);


$num_urls = $temp_url_max_val - $temp_url_min_val + 1;
$num_ranks = $rank_max_val - $rank_min_val + 1;

if(($temp_url_max_val < 0) || ($temp_url_min_val > ($x_max+1)) || ($rank_max_val<1) 
   || ($rank_min_val>$y_max)){

    open dsFile, "+< /p/devise/demo2/MetaSearch/db/$dir/dat/replypage.html";
    while($string=<dsFile>){
	print $string;
    }
    close dsFile;
    exit 0;
}

#print STDERR $temp_url_min_val," ",$rank_min_val,"\n";
#print STDERR $temp_url_max_val," ",$rank_max_val,"\n";
#get the records corresponding to $rank_val
open(datFile,"< /p/devise/demo2/MetaSearch/db/$dir/dat/$SE1\.dat");
for($i=0;$i<($rank_min_val-1);$i++){
    <datFile>;
}
$i=$rank_min_val-1;
for($j=0;$i<$rank_max_val;$i++){
    if($record1[$j]=<datFile>){
	$j++;
	next;
    }
    else {
	$num_ranks =  $j;
	last;
    }
}
close datFile;
#print STDERR @record1, "\n";

open(datFile,"< /p/devise/demo2/MetaSearch/db/$dir/dat/$SE2\.dat");
for($i=0;$i<($rank_min_val-1);$i++){
    <datFile>;
}
$i=$rank_min_val-1;
for($j=0;$i<$rank_max_val;$i++){
    if($record2[$j]=<datFile>){
	$j++;
	next;
    }
    else {
	$num_ranks =  $j;
	last;
    }
}

close datFile;

#print STDERR @record2,"\n";



#get the corresponding strings in devise.strings
open(stringFile,"< /p/devise/demo2/MetaSearch/db/$dir/dat/devise\.strings");
for($i=0;$i<($temp_url_min_val-1);$i++){
    <stringFile>;
}
$i=$temp_url_min_val-1;
for($j=0;$i<$temp_url_max_val;$i++,$j++){
    if($sch_url[$j]=<stringFile>){
	chomp($sch_url[$j]);
	$mysch_url[$j]=$sch_url[$j];
	$sch_url[$j] =~ s/([^a-zA-z0-9 ])/\\$1/g;
	next;
    }
    else {
	$num_urls=$j;
	last;
    }
}

close stringFile;


#print STDERR @sch_url,"\n";


$match_url=0;
$match_engine=0;
for($i=0;$i<$num_urls;$i++){
    for($j=0;$j<$num_ranks;$j++){
	if(@record1){
	    if($record1[$j] =~ m#$sch_url[$i]#){
	       $match_url=$mysch_url[$i];
	       $match_engine=($SE1);
	       $rank_val = $rank_min_val + $j;
	       last;
	   }
	}
	if(@record2){
	    if($record2[$j] =~ m#$sch_url[$i]#){
	       $match_url=$mysch_url[$i];
	       $match_engine=($SE2);
	       $rank_val = $rank_min_val + $j;
	       last;
	   }
	}
	#print STDERR "$sch_url[$i] $record1[$j] $record2[$j]\n";
    }
}
	
open dsFile, "+< /p/devise/demo2/MetaSearch/db/$dir/dat/replypage.html";
for($i=0;$i<16;$i++){
    $string=<dsFile>;
    print $string;
}

if($match_engine ne 0){
    <dsFile>;
    print "URL from <a href=\"http://www\.$match_engine\.com\">";
    print ucfirst($match_engine),"</a>, rank $rank_val   :   ";
    print "<a href=\"$match_url\">$match_url</a><br>";
}


while($string=<dsFile>){
    print $string;
}
close dsFile;

if($searchAnew==0){
    open(synchFile,"+< /p/devise/demo2/MetaSearch/db/$dir/synch_infomgr") 
	or die "Could not open synch_infomgr\n";
    <synchFile>;
    $firstRequest=<synchFile>;
    $numberOfRequestsMade=<synchFile>+1;
    <synchFile>;
    $secondsOfService=<synchFile>+$seconds;
    seek synchFile,0,0;
    print synchFile 
	"0\n$firstRequest$numberOfRequestsMade\n$inTime\n$secondsOfService\n";
    close synchFile;
}
exit 0;


}

sub roundoff {
    my($val)=$_[0];
    
    if(($val - int($val))<0.5){
	return int($val);
    }
    else {
	return (int($val) + 1);
    }
}

sub printHeader {
    open dsFile, "+< /p/devise/demo2/MetaSearch/db/replypage.html";
    for($i=0;$i<16;$i++){
	$string=<dsFile>;
	print $string;
    }
    close dsFile;
}









 open(synchFile,"+< /p/devise/demo2/MetaSearch/db/$dir/synch_infomgr") 
     or die "Could not open synch_infomgr\n";
 <synchFile>;
 $firstRequest=<synchFile>;
 $numberOfRequestsMade=<synchFile>+1;
 <synchFile>;
 $secondsOfService=<synchFile>+$seconds;
 seek synchFile,0,0;
 print synchFile 
     "0\n$firstRequest$numberOfRequestsMade\n$inTime\n$secondsOfService\n";
 
 close synchFile;
