#!/usr/local/bin/perl

#########################################################################
##
## RUN_AGONISTICS
##
## Warren Sack (wsack@media.mit.edu)
##
## February 2005
##
## Usage: perl run_agonistics.pl <configuration file>
##
## Preconditions: (1) The file configuration file needs to exist and contain
##                    a correct set of values for all of the necessary
##                    parameters.
##
##                (2) The sub-directory "Resources" needs to be exist within
##                    the directory that contains this script.  The Resources
##                    sub-directory contains a number of images and CGI scripts.
##
##                (3) If this script is going be run by polling new messages
##                    periodically from a server, a means to connect and
##                    download those messages needs to be arranged before the
##                    script is run.  See the example configuration files
##                    *.conf for details concerning, for example, the use of 
##                    a Yahoo email account as an means to poll and archive 
##                    new messages from a mailing list.
##
#########################################################################
##
## Copyright (c) 2005 by Warren Sack
##
## This work is copyrighted with a Creative Commons 
## (http://creativecommons.org) Attribution-NonCommercial-NoDerivs 2.0
## License.
##
## Here is a short summary of the license:
##
## You are free to copy, distribute, display, and perform the work
## under the following conditions:
##
## Attribution: You must give the original author (Warren Sack) credit.
##
## Noncommercial: You may not use this work for commercial purposes.
##
## No Derivative Works: You may not alter, transform, or build upon this work.
##
## * For any reuse or distribution, you must make clear to others the 
##   license terms of this work.
## * Any of these conditions can be waived if you get permission from 
##   the copyright holder (Warren Sack).
## * Your fair use and other rights are in no way affected by the above.
## * The details and full text of the license can be found at this URL:
##   http://creativecommons.org/licenses/by-nc-nd/2.0/legalcode
##
#########################################################################

if ( $#ARGV < 0 ) {
    die "Usage: run_agonistics.pl <configuration file>\n".
	"This script requires one argument.\n";
}
elsif ( not(-e $ARGV[0]) ) {
    die "Usage: run_agonistics.pl <configuration file>\n".
	"This script requires a configuration file.  The given configuration file cannot be found.\n";
}


use utf8;
use Unicode::Normalize;
use HTML::Entities();	
use File::Temp qw(tempfile);
use File::Path;
use Date::Manip qw(ParseDate ParseDateString UnixDate);
use Time::Local;
use File::Copy;
use MIME::WordDecoder;
use Lingua::Stem;
use Fcntl qw(:flock);
use Crypt::SSLeay;
use Mail::Client::Yahoo;
use Digest::MD5;
use Net::NNTP;
use Net::SSLeay;
use IO::Socket::SSL;
use Net::IMAP::Simple::SSL;

$batch_of_messages = 0;

## READ_CONFIG_FILE
##
sub read_config_file
{
    my($key,$value);

    open(CONFIG,$ARGV[0]) || die "Can't find configuration file: $ARGV[0]\n";
    while(<CONFIG>) {
	chomp;
	if ( /^\#/ ) { next; }
	if ( /^\s*$/ ) { next; }
	($key,$value) = $_ =~ /^(\S+)\s+(\S+)/;
	$agonistics_config{$key} = $value;
    }
    close(CONFIG);
}


## INITIALIZE_GLOBAL_VARIABLES
##
## Input:   None
##
## Effects: Global variables associated with this package are initialized.
##
## Output:  None
##
sub initialize_global_variables
{
    ## initializations done only before the first batch of messages is processed
    if ( $batch_of_messages == 0 ) {
	## Open and read the config file into a hash.
	%agonistics_config = ();
	&read_config_file();
	## Before setting slash, the machine's OS is checked.
	if ( $ENV{'OSTYPE'} =~ /^win/i ) { $slash = '\\'; }
	else { $slash = '/'; }
	$news_group = $agonistics_config{'Newsgroup'};
	$archive_file_name = $agonistics_config{'FileNameOfArchive'};
	$language_locale = $agonistics_config{'LanguageTag'};
	if ( defined($agonistics_config{'MaxFrames'}) ) {
	    $max_frames = $agonistics_config{'MaxFrames'};
	}
	else { $max_frames = 1000; }
	if ( defined($agonistics_config{'PauseBetweenFrames'}) ) {
	    $pause_between_frames = $agonistics_config{'PauseBetweenFrames'};
	}
	else { $pause_between_frames = 7; }
	## Load the correct end-of-sentence tagger.  English and French
	## texts both use the English tagger.  German texts use a 
	## different tagger.
	if ($language_locale eq 'DE') { 
	    require Lingua::DE::Sentence;
	    Lingua::DE::Sentence->import( qw(get_sentences) );
	}
	else { 
	    require Lingua::EN::Sentence;
	    Lingua::EN::Sentence->import( qw(get_sentences) );
	}
	$archive_name = $agonistics_config{'DirectoryForOutput'};
	$recency = $agonistics_config{'Recency'};
	$documents_url = $agonistics_config{'DocumentsURL'};
	$cgi_url = $agonistics_config{'CGIURL'};
	$web_server_directory = $agonistics_config{'WebServerDirectory'};
	$web_server_cgi_directory = $agonistics_config{'WebServerCGIDirectory'};
	$is_interactive_p = $agonistics_config{'InteractiveMode'};
	if ( $is_interactive_p =~ /n/i ) { $is_interactive_p = 'NO'; }
	else { $is_interactive_p = 'YES'; }
	## Record the address to be used for posting messages to the list analyzed.
	$mailing_list_address = $agonistics_config{'MailingListAddress'};
	## Note information about the Yahoo mail account, if it is to be used.
	$yahoo_uid = $agonistics_config{'YahooUID'};
	$yahoo_password = $agonistics_config{'YahooPassword'};
	$yahoo_outbox = $agonistics_config{'YahooOutbox'};
	## Note information about the IMAP mail server and account, if it is to be used.
	$imap_server = $agonistics_config{'IMAPServer'};
	$imap_uid = $agonistics_config{'IMAPUID'};
	$imap_password = $agonistics_config{'IMAPPassword'};
	$imap_outbox = $agonistics_config{'IMAPOutbox'};
	## Note information about the NNTP account and server, if it is to be used.
	$nntp_server = $agonistics_config{'NNTPServer'};
	$nntp_uid = $agonistics_config{'NNTPUID'};
	$nntp_password = $agonistics_config{'NNTPPassword'};
	if ( ( $nntp_server and ( $yahoo_uid or $imap_server ) )
	     or ( $yahoo_uid and ( $nntp_server or $imap_server ) )
	     or ( $imap_server and ( $nntp_server or $yahoo_uid ) ) ) {
	    die "Only one of the following may be defined: (a) NNTP server; (b) Yahoo UID; (c) IMAP server".
		"\nTwo of the three need to be commented out in the configuration file $ARGV[0]\n";
	}
	## How many seconds should the script wait between tries to download
	## messages from the server?
	$pause = $agonistics_config{'PauseBetweenFetches'};
	$output_directory = $web_server_directory.$slash.'Agonistics'.$slash.$archive_name;
	$cgi_directory = $web_server_cgi_directory.$slash.'Agonistics';
	## create the directory to house the CGI scripts
	mkdir($cgi_directory);
	## copy the CGI scripts into the CGI directory
	my $cgi_script_file;
	my $send_script_file_found_p = 0;
	opendir CGISCRIPTS, 'Resources'.$slash.'CGIScripts' or die "Cannot open CGIScripts directory: $!";
	foreach $cgi_script_file (readdir CGISCRIPTS) {
	    if ( $cgi_script_file eq 'send_message.pl' ) {
		&rewrite_send_script_file('Resources'.$slash.'CGIScripts'.$slash.$cgi_script_file,$cgi_directory.$slash.$cgi_script_file);
		$send_script_file_found_p = 1;
	    }
	    else { copy('Resources'.$slash.'CGIScripts'.$slash.$cgi_script_file,$cgi_directory.$slash.$cgi_script_file); }
	    chmod(0777,$cgi_directory.$slash.$cgi_script_file);
	}
	closedir CGISCRIPTS;
	if ( $send_script_file_found_p == 0 ) { die "Can't find send_message.pl CGI script\n"; }
	## Alternatively load the German or English+French end-of-sentence
	## tagger depending upon the language specified on the command line.
	if ( $language_locale =~ /^DE/ ) {
	    require Lingua::DE::Sentence;
	    Lingua::DE::Sentence->import( qw(get_sentences) );
	}
	else { 
	    require Lingua::EN::Sentence;
	    Lingua::EN::Sentence->import( qw(get_sentences) );
	}
	$stemmer = Lingua::Stem->new({-locale => $language_locale});
	$stemmer->stem_caching({ -level => 2 });
	$log_file = $agonistics_config{'FileNameOfLog'};
	open(LOG,'>'.$log_file);
	close(LOG);
	$radius_of_circle = 100000;
	my $random_number = int(rand(10000));
	$raw_messages_file = 'raw_messages_file_'.$random_number.'.txt';
	$end_of_message_marker = '__end_of_message_marker__';
	## English stop words
	@english_stop_words = ("a", "about", "above", "according", "across", "actually", "adj", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also", "although", "always", "among", "amongst", "an", "and", "another", "any", "anyhow", "anyone", "anything", "anywhere", "are", "aren", "around", "as", "at", "b", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "begin", "beginning", "behind", "being", "below", "beside", "besides", "between", "beyond", "billion", "both", "but", "by", "c", "can", "can", "cannot", "caption", "co", "could", "couldn", "d", "did", "didn", "do", "does", "doesn", "don", "down", "during", "e", "each", "eg", "eight", "eighty", "either", "else", "elsewhere", "end", "ending", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "f", "few", "fifty", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "further", "g", "h", "had", "has", "hasn", "have", "haven", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed", "instead", "into", "is", "isn", "it", "its", "itself", "j", "k", "l", "last", "later", "latter", "latterly", "least", "less", "let", "like", "likely", "ll", "ltd", "m", "made", "make", "makes", "many", "maybe", "me", "meantime", "meanwhile", "might", "million", "miss", "more", "moreover", "most", "mostly", "mr", "mrs", "much", "must", "my", "myself", "n", "namely", "neither", "never", "nevertheless", "next", "nine", "ninety", "no", "nobody", "none", "nonetheless", "noone", "nor", "not", "nothing", "now", "nowhere", "o", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "overall", "own", "p", "per", "perhaps", "q", "r", "rather", "re", "recent", "recently", "s", "same", "seem", "seemed", "seeming", "seems", "seven", "seventy", "several", "she", "should", "shouldn", "since", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "stop", "such", "t", "taking", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thirty", "this", "those", "though", "thousand", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "toward", "towards", "trillion", "twenty", "two", "u", "under", "unless", "unlike", "unlikely", "until", "up", "upon", "us", "used", "using", "v", "ve", "very", "via", "w", "was", "wasn", "we", "well", "were", "weren", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whomever", "whose", "why", "will", "with", "within", "without", "would", "wouldn", "wrote", "x", "y", "yes", "yet", "you", "your", "yours", "yourself", "yourselves", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "http", "www", "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", "com", "edu", "org", "just", "net", "ftp", "nntp", "http", "html");
	@english_present_tense_of_to_be = ( "am", "are", "is" );
	## French stop words
	## Note that some words here are intentionally misspelled (i.e., spelled without diacritical marks).  
	## This appears to be necessary since the diacritical marks are not always employed by the participants
	## and they are oftentimes lost between the client and server.
	@french_stop_words = ( "a", "au", "aux", "avec", "ce", "ces", "dans", "de", "des", "du", "elle", "elles", "en", "et", "eux", "il", "ils", "je", "la", "le", "les", "leur", "lui", "ma", "mais", "me", "m\x{EA}me", "mes", "moi", "mon", "ne", "nos", "notre", "nous", "on", "ou", "par", "pas", "plus", "pour", "qu", "que", "qui", "bien", "bon", "bonne", "bonnes", "faire", "fais", "fait", "faisons", "font", "sa", "se", "ses", "son", "sur", "ta", "te", "tes", "toi", "ton", "tu", "un", "une", "vos", "votre", "vous", "c", "d", "j", "l", "\x{E0}", "m", "n", "s", "t", "y", "\x{E9}t\x{E9}", "ete", "\x{E9}t\x{E9}e", "etee", "\x{E9}t\x{E9}es", "etees", "\x{E9}t\x{E9}s", "etes", "\x{E9}tant", "etant", "\x{E9}tante", "etante", "\x{E9}tants", "etants", "\x{E9}tantes", "etantes", "suis", "es", "est", "sommes", "\x{EA}tes", "etes", "sont", "serai", "seras", "sera", "serons", "serez", "seront", "serais", "serait", "serions", "seriez", "seraient", "\x{E9}tais", "etais", "\x{E9}tait", "etait", "\x{E9}tions", "etions", "\x{E9}tiez", "etiez", "\x{E9}taient", "etaient", "fus", "fut", "f\x{FB}mes", "fumes", "f\x{FB}tes", "futes", "furent", "sois", "soit", "soyons", "soyez", "soient", "fusse", "fusses", "f\x{FB}t", "fut", "fussions", "fussiez", "fussent", "ayant", "ayante", "ayantes", "ayants", "eu", "eue", "eues", "eus", "ai", "as", "avons", "avez", "ont", "aurai", "auras", "aura", "aurons", "aurez", "auront", "aurais", "aurait", "aurions", "auriez", "auraient", "avais", "avait", "avions", "aviez", "avaient", "eut", "e\x{FB}mes", "eumes", "e\x{FB}tes", "eutes", "eurent", "aie", "aies", "ait", "ayons", "ayez", "aient", "eusse", "eusses", "e\x{FB}t", "eut", "eussions", "eussiez", "eussent", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "un", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "http", "message", "jamais", "fr", "sans", "non", "oui", "dire", "dis", "dit", "disez", "disons", "disent", "faut", "toujours", "que", "quel", "quelle", "quelles", "quelqu", "quelque", "quelques", "quels", "qui", "quiconque", "quoi", "quoiqu", "quoique", "\x{E7}a", "ca", "car", "ce", "ceci", "cela", "celle", "l\x{E0}", "la", "celles", "ci", "celles", "celui", "ces", "cet", "cette", "ceux", "comme", "vrai", "\x{EA}tre", "etre", "\x{E9}crit", "ecrit", "rien", "tout", "tous", "toute", "toutes", "si", "fr", "com", "edu", "org", "autres", "autre", "sauf", "vais", "vas", "va", "venons", "venez", "vont", "aller", "aimer", "aiment", "aimons", "aime", "aimes", "mal", "news", "nntp", "html", "ftp", "net", "part", "puis", "voir", "quand", "tant", "autant", "leur", "leurs", "tien", "tienne", "tiens", "tiennes", "mien", "mienne", "miens", "miennes", "soi", "aussi", "ailleurs", "moins", "alors", "passe", "pass\x{E9}", "avoir" );
	@french_present_tense_of_to_be = ( "suis", "es", "est", "\x{EA}tes", "etes", "sommes", "sont" );
	## German stop words
	## Note that some words here are intentionally misspelled (i.e., spelled without diacritical marks).  
	## This appears to be necessary since the diacritical marks are not always employed by the participants
	## and they are oftentimes lost between the client and server.
	@german_stop_words = ( "aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am", "an", "ander", "andere", "anderem", "anderen", "anderer", "anderes", "anderm", "andern", "anderr", "anders", "auch", "auf", "aus", "bei", "bin", "bis", "bist", "da", "damit", "dann", "der", "den", "des", "dem", "die", "das", "da\x{DF}", "derselbe", "derselben", "denselben", "desselben", "demselben", "dieselbe", "dieselben", "dasselbe", "dazu", "dein", "deine", "deinem", "deinen", "deiner", "deines", "denn", "derer", "dessen", "dich", "dir", "du", "dies", "diese", "diesem", "diesen", "dieser", "dieses", "doch", "dort", "durch", "ein", "eine", "einem", "einen", "einer", "eines", "einig", "einige", "einigem", "einigen", "einiger", "einiges", "einmal", "er", "ihn", "ihm", "es", "etwas", "euer", "eure", "eurem", "euren", "eurer", "eures", "f\x{FC}r", "fur", "gegen", "gewesen", "hab", "habe", "haben", "hat", "hatte", "hatten", "hier", "hin", "hinter", "ich", "mich", "mir", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "euch", "im", "in", "indem", "ins", "ist", "jede", "jedem", "jeden", "jeder", "jedes", "jene", "jenem", "jenen", "jener", "jenes", "jetzt", "kann", "kein", "keine", "keinem", "keinen", "keiner", "keines", "k\x{F6}nnen", "konnen", "k\x{F6}nnte", "konntemachen", "man", "manche", "manchem", "manchen", "mancher", "manches", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mit", "muss", "musste", "nach", "nicht", "nichts", "noch", "nun", "nur", "ob", "oder", "ohne", "sehr", "sein", "seine", "seinem", "seinen", "seiner", "seines", "selbst", "sich", "sie", "ihnen", "sind", "so", "solche", "solchem", "solchen", "solcher", "solches", "soll", "sollte", "sondern", "sonst", "\x{FC}ber", "uber", "um", "und", "uns", "unse", "unsem", "unsen", "unser", "unses", "unter", "viel", "vom", "von", "vor", "w\x{E4}hrend", "wahrend", "war", "waren", "warst", "was", "weg", "weil", "weiter", "welche", "welchem", "welchen", "welcher", "welches", "wenn", "werde", "werden", "wie", "wieder", "will", "wir", "wird", "wirst", "wo", "wollen", "wollte", "w\x{FC}rde", "wurde", "w\x{FC}rden", "wurden", "zu", "zum", "zur", "zwar", "zwischen", "dk", "com", "edu", "org", "news", "http", "ftp", "html", "com", "org", "net", "nntp");
	@german_present_tense_of_to_be = ( "bin", "bist", "ist", "sind", "seid" );
	@stop_words = ();
	if ( $language_locale =~ /^EN/ ) { @stop_words = @english_stop_words; }
	## Many French and German discussions include English posts, so English stop words are used for them too.
	elsif ( $language_locale =~ /^FR/ ) { push(@stop_words,@french_stop_words,@english_stop_words); }
	elsif ( $language_locale =~ /^DE/ ) { push(@stop_words,@german_stop_words,@english_stop_words); }
	%isa_stop_word = ();
	my $stop_word;
	foreach $stop_word (@stop_words) { $isa_stop_word{NFD($stop_word)} = 1; }
	## Connect to NNTP server if specified.  Note that this connection is made once and only once.
	## Attempts to connect and the reconnect to the NNTP server can result in multiple, hanging
	## connections and, thus, ultimately a refusal by the server to download new messages.
	if ( $nntp_server and ( $is_interactive_p eq 'YES' ) ) {
	    &print_to_log("Connecting to NNTP server $nntp_server...");
	    eval { $nntp = Net::NNTP->new($nntp_server) };
	    if ( $@ ) {
		&print_to_log("ERROR: Couldn't connect to NNTP server...");
		return;
	    }
	    &print_to_log("Logging in to NNTP server $nntp_server...");
	    if ( $nntp_uid and $nntp_password ) {
		eval { $nntp->authinfo($nntp_uid,$nntp_password) };
		if ( $@ ) {
		    &print_to_log("ERROR: Couldn't log in to NNTP server...");
		    return;
		}
	    }
	}
	## create the output directory
	if (-d $output_directory) { rmtree($output_directory,0,1); }
	mkdir($output_directory);
	## create a place to put the interface
	$interface_directory = $output_directory.$slash.'Interface';
	mkdir($interface_directory);
	## copy the face images into the interface subdirectory
	$face_images_directory = $interface_directory.$slash.'FaceImages';
	mkdir($face_images_directory);
	my $fimage_file;
	opendir FID, 'Resources'.$slash.'FaceImages' or die "Cannot open FaceImages: $!";
	my $number_of_face_images = 0;
	foreach $fimage_file (readdir FID) {
	    $number_of_face_images++;
	    copy('Resources'.$slash.'FaceImages'.$slash.$fimage_file,$face_images_directory.$slash.$fimage_file);
	}
	closedir FID;
	## Three images needed for every face (left, right and front);
	$number_of_faces = $number_of_face_images / 3; 
	## Copy the forward and rewind button images into the Interface directory.
	if (-e 'Resources'.$slash.'forward.gif') {
	    copy('Resources'.$slash.'forward.gif',$interface_directory.$slash.'forward.gif');
	}
	else { die "Can't find GIF file for forward button\n"; }
	if (-e 'Resources'.$slash.'rewind.gif') {
	    copy('Resources'.$slash.'rewind.gif',$interface_directory.$slash.'rewind.gif');
	}
	else { die "Can't find GIF file for rewind button\n"; }
	## Copy the help files into the Interface directory.
	if ( (-e 'Resources'.$slash.'help-en.html') 
	     and (-e 'Resources'.$slash.'help-fr.html') 
	     and (-e 'Resources'.$slash.'help-de.html') ) {
	    &rewrite_help_file('Resources'.$slash.'help-en.html',$interface_directory.$slash.'help-en.html');
	    &rewrite_help_file('Resources'.$slash.'help-fr.html',$interface_directory.$slash.'help-fr.html');
	    &rewrite_help_file('Resources'.$slash.'help-de.html',$interface_directory.$slash.'help-de.html');
	}
	else { die "Can't find help files\n"; }
	## create a place to put the messages and message indices
	$message_directory = $interface_directory.$slash.'Messages';
	mkdir($message_directory);
	## create a file name for a message index to be consulted by CGI scripts
	$message_index_file_name = $message_directory.$slash.'message_index';
	## Copy the style sheet into the output directory and the CGI directory
	if (-e 'Resources'.$slash.'message_style.css') {
	    copy('Resources'.$slash.'message_style.css',$message_directory.$slash.'message_style.css');
	    copy('Resources'.$slash.'message_style.css',$cgi_directory.$slash.'message_style.css');
	}
	else { die "Can't find message_style.css file\n"; }
	## Copy the style sheet into the interface directory.
	if (-e 'Resources'.$slash.'interface_style.css') {
	    copy('Resources'.$slash.'interface_style.css',$interface_directory.$slash.'interface_style.css');
	}
	else { die "Can't find interface_style.css file\n"; }
    } 
    ## initializations done only before the second (or later) batch of messages is processed
    else {
	## Replace the interface webpages of the last batch with blank pages.
	&create_blank_pages();
	## Remove the messages from the last batch.
	if (-d $message_directory) { rmtree($message_directory,0,1); }
	## Recreate a place to put the messages and message indices.
	mkdir($message_directory);
	copy('Resources'.$slash.'message_style.css',$message_directory.$slash.'message_style.css');
	&reorganize_archive();
    }
    undef %analyzed_p;
    $time_of_last_nntp_server_check = 0;
    $current_simplified_message_id = 0;
    $current_simplified_author_id = 0;
    $frame_number = 0;
    $current_simplified_sentence_id = 0;
    $max_score = 1;
    $min_score = 1;
    $current_winner = 0;
    $previous_winner = 0;
    $previous_winning_theme = 'previous';
    $current_winning_theme = 'current';
    $current_winning_sentence = '';
    $social_network_index = 0;
    $new_messages_downloaded = 0;
    %new_social_network = ();
    %old_social_network = ();
    %link = ();
    %sn2saids = ();
    %said2sn = ();
    %previous_sn2saids = ();
    %said2previous_sn = ();
    %sn2theta = ();
    %ssid2start = ();
    %ssid2end = ();
    %ssid2sentence = ();
    %ssid2smid = ();
    %smid2ssids = ();
    %token2stem = ();
    %stem2tokens = ();
    %stem2ssids = ();
    %ssid2stems = ();
    %citers = ();
    %repliers = ();
    %mid2smid = ();
    %smid2mid = ();
    %smid2sdt = ();
    %ana2said = ();
    %said2ana = ();
    %smid2said = ();
    %said2smids = ();
    %smid2nsl = ();
    %nsl2smids = ();
    %nsl2stems = ();
    %smid2newsgroups = ();
    %smid2date = ();
    %smid2subject = ();
    %mid2pid = ();
    %smid2spid = ();
    %spid2smids = ();
    %ancestors = ();
    %smid2fptrs = ();
    %said2lines = ();
    %tline2smidslns = ();
    %smidln2tline = ();
    %signature = ();
    %line_numbers_of_signature = ();
    %source = ();
    %smidln2source = ();
    %smid2sources = ();
    %prefix = ();
    %said2addr = ();
    @stop_words = ();
    %said2theta = ();
    %said2previous_theta = ();
    %said2theta_movement = ();
    %said2rho = ();
    %said2previous_rho = ();
    %said2rho_movement = ();
    %said2image_id = ();
    %said_is_speaker = ();
    %isa_quotation = ();
    %said2percentage_score = ();
    %said2previous_percentage_score = ();
    %said2score = ();
    %said2previous_score = ();
    %winning_sentences = ();
    %yahooid = ();

    return;
}


## REWRITE_HELP_FILE
##
sub rewrite_help_file
{
    my($template_file,$specific_file) = @_;
    
    my(@line,@new_line,$new_line,$word);

    open(TEMPLATE,$template_file);
    open(HELP,'>'.$specific_file);
    while($line = <TEMPLATE>) {
	chomp($line);
	@line = split /\s+/,$line;
	@new_line = ();
	foreach $word (@line) {
	    if ( $word =~ /(.*)Newsgroup(.*)/ ) { push(@new_line,$1.$news_group.$2); }
	    elsif ( $word =~ /(.*)PauseBetweenFetches(.*)/ ) { push(@new_line,$1.$pause.$2); }
	    else { push(@new_line,$word); }
	}
	$new_line = join ' ',@new_line;
	print HELP $new_line."\n";
    }
    close(TEMPLATE);
    close(HELP);
}


## REWRITE_SEND_SCRIPT_FILE
##
sub rewrite_send_script_file
{
    my($template_file,$specific_file) = @_;
    
    my(@line,@new_line,$new_line,$word);

    open(TEMPLATE,$template_file);
    open(SEND,'>'.$specific_file);
    while($line = <TEMPLATE>) {
	chomp($line);
	if ( $line =~ /NNTPServer/ ) { $new_line = '$nntp_server = '."'".$nntp_server."';"; }
	elsif ( $line =~ /NNTPUID/ ) { $new_line = '$nntp_uid = '."'".$nntp_uid."';"; }
	elsif ( $line =~ /NNTPPassword/ ) { $new_line = '$nntp_password = '."'".$nntp_password."';"; }
	else { $new_line = $line; }
	print SEND $new_line."\n";
    }
    close(TEMPLATE);
    close(SEND);
}


## PROCESS_NEXT_MESSAGE
##
## Input:   An integer: a pointer indicating the start of the current message in the archive.
##
## Effects: Several hashes are created.
##
## Output:  An integer: a pointer indicating the current end of the file
##          containing the messages.
##
## Notes:   The file handle is for a file of messages separated by __end_of_message__marker__
##          Important: no whitespace can follow the __end_of_message_marker_!
##          The file pointer indicates where, in the file of message, reading
##          of the messages should begin.
##
sub process_next_message
{
    my($start) = @_;

    my($message,$message_body,$end,%field_values,$field);
    my($marker_length,$message_id,$source_smid,@source_smids,$simplified_message_id);
    my($simplified_author_id);
    
    open(ARCHIVEFILE,$archive_file_name) || die "Can't open archive file\, $archive_file_name\, $! \n";
    $marker_length = length($end_of_message_marker);
    seek(ARCHIVEFILE,$start,0);
    $/ = $end_of_message_marker;
    $message = <ARCHIVEFILE>;
    $end = tell(ARCHIVEFILE);
    close(ARCHIVEFILE);
    if ( $message ) {
	undef %field_values;
	%field_values = &extract_fields_and_values_from_message($message);
	$message_body = '';
	if ( $message =~ m/\n\n/g ) { 
	    $message_body = substr($message,pos($message),length($message)-(pos($message)+$marker_length)); 
	}
	## If the message is a cancel message or if it does not have an id, then skip it.
	if ( ( defined($field_values{'control:'})
	       and ( $field_values{'control:'} =~ /cancel/i ) )
	     or (not defined($field_values{'message-id:'}) ) ) { 
	    $frame_number--;
	    return($end);
	}
	## Downcase the message-id and replace any spaces with underscores
	$field_values{'message-id:'} = lc($field_values{'message-id:'});
	$field_values{'message-id:'} =~ tr/ /_/;
	## Make sure the message is not a duplicate of one already in the database.
	if ( defined($analyzed_p{$field_values{'message-id:'}}) ) {
	    $frame_number--;
	    return($end); 
	}
	$analyzed_p{$field_values{'message-id:'}} = 1;
	$simplified_message_id = &assert_simplified_message_id($field_values{'message-id:'});
	$message_id = $field_values{'message-id:'};
	&assert_simplified_date_and_time($simplified_message_id,$field_values{'date:'});
	$simplified_author_id = &assert_simplified_author_id($field_values{'from:'});
	&assert_message_authorship($simplified_message_id,$simplified_author_id);
	&assert_normalized_subject_line($simplified_message_id,$field_values{'subject:'});
	&assert_message_header($simplified_message_id,
			       $field_values{'newsgroups:'},
			       $field_values{'date:'},
			       $field_values{'subject:'});
	if (not(defined $field_values{'references:'})) { $field_values{'references:'} = ' '; }
	if (not(defined $field_values{'in-reply-to:'})) { $field_values{'in-reply-to:'} = ' '; }
	&assert_reference_relation($field_values{'message-id:'},
				   $field_values{'references:'},
				   $field_values{'in-reply-to:'});
	&assert_file_pointers($simplified_message_id,$start,$end);
	&assert_message_body_authorship($simplified_author_id,$simplified_message_id,$message_body);
	&assert_threading_relations($message_id);
	&assert_line_positions($simplified_message_id);
	&assert_signature($simplified_author_id);
	&assert_sources($simplified_message_id);
	&assert_sentences($simplified_message_id);
	&write_html_for_message($simplified_message_id);
	if (defined($smid2sources{$simplified_message_id}) ) {
	    @source_smids = @{$smid2sources{$simplified_message_id}};
	}
	else { @source_smids = (); }
	push(@source_smids,$smid2spid{$simplified_message_id});
	foreach $source_smid (@source_smids) { &write_html_for_message($source_smid); }
	&update_social_networks($simplified_message_id,@source_smids);
	&update_scores();
	&update_themes($simplified_message_id);
	&write_html_for_interface($simplified_message_id,@source_smids);
	&write_entry_in_message_index($simplified_message_id);
	return($end);
    }
    else { return($start); }
}


## UPDATE_SOCIAL_NETWORKS
##
## Input:   A simplified message id (integer) and a list of other 
##          simplified message ids (integers) that represent either
##          the messages quoted or replied to by the first message.
##
## Effects: Three hashes are updates: %link, %social_network2saids,
##          and %said2sn.  The elements $link(A,B) and
##          $link(B,A) are added if B previously cited or replied to
##          A, and now, in the current message A has either cited or
##          replied to B.  (This information is stored in the 
##          hashes %repliers and %citers updated in other functions.)
##          If any new links are added, the social networks for A and
##          B are checked.  If they are the same, then nothing is done.
##          If they are different (and, thus, the new link constitutes
##          a bridge between the two social networks), the smaller of
##          the two social networks is dissolved and its elements are
##          added into the larger social network.
##
## Output:  none
##          
sub update_social_networks
{
    my($smid,@source_smids) = @_;

    my($said,$s_said,$weight_from_said,$weight_from_s_said,@new_links);
    my($acquiring_social_network,$disappearing_social_network,$ss);
    my($isa_member_of_new_social_network,$sni);

    ## For each player, save their current social network.  The update
    ## may change some of all of the players' social networks.  The
    ## previous positions of the players is necessary for updating
    ## the interface.
    undef %said2previous_sn;
    undef %previous_sn2saids;
    foreach $ss (keys %said2sn) {
	$said2previous_sn{$ss} = $said2sn{$ss};
	${%{$previous_sn2saids{$said2sn{$ss}}}}{$ss} = 1;
    }
    $said = $smid2said{$smid};
    ## If the author of the current message is appearing for the first
    ## time, s/he will constitute a separate new social network.  
    ## Since it is not possible for one to join another
    ## network with just one post, the new network will not be
    ## joined to any existing networks and, consequently, the subsequent
    ## code can be skipped.
    $isa_member_of_new_social_network = 0;
    foreach $sni (keys %new_social_network) {
	if ( ${%{$sn2saids{$sni}}}{$said} ) { 
	    $isa_member_of_new_social_network = 1;
	    last;
	}
    }
    if ( $isa_member_of_new_social_network ) { return; }
    ## Otherwise, the new message comes from someone who is already in an
    ## existing social network.  Test to see if the message constitutes
    ## a new link between two, as yet separate, social networks.
    else {
	@new_links = ();
	if ( defined($smid2said{$smid}) 
	     and defined($said2sn{$said}) ) {
	    foreach $s_smid (@source_smids) {
		$s_said = $smid2said{$s_smid};
		## If the author of the cited or replied to message
		## has ever cited or replied to the author of the
		## current message, then the two authors should be 
		## linked together in a social network.
		if ( defined($smid2said{$s_smid})
		     and ( defined(${%{$citers{$said}}}{$s_said})
			   or defined(${%{$repliers{$said}}}{$s_said}) ) ) {
		    if ( not(defined($link{$said,$s_said})) ) { push(@new_links,$s_said); }
		    $weight_from_s_said = 0;
		    if ( defined(${%{$citers{$said}}}{$s_said}) ) {
			$weight_from_s_said += ${%{$citers{$said}}}{$s_said};
		    }
		    if ( defined(${%{$repliers{$said}}}{$s_said}) ) {
			$weight_from_s_said += ${%{$repliers{$said}}}{$s_said};
		    }
		    $weight_from_said = 0;
		    if ( defined(${%{$citers{$s_said}}}{$said}) ) {
			$weight_from_said += ${%{$citers{$s_said}}}{$said};
		    }
		    if ( defined(${%{$repliers{$s_said}}}{$said}) ) {
			$weight_from_said += ${%{$repliers{$s_said}}}{$said};
		    }
		    if ( $weight_from_s_said < $weight_from_said ) {
			$link{$said,$s_said} = $weight_from_s_said;
			$link{$s_said,$said} = $weight_from_s_said;
		    }
		    else {
			$link{$said,$s_said} = $weight_from_said;
			$link{$s_said,$said} = $weight_from_said;		
		    }
		}
	    }
	    foreach $s_said (@new_links) {
		if ( defined($said2sn{$s_said}) 
		     and ( $said2sn{$s_said} != $said2sn{$said} ) ) {
		    ## The smaller social network joins the larger one and the small
		    ## network is dissolved.
		    if ( scalar(keys %{$sn2saids{$said2sn{$said}}})
			 > scalar(keys %{$sn2saids{$said2sn{$s_said}}}) ) {
			$acquiring_social_network = $said2sn{$said};
			$disappearing_social_network = $said2sn{$s_said};
		    }
		    else {
			$acquiring_social_network = $said2sn{$s_said};
			$disappearing_social_network = $said2sn{$said};
		    }		    
		    foreach $ss (keys %{$sn2saids{$disappearing_social_network}}) {
			$said2sn{$ss} = $acquiring_social_network;
			${%{$sn2saids{$acquiring_social_network}}}{$ss} = 1;
		    }
		    delete($sn2saids{$disappearing_social_network});
		    ## Note: these might chain: e.g., network 1 might disappear into
		    ## network 2; and, then network 2 disappear into network 3; etc.
		    ## all in one move (i.e., all after the posting of just one
		    ## message if it cites and/or replies to multiple messages
		    ## from multiple social networks.
		    $old_social_network{$disappearing_social_network} = $acquiring_social_network;
		}
	    }
	    &update_old_social_networks();
	}
	return;
    }
}


## UPDATE_OLD_SOCIAL_NETWORKS
##
sub update_old_social_networks
{
    my($acquiring_social_network,$osn);

    ## If any social networks are being merged into other social networks,
    ## make sure that none of the acquiring networks (i.e., those networks
    ## that will absorb the old networks) are, themselves old networks.  This
    ## might happen, because the merging process can "chain."  see the note in the 
    ## update_social_networks subroutine.  Thus, the ultimate acquiring network
    ## will be that one that does not appear as an old network.
    if (keys %old_social_network) {
	$acquiring_social_network = -1;
	foreach $osn (keys %old_social_network) {
	    if ( not(defined($old_social_network{$old_social_network{$osn}})) ) {
		$acquiring_social_network = $old_social_network{$osn};
		last;
	    }
	}
	if ( $acquiring_social_network != -1 ) {
	    ## Rewrite the %old_social_network hash so that
	    ## every old network points to the final acquiring network.
	    ## Note that, since this is updated after every message
	    ## posted to the group, we need only be concerned with what 
	    ## can happen to the social networks with the addition of
	    ## one message and one author (i.e., the player posting the
	    ## message).  Since a player can only be a member of one
	    ## social network there can only be, in principle, one
	    ## acquiring network, even if the one post dissolves
	    ## the distinctions between several existing networks.
	    foreach $osn (keys %old_social_network) {
		$old_social_network{$osn} = $acquiring_social_network;
	    }
	}
    }
    return;
}


## UPDATE_THEMES
##
sub update_themes
{
    my($current_smid) = @_;

    my($current_said,$sni,%subject_stem2saids,$smid,$said,$stem,%saids,$ssid);
    my(@shared_subject_stems,%sentence_stem2saids,@shared_sentence_stems);

    $current_said = $smid2said{$current_smid};
    $sni = $said2sn{$current_said};
    ## If the poster of the current message is not a member of
    ## a social network, then return.
    if ( scalar(keys %{$sn2saids{$sni}}) <= 1 ) { 
	$current_winning_theme = '';
	$current_winning_sentence = '';
	return;
    }
    undef %subject_stem2saids;
    ## Otherwise, examine all of the subject lines used in the thread from the
    ## current message back to the first post in the thread.
    foreach $smid (&smids_of_ancestors($current_smid)) {
	if ( defined($smid2said{$smid}) ) {
	    $said = $smid2said{$smid};
	    if ( defined($smid2nsl{$smid})
		 and defined ($nsl2stems{$smid2nsl{$smid}}) ) {
		foreach $stem (@{$nsl2stems{$smid2nsl{$smid}}}) {
		    push(@{$subject_stem2saids{$stem}},$said);
		}
	    }
	}
    }
    ## Remove the duplicates in the %subject_stem2saids entries.
    foreach $stem (keys %subject_stem2saids) {
	undef %saids;
	foreach $said (@{$subject_stem2saids{$stem}}) { $saids{$said} = 1; }
	@{$subject_stem2saids{$stem}} = keys %saids;
    }
    @shared_subject_stems = ();
    foreach $stem (keys %subject_stem2saids) { push(@shared_subject_stems,$stem); }
    ## After the sort below, the first shared stem should be the stem employed by the 
    ## greatest number of people in the thread in the subject line of their messages.
    @shared_subject_stems = sort { $#{$subject_stem2saids{$b}} <=> $#{$subject_stem2saids{$a}} } @shared_subject_stems;
    ## Hunt for a sentence in the current post that contains one of the frequent
    ## terms from the subject lines of the thread.
    foreach $stem (@shared_subject_stems) {
	foreach $ssid (@{$smid2ssids{$current_smid}}) {
	    if (not $isa_quotation{$ssid}) {		
		if ( grep { $_ eq $stem } @{$ssid2stems{$ssid}} ) {
		    if ( defined($stem2tokens{$stem}) ) {
			$current_winning_theme = ${@{$stem2tokens{$stem}}}[0];
		    }
		    else { $current_winning_theme = $stem; }
		    $current_winning_sentence = $ssid2sentence{$ssid};
		    push(@{$winning_sentences{$stem}},$ssid);
		    return;
		}
	    }
	}
    }
    ## If no winning sentence was found that might represent one of the themes
    ## mentioned in the subject lines, then look into the body of the messages
    ## for themes not mentioned in the subject lines and try again to find a
    ## winning sentence.
    undef %sentence_stem2saids;
    ## Examine all of the words used in the thread from the
    ## current message back to the first post in the thread.
    foreach $smid (&smids_of_ancestors($current_smid)) {
	if ( defined($smid2said{$smid}) ) {
	    $said = $smid2said{$smid};
	    foreach $ssid (@{$smid2ssids{$smid}}) {
		if (not $isa_quotation{$ssid}) {
		    foreach $stem (@{$ssid2stems{$ssid}}) { 
			push(@{$sentence_stem2saids{$stem}},$said);
		    }
		}
	    }
	}
    }
    ## Remove the duplicates in the %sentence_stem2saids entries.
    foreach $stem (keys %sentence_stem2saids) {
	undef %saids;
	foreach $said (@{$sentence_stem2saids{$stem}}) { $saids{$said} = 1; }
	@{$sentence_stem2saids{$stem}} = keys %saids;
    }
    @shared_sentence_stems = ();
    foreach $stem (keys %sentence_stem2saids) { push(@shared_sentence_stems,$stem); }
    ## After the sort below, the first shared stem should be the stem employed by the 
    ## greatest number of people in the thread in their own sentences.
    @shared_sentence_stems = sort { $#{$sentence_stem2saids{$b}} <=> $#{$sentence_stem2saids{$a}} } @shared_sentence_stems;
    ## Hunt for a sentence in the current post that contains one of the frequent
    ## terms from the subject lines of the thread.
    foreach $stem (@shared_sentence_stems) {
	foreach $ssid (@{$smid2ssids{$current_smid}}) {
	    if (not $isa_quotation{$ssid}) {		
		if ( grep { $_ eq $stem } @{$ssid2stems{$ssid}} ) {
		    if ( defined($stem2tokens{$stem}) ) {
			$current_winning_theme = ${@{$stem2tokens{$stem}}}[0];
		    }
		    else { $current_winning_theme = $stem; }
		    $current_winning_sentence = $ssid2sentence{$ssid};
		    push(@{$winning_sentences{$stem}},$ssid);
		    return;
		}
	    }
	}
    }
    ## Otherwise, nothing was found and the winning theme and sentence are
    ## set to nil.
    $current_winning_theme = '';
    $current_winning_sentence = '';
    return;
}


## ASSERT_SENTENCES
##
sub assert_sentences
{
    my($smid) = @_;

    my(@signature_lines,$message_text,$ln,$sentences,$sentence);
    my($token,@sentence,@modified_sentence,@line_numbers,$said,$stem);
    my(%stem_of_modified_sentence,$ssid);

    ## Find which lines are signature lines: these will be skipped.
    @signature_lines = sort {$a <=> $b} &find_line_numbers_of_signature($smid);
    $message_text = '';
    $ln = 0;
    while (exists $smidln2line{"$smid $ln"}) {
	if ( not(grep {$_ == $ln} @signature_lines) ) {
	    $message_text .= '  __line_number_'.$ln.'__  ';
	    if (defined($smidln2tline{"$smid $ln"})) {
		$message_text .= $smidln2tline{"$smid $ln"};
	    }
	}
	$ln++;
    }
    ## Get the sentences from the text of the message body.
    $sentences=get_sentences($message_text);     
    $ln = 0;
    foreach $sentence (@{$sentences}) {
	$current_simplified_sentence_id++;
	$ssid = $current_simplified_sentence_id;
	$ssid2smid{$ssid} = $smid;
	push(@{$smid2ssids{$smid}},$ssid);
	@sentence = split /\s+/,$sentence;
	@modified_sentence = ();
	@line_numbers = $ln;
	foreach $token (@sentence) {
	    if ( $token =~ /__line_number_(\d+)__/ ) {
		push(@line_numbers,$1);
	    }
	    else { push(@modified_sentence,$token); }
	}
	$ssid2sentence{$ssid} = join ' ',@modified_sentence;
	@line_numbers = sort {$a <=> $b} @line_numbers;
	## assert on which lines the sentence starts and ends
	$ssid2start{$ssid} = $line_numbers[0];
	$ssid2end{$ssid} = $line_numbers[$#line_numbers];
	## determine whether or not the sentence is a quotation
	$isa_quotation{$ssid} = 0;
	foreach $ln (@line_numbers) {
	    if ( $smidln2source{"$smid $ln"} ne "$smid $ln" ) { $isa_quotation{$ssid} = 1; }
	}
	$ln = $ssid2end{$ssid};
	@modified_sentence = &normalize_sentence(join ' ',@modified_sentence);
	$said = $smid2said{$smid};
	undef %stem_of_modified_sentence;
	foreach $stem (@modified_sentence) {
	    ## Only count a stem once per sentence. 
	    if ( not(defined($stem_of_modified_sentence{$stem})) ) {
		$stem_of_modified_sentence{$stem} = 1;
		push(@{$stem2ssids{$stem}},$ssid); 
		push(@{$ssid2stems{$ssid}},$stem);
	    }
	}
    }
    return;
}


## EXTRACT_FIELDS_AND_VALUES_FROM_MESSAGE
##
## Input:   A string containing a message formatted according to
##          RFC1036.
##
## Effects: Fill the hash %field_values; the keys of the hash are 
##          RFC1036-defined fields.
##
## Output:  A hash: %field_values
##
sub extract_fields_and_values_from_message
{
    my($message) = @_;

    my($field,$message_header,%field_values,$input_record_separator);
    my($message_header_line,@message_header_lines);
    my(@rewritten_message_header_lines,$last_line);
    my @rfc1036_fields = (
	'from:',
	'subject:',
	'date:',
	'message-id:',
	'control:',
	'references:',
	'in-reply-to:',
	'organization:',
	'newsgroups:',
	'content-type:'
    );
    $message_header = '';
    if ( $message =~ m/\n\n/g ) { $message_header = substr($message,0,pos($message)); }
    ## Make sure that the input record separator is set to "\n"
    $input_record_separator = $/;
    $/ = "\n";
    ## Remove any newlines in the values
    @message_header_lines = split /\n/,$message_header;
    $last_line = '';
    foreach $message_header_line (@message_header_lines) {
	if ( $message_header_line =~ /^(\S+\:).*$/ ) {
	    if ( $last_line ) { push(@rewritten_message_header_lines,$last_line); }
	    $last_line = $message_header_line;
	}
	else { $last_line .= $message_header_line; }
    }
    if ( $last_line ) { push(@rewritten_message_header_lines,$last_line); }
    foreach $field (@rfc1036_fields) {
	foreach $message_header_line (@rewritten_message_header_lines) {
	    if ($message_header_line =~ /^$field\s*(.*)\s*$/i) {
		$field_values{$field} = $1;
		last;
	    }
	}
    }
    $/ = $input_record_separator;
    return (%field_values);
}


## ASSERT_SIMPLIFIED_MESSAGE_ID
##
## Input:   A string: message_id
##
## Effects: Update the two hashes %mid2smid and %smid2mid
##          message_id <--> simplified_message_id
##          message_ids are the ones that appear in the messages;
##          simplified_message_ids are integers
##          If message_id is a new id, then the variable,
##          $current_simplified_message_id, is incremented.
##
## Output:  An integer: the current value of the variable 
##          $current_simplified_message_id
##
sub assert_simplified_message_id
{
    my($message_id) = @_;

    if (not(defined $mid2smid{$message_id})) {
	$current_simplified_message_id++;
	$mid2smid{$message_id} = $current_simplified_message_id;
	$smid2mid{$current_simplified_message_id} = $message_id;
    }
    return($current_simplified_message_id); 	
}


## ASSERT_SIMPLIFIED_DATE_AND_TIME
##
## Input:   Two strings: simplified_message_id 
##          and unsimplified_date_and_time
##
## Effects: Update the hash %smid2sdt
##          simplified_message_id --> simplified_date_and_time
##          simplified_date_and_time is an integer indicating
##          the date and time when the message was posted
##          (measured in seconds since the epoch, where the
##          epoch is 1/1/1970 GMT).
##
## Output:  None
##
## Notes:   This function uses the Date::Manip package;
##          specifically the methods ParseDate() and UnixDate().
##         
sub assert_simplified_date_and_time
{
    my($simplified_message_id,$unsimplified_date_and_time) = @_;

    my($date,$seconds);

    $seconds = -1;
    if (not(exists $smid2sdt{$simplified_message_id})) {
	$date = ParseDate($unsimplified_date_and_time);
	if (!$date) {
	    print STDERR "Unparsable date: $unsimplified_date_and_time\n";
	} 
	else { 
	    $seconds = &UnixDate($date,"%s"); 
	    $smid2sdt{$simplified_message_id} = $seconds;
	}
    }
    return;
}


## ASSERT_SIMPLIFIED_AUTHOR_ID
##
## Input:   A string: an author name and/or email address;
##
## Effects: Update the %ana2said and %said2ana hashes:
##          author_name_and_address <--> simplified_author_id
##          simplified_author_ids are integers
##          If author is not in the database, then the variable,
##          $current_simplified_author_id, is incremented.
##
## Output:  An integer: the current value of the variable 
##          $current_simplified_author_id, or the 
##          simplified author id previously assigned to the
##          given author name.
##
## Notes:   The author name and/or email address is unmimed using
##          the method unmime from the package MIME::WordDecoder.
##          Thus, any subsequent fetches from the database assume
##          that all RFC-1522 encoded words in the author name and address 
##          have been decoded to the local representation (which
##          should be UTF-8 characters).  Note also that all quotation
##          marks are removed from the author's name and address.
##          Also, some ISPs or email clients add a unique id to the 
##          author's email address with each post 
##          (e.g., wsack+123456@media.mit.edu).  The unique id is 
##          stripped out of the email address.
##
sub assert_simplified_author_id 
{
    my($author_name_and_address) = @_;

    my($unmimed_author_name_and_address);

    $unmimed_author_name_and_address = unmime($author_name_and_address);
    ## Delete quotation marks.
    $unmimed_author_name_and_address =~ s/"//g;                           
    ## Delete any trailing whitespace.
    $unmimed_author_name_and_address =~ s/\p{IsSpace}+$//;                
    ## Delete any unique id added by ISP
    if ( $unmimed_author_name_and_address =~ /^([^+]+)\+[0-9]+@(.+)$/ ) {  
	$unmimed_author_name_and_address = $1.'@'.$2;
    }
    if (not(exists $ana2said{$unmimed_author_name_and_address})) {
	$current_simplified_author_id++;
	$ana2said{$unmimed_author_name_and_address} = $current_simplified_author_id;
	$said2ana{$current_simplified_author_id} = $unmimed_author_name_and_address;
	&assert_said2addr($current_simplified_author_id,$unmimed_author_name_and_address);
	return($current_simplified_author_id);
    }
    else { return($ana2said{$unmimed_author_name_and_address}); }
}


## ASSERT_SAID2ADDR
##
## Input:   An email name and address
##
## Effects: Store an abbreviated form of the address in the hash %said2addr
##
sub assert_said2addr
{
    my($said,$ana) = @_;

    my($user_name);

    ($user_name) = split /@/,$ana;
    $user_name =~ /^(.*)(<)(.*)$/;      ## Find the last (rightmost) '<' on the line.
    if ($2) { $user_name = $1; }        ## Delete everything after the last '<'; and,
    $said2addr{$said} = $user_name;
    return;
}


## ASSERT_MESSAGE_AUTHORSHIP
##
## Input:   Two strings: (1) a simplified message id; and,
##          (2) a simplified author id
##
## Effects: Update the hashes %smid2said and %said2smids
##          [simplified_message_id]* <--> simplified_author_id
##          These hashes keep track of (a) the author of any given
##          message; and, (b) the ids of all of the messages
##          authored by any given poster.  If the author
##          is new, assign the author to a social_network.
##
## Output:  None
##
sub assert_message_authorship
{
    my($simplified_message_id,$simplified_author_id) = @_;
    
    if (not(exists $smid2said{$simplified_message_id})) {
	$smid2said{$simplified_message_id} = $simplified_author_id;
	push(@{$said2smids{$simplified_author_id}},$simplified_message_id);
    }
    if (not(defined($said2sn{$simplified_author_id}))) {
	$social_network_index++;
	$said2sn{$simplified_author_id} = $social_network_index;
	${%{$sn2saids{$social_network_index}}}{$simplified_author_id} = 1;
	$new_social_network{$social_network_index} = 1;
    }
    return;
}


## ASSERT_MESSAGE_HEADER
##
## Input:   One integer and three strings: a simplified message id,
##          and the newsgroups, date, and subject fields
##          from a message header.  These are the fields unrecorded
##          in other hashes.
##
## Effects: Parts of the text of the message header is saved so 
##          that the message can be more easily printed.
##
## Output:  None
##
sub assert_message_header
{
    my($smid,$newsgroups,$date,$subject) = @_;

    $smid2newsgroups{$smid} = $newsgroups;
    $smid2date{$smid} = $date;
    $smid2subject{$smid} = unmime($subject);
    return;
}


## DELETE_PREFIXES_FROM_SUBJECT_LINE
##
## Input:  String in which all characters have been normalized and downcased.
##
## Output: String in which all prefixing "re: " and "fwd: " substrings have
##         been deleted.
##
sub delete_prefixes_from_subject_line
{
    my($subject_line) = @_;

    $subject_line =~ s/re\: //gi;
    $subject_line =~ s/re\://gi;
    $subject_line =~ s/fw\: //gi;
    $subject_line =~ s/fw\://gi;
    $subject_line =~ s/fwd\: //gi;
    $subject_line =~ s/fwd\://gi;
    return $subject_line;
}


## TOKENIZE
##
## Input:   A string
##
## Output:  An array of words.  Words, contractions and hyphenations
##          in the input string are separated and punctuation is removed.
##
sub tokenize
{
    my($s) = @_;

    $s =~ s/\P{Alnum}/ /g;
    return(split /\P{Alnum}+/,$s);
}


## NORMALIZE_SENTENCE
## 
## Input:   A string
##
## Output:  A string of stemmed, tokenized words with all punctuation 
##          and stop words removed.
##
## Notes:   Words in the input string are stemmed using the Lingua::Stem
##          package.  The language assumed by the stemmer must be
##          specified on the command line: EN-UK, EN-US, FR, and DE are
##          all possibilities.  The language specified on the command
##          line also determines which stop-word list is used.
##
sub normalize_sentence
{
    my($sentence) = @_;

    my($stemmed_words,$stem,@normalized_sentence,$normalized_sentence);
    my(@words,$i);
    
    foreach $token (&tokenize($sentence)) {
	$token = lc($token);
	if (not(exists $isa_stop_word{NFD($token)})) { push(@words,$token); }
    }
    $stemmed_words = $stemmer->stem(@words);
    $i = 0;
    foreach $stem (@{$stemmed_words}) {
	if ($stem !~ /^\s*$/) {
	    push(@normalized_sentence,$stem); 
	    $token2stem{$words[$i]} = $stem;
	    if ( not(grep {$_ eq $words[$i]} @{$stem2tokens{$stem}}) ) {
		push(@{$stem2tokens{$stem}},$words[$i]);
	    }
	}
	$i++;
    }
    return(@normalized_sentence);
}
    

## ASSERT_NORMALIZED_SUBJECT_LINE
##
## Input:   An integer and a string: (1) a simplified message id; and,
##          (2) the subject line of the message.
##
## Effects: Update the hashes %smid2nsl and  %nsl2smids
##          [simplified_message_id]* <--> normalized_subject_line
##          simplified_message_ids map to one normalized_subject_line;
##          normalized_subject_lines map to a list of simplified_message_ids
##          in the normalized_subject_lines all characters are normalized
##          to UTF8 format; all the words have been stemmed;
##          all punctuation marks and stopwords have been removed; 
##          and, all "RE: ", "FWD: ", etc. prefixes have been removed.
##          Also update the hash %nsl2stems so that the contents of
##          the subject lines can be compared to the tokenized and stemmed
##          contents of the message bodies.
##
## Output:  None
##
## Notes:   "RE: " and "FWD: " prefixes are removed and the unmime method 
##          from the MIME::WordDecoder package is applied to the words in 
##          the subject line before the words are tokenized and stemmed by
##          the &normalize_sentence function.  
##
sub assert_normalized_subject_line
{
    my($simplified_message_id,$subject_line) = @_;

    if (not(exists $smid2nsl{$simplified_message_id})) {
	$subject_line = &delete_prefixes_from_subject_line(unmime($subject_line));
	$smid2nsl{$simplified_message_id} = $subject_line;
	push(@{$nsl2smids{$subject_line}},$simplified_message_id);
	if ( not(defined($nsl2stems{$subject_line})) ) {
	    @{$nsl2stems{$subject_line}} = &normalize_sentence($subject_line);
	}
    }
    return;
}


## ASSERT_REFERENCE_RELATION
##
## Input:   Three strings: one (unsimplified) message id and a space- or comma-delimited
##          list of unsimplified message ids (the contents of the 'references:' field); 
##          and/or a single message id (the contents of the 'in-reply-to:' field)
##
## Effects: One hash is updated, %mid2pid, message_id --> parent message id.  If the
##          first message id is not the id of a message that is a reply to another 
##          message, then its parent message id will be 'no_parent'.
##
## Output:  None
##
## Notes:   Since some messages may be replies to others not in the corpus, the 
##          current message  might be a reply to a message not yet processed.  So, 
##          we assert the threading information here with the full message ids and 
##          then later, in &assert_threading_relations, the same information is 
##          reasserted using simplified message ids.  Note that some messages use the 
##          'in-reply-to:' field instead of the 'references:' field to record threading 
##          information.  All message ids are coerced to lowercase before they are 
##          recorded.
##
sub assert_reference_relation
{
    my($message_id,$references_list,$in_reply_to) = @_;

    my(@references,$parent_id);

    $parent_id = 'no_parent';
    if ( ( $references_list =~ /^\s*$/ ) and ( $in_reply_to =~ /^\s*$/ ) ) { 
	$mid2pid{$message_id} = $parent_id;
    }
    elsif ( $references_list !~ /^\s*$/ ) {
	@references = split /\s+/,$references_list;
	## Some reference lists are comma- rather than space-delimited
	if ( $#references == 0 ) {
	    $comma = ',';
	    @references = split /$comma/,$references_list;
	}
	$parent_id = $references[$#references];
	if ( $parent_id =~ /^\s*$/ ) { $parent_id = 'no_parent'; }
	$mid2pid{$message_id} = lc($parent_id);
    }
    ## else if there is a 'in-reply-to:' field, but not a 'references:' field,
    ## fill in the references with the contents of the 'in-reply-to:' field.
    else {
	## Assume that the parent_id is that token in the field that is
	## bracketed by '<' and '>'.  If this is not the case, this
	## will not yield the right result.
	$in_reply_to =~ /(\<[^\>]+\>)/;
	$parent_id = $1;
	if ( $parent_id =~ /^\s*$/ ) { 
	    print STDERR "Could not find message id in in-reply-to field: $in_reply_to.\n";
	    $parent_id = 'no_parent'; 
	}	
	$mid2pid{$message_id} = lc($parent_id);
    }
    return;
}


## ASSERT_THREADING_RELATIONS
## 
## Input:   A string: a message id
##
## Effects: The hashes %smid2spid and %spid2smid are updated
##          [simplified_message_id]* <--> simplified_parent_id
##          simplified_parent_ids are simplified_message_ids
##          each simplified_message_id has zero or one parent
##          according to whether the message was (or was not)
##          written in-reply-to another message.
##          Each simplified_parent_id maps to a list of
##          simplified_message_ids according to the number of
##          replies written in response to the posted parent
##          message.  A separate hash that aggregates the smid2spid
##          relations by author id is also updated here: %repliers,
##          records which authors have replied to a given author.
##
## Output:  None
##
## Notes:   This function re-writes the information stored in
##          the hash %mid2pid and computed in the function
##          assert_reference_relation.
##
sub assert_threading_relations
{
    my($mid) = @_;
    
    my($smid,$pid,$spid,$said,$psaid);

    $smid = $mid2smid{$mid};
    $pid = $mid2pid{$mid};
    if ($pid ne 'no_parent') {
	## Note: some parent messages may not be included in the processed corpus.
	## The following line creates a simplified message id for such unincluded
	## parent messages.
	if (not (exists $mid2smid{$pid})) { 
	    $spid = &assert_simplified_message_id($pid); 
	    $smid2sdt{$spid} = -1;
	    $smid2said{$spid} = -1;
	    $said2ana{-1} = '';
	    $smid2nsl{$spid} = '';
	}
	$spid = $mid2smid{$pid};
	if (not (exists $smid2spid{$smid})) {
	    $smid2spid{$smid} = $spid;
	    push(@{$spid2smids{$spid}},$smid);
	}
	if ( defined($smid2said{$spid}) 
	     and defined ($smid2said{$smid}) ) {
	    $said = $smid2said{$smid};
	    $psaid = $smid2said{$spid};
	    if ( $said != $psaid ) { ${%{$repliers{$smid2said{$spid}}}}{$smid2said{$smid}}++; }
	}
    }
    ## If a message has no parent, it's entry in %smid2spid is ''.
    else { $smid2spid{$smid} = ''; }
    return;
}


## ASSERT_FILE_POINTERS
##
## Input:   Three integers: a simplified message id and
##          two file pointers (a start and and end).
##
## Effects: The hash %smid2fptrs is updated:
##          simpified_message_id --> start_and_end_file_pointers
##          Since the original text of each message is stored in a file,
##          each simplified_message_id has associated start and
##          end file pointers according to where, in the file,
##          the text of the message is located.
##
## Output:  None
##
sub assert_file_pointers
{
    my($simplified_message_id,$start_pointer,$end_pointer) = @_;

    if (not (exists $smid2fptrs{$simplified_message_id})) {
	push(@{$smid2fptrs{$simplified_message_id}},$start_pointer,$end_pointer);
    }
    return;
}


## ASSERT_MESSAGE_BODY_AUTHORSHIP
##
## Input:   Two integers (a simplified author id and  a simplified message id)
##          and a string (the lines of the body of a message)
##
## Effects: One hash of hashes (%said2lines) is updated; and, another hash
##          is updated: %smidln2line (this allows future, quick access to each
##          of the lines of a message);
##          For each author a separate hash is created to record
##          every line of every message posted by the author.
##          Each such hash has the following form: %line2smidslns
##          line_of_text --> [simplified_message_id and line_number]*
##          The text of each line of each message is stored as a key
##          in this hash.  The values of the hash are arrays of strings.  
##          Each string consists of a simplified_message_id and a line
##          number concatenated together, indicting where the line of text 
##          appeared in the corpus of messages.  These hashes are used to 
##          identify the signature (if any) of each author.  Entries for 
##          each of the lines of the message_body are entered in the 
##          authors' hash.
##
## Output:  None
##
sub assert_message_body_authorship
{
    my($said,$smid,$body) = @_;

    my(%line2smidslns,$line,$ln);
    
    if (defined $said2lines{$said}) { %line2smidslns = %{$said2lines{$said}}; }
    else { %line2smidslns = (); }
    $ln = 0;
    foreach $line (split /\n/,$body) {
	push(@{$line2smidslns{$line}},"$smid $ln");
	$smidln2line{"$smid $ln"} = $line;
	$ln++;
    }
    $said2lines{$said} = \%line2smidslns;
    return;
}


## ASSERT_LINE_POSITIONS
##
## Input:   An integer: a simplified author id
##
## Effects: The hash %tline2smidslns is created from the hash %smidln2line
##          truncated_line_of_text --> [simplified_message_id and line_number]*
##          Each line of text is truncated at the beginning to remove any
##          leading whitespace and/or punctuation, and then its position is
##          recorded in a list of simplified_message_id and line_number pairs
##          (each pair is recorded as a string with a space between the two
##          integers).  This hash created is later used to identify quotations.
##          hashes and is used to identify quotations.  Its inverse is also
##          created: %smidln2tline.
##          The hash %source is also created.  The keys of %source are truncated 
##          lines of text (where all of the quoting prefixes have been removed).  
##          The values are pairs of simplified message ids and line numbers.
##          This hash is also used to identify quotations.
##
## Output:  None
##
sub assert_line_positions
{
    my($smid) = @_;

    my($said,%line2smidslns,$line,$tline,@smidslns,$smidln);
    my($prefix,$ln);

    $ln = 0;
    while (defined $smidln2line{"$smid $ln"}) {
	$line = $smidln2line{"$smid $ln"};
	$tline = $line;
	## Delete any leading punctuation and any trailing whitespace.
	$tline =~ /^(.*)(>)(.*)$/;      ## Find the last (rightmost) '>' on the line.
	if ($2) { 
	    $tline = $3;                ## Delete everything up to the last '>'; and,
	    $prefix = $1.'>';           ## store the prefix under the smid and line number.
	    $prefix{$smid,$ln} = $prefix;
	}
	$tline =~ s/^\p{IsSpace}+//;    ## Delete any whitespace at the beginning of the line.
	$tline =~ s/\p{IsSpace}+$//;    ## Delete any trailing whitespace.
	## Repeatedly delete any whitespace or punctuation at the (possibly new) beginning of the line.
	for ($tline) { 1 while s/^\P{IsWord}+//; }
	## If there are any non-whitespace characters in the truncated line,
	## assert its line positions.
	if ( $tline =~ /\S/ ) {
	    ## Use the edited line as a key to the new hash.  Note that the
	    ## hash might already contain a list for the edited line, since
	    ## many different lines might reduce to the same truncated form.
	    push(@{$tline2smidslns{$tline}},"$smid $ln");
	    $smidln2tline{"$smid $ln"} = $tline;
	    ## Assume that the oldest messages are processed first.
	    if (not defined($source{$tline})) { $source{$tline} = "$smid $ln"; }
	}
	$ln++;
    }
    return;
}


## BY_REPETITIONS
## 
sub by_repetitions
{
    $#{@{$tline2smidslns{$a}}} <=>  $#{@{$tline2smidslns{$b}}};
}


## SMIDS_OF_ANCESTORS
##
## Input:   An integer: a simplified message id.
##
## Effects: Cache the results in the hash %ancestors.
##
## Output:  A list: the simplified message id of the
##          parent, the parent's parent, etc.  The hash
##          %smid2spid is traversed from the input id to
##          the root of the thread containing the message.
##
sub smids_of_ancestors
{
    my($smid) = @_;

    if ( not($smid) ) { return(); }
    if (not exists($ancestors{$smid})) { 
	push(@{$ancestors{$smid}},&smids_of_ancestors($smid2spid{$smid}),$smid);
    }
    return(@{$ancestors{$smid}});
}


## ASSERT_SOURCES
##
## Input:   An integer: a simplified message id
##
## Effects: Create the hash %smidln2source.
##          The keys of %smidln2source are simplified message ids and line
##          numbers, as are the values.  %smidln2source contains the same
##          information as %source (see above, in the documentation for
##          the function &assert_line_positions) except it also corrects 
##          some of the information in %source.  Aggregated counts are
##          also updated in this function: %citers records which authors
##          have cited others; and, %smid2sources records all of the
##          messages cited in the current message.  
##
## Output:  None
##
sub assert_sources
{
    my($smid) = @_;

    my($tline,$smidln,$ln,$earliest,$i,$source_smid,$source_ln);
    my($i_minus_one,$i_plus_one,$prefix_above,$source_smid_above,$source_ln_above);
    my($prefix_below,$source_smid_below,$source_ln_below,@tline,%prefix2sourcesmid,@ancestors);
    my($start,%block_starts,$last_source_smid,$last_source_ln,$source,%current_sources);
    my(@signature_lines,$source_author,$said,$source_said);

    ## Correct the %source hash by examining the sources of each line and 
    ## its surrounding context.  Process the corpus message-by-message.
    ## Save the result in the hash %smidln2source.
    $ln = 0;
    ## find the source of each line; 
    while (defined $smidln2line{"$smid $ln"}) {
	if (defined $smidln2tline{"$smid $ln"}) { $tline = $smidln2tline{"$smid $ln"}; }
	## if no tline is associated with the smid-ln pair, then assume the line is blank.
	else { $tline = ''; }
	if (defined $source{$tline}) { 
	    ($source_smid,$source_ln) = $source{$tline} =~ /^(\d+) (\d+)$/;
	    ## If a line has been assigned as a source another line in the
	    ## the same message, then reset its source to itself.
	    if ( $source_smid eq $smid ) { $smidln2source{"$smid $ln"} = "$smid $ln"; }
	    else { $smidln2source{"$smid $ln"} = $source{$tline}; }
	}
	## if no source exists, assign the source of a line to itself
	else { $smidln2source{"$smid $ln"} = "$smid $ln"; }
	$ln++;
    }
    ## Re-examine the sources of each line using the prefixes.
    ## If a line has a prefix and has no source,
    ## and the line has the same prefix as the line above, or the line below,
    ## and the line above or below has a source that is another message,
    ## then reassign the source of the line.
    undef %prefix2sourcesmid;
    for ($i = 0; $i < $ln; $i++) {
	$i_minus_one = $i-1;
	$i_plus_one = $i+1;
	if (defined $prefix{$smid,$i}) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;
	    if ( $source_smid == $smid ) {
		## Get the prefix (if any) of the line above the current line.
		if ( ( $i > 0 ) and (defined $prefix{$smid,$i_minus_one}) ) { 
		    $prefix_above = $prefix{$smid,$i_minus_one}; 
		    ($source_smid_above,$source_ln_above) = $smidln2source{"$smid $i_minus_one"} =~ /^(\d+) (\d+)$/;
		}
		else { $prefix_above = ''; }
		## Get the prefix (if any) of the line below the current line.
		if ( ( $i < ($ln - 1) ) and (defined $prefix{$smid,$i_plus_one}) ) { 
		    $prefix_below = $prefix{$smid,$i_plus_one}; 
		    ($source_smid_below,$source_ln_below) = $smidln2source{"$smid $i_plus_one"} =~ /^(\d+) (\d+)$/;
		}
		else { $prefix_below = ''; }
		if ( ( $prefix_above eq $prefix{$smid,$i} ) and ( $source_smid_above != $smid ) ) { 
		    $smidln2source{"$smid $i"} = $smidln2source{"$smid $i_minus_one"};
		}
		elsif ( ( $prefix_below eq $prefix{$smid,$i} ) and ( $source_smid_below != $smid ) ) { 
		    $smidln2source{"$smid $i"} = $smidln2source{"$smid $i_plus_one"};
		}
	    }
	    ## Keep track of which prefixes point to which cited messages.
	    ## Only assign a prefix to source_smid if the source_smid is not
	    ## equal to the current smid.
	    ($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;
	    if ( $source_smid != $smid ) {
		$prefix2sourcesmid{$prefix{$smid,$i}} = $source_smid;
	    }
	}
    }
    ## Go through the lines again.  If a line has a known prefix with a known source,
    ## but the line has not been assigned a source (other than itself), then assign it the
    ## source message associated with its prefix.  Match the line against every line in the
    ## source message to determine the exact line in the source message.  Sometimes this
    ## won't be possible to do, in which case the line source is simply set to 0.
    ## This procedure is necessary to correctly re-assign the source of lines that 
    ## have wrapped or been truncated.
    for ($i = 0; $i < $ln; $i++) {
	if (defined $prefix{$smid,$i}) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;
	    if ( ( $source_smid == $smid ) 
		 and ( defined($prefix2sourcesmid{$prefix{$smid,$i}}) )
		 and ( $prefix2sourcesmid{$prefix{$smid,$i}} != $smid ) ) {
		$source_smid = $prefix2sourcesmid{$prefix{$smid,$i}};
		## Get lines of source message and match them against the
		## current line (if the current line is long enough).
		$source_ln = 0;
		## First, make the current line into a pattern that can be matched against.
		if (defined $smidln2tline{"$smid $i"}) { 
		    $tline = $smidln2tline{"$smid $i"}; 
		    @tline = split /\s+/,$tline;
		    ## Only attempt to match the line if it is at least five tokens in length.
		    if ( $#tline >= 5 ) { 
			## Delete the first two tokens from the front of the line.
			for ($tline) { 1 while s/^\P{IsSpace}+//; }
			for ($tline) { 1 while s/^\p{IsSpace}+//; }
			for ($tline) { 1 while s/^\P{IsSpace}+//; }
			for ($tline) { 1 while s/^\p{IsSpace}+//; }
			## quote all the meta characters
			$tline = quotemeta($tline);
			## Now try matching the line
			while (defined $smidln2line{"$source_smid $source_ln"}) {
			    if ( $smidln2line{"$source_smid $source_ln"} =~ /$tline/ ) { last; }
			    else { $source_ln++; }
			}
		    }
		}
		## Reassign the source
		$smidln2source{"$smid $i"} = "$source_smid $source_ln";
	    }
	}
    }
    ## Go through the lines one more time.  If the source of a given line is
    ## a signature line in the sourced message, then reassign the source of
    ## the line to itself.
    for ($i = 0; $i < $ln; $i++) {
	($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;
	@signature_lines = sort {$b <=> $a} &find_line_numbers_of_signature($source_smid);
	if ( grep {$_ == $source_ln} @signature_lines ) { $smidln2source{"$smid $i"} = "$smid $i"; }
    }
    ## If the current message contains at least one line, then
    ## gather the lines into blocks with the same source.
    if ( not(exists($smidln2source{"$smid 0"})) ) { return; }
    ($last_source_smid,$last_source_ln) = $smidln2source{"$smid 0"} =~ /^(\d+) (\d+)$/;
    if ( not($last_source_smid or $last_source_ln) ) { return; }
    undef %block_starts;
    $start = 0;
    push(@{$block_starts{$start}},0);
    for ($i = 1; $i < $ln; $i++) {
	if (defined $smidln2source{"$smid $i"}) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;
	    if ( $source_smid or $source_ln ) {
		if ( $source_smid == $last_source_smid ) { push(@{$block_starts{$start}},$i); }
		else { $start = $i; }
		$last_source_smid = $source_smid;
		$last_source_ln = $source_ln;
	    }
	}
    }
    ## find the ids of the ancestors of the message;
    @ancestors = &smids_of_ancestors($smid);  
    ## Look through the lines again.  Only recognize non-ancestor sources if
    ## they are assigned to a block of lines because single, isolated lines 
    ## with non-ancestor sources are likely to be incorrectly assigned.
    foreach $start (keys %block_starts) {
	if ( defined($smidln2source{"$smid $start"}) ) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $start"}  =~ /^(\d+) (\d+)$/;
	    if ( $source_smid or $source_ln ) {
		## if the source is itself or an ancestor, leave it;
		if ( grep {$_ == $source_smid} @ancestors ) { next; }
		## else, if a line is by itself and has a non-ancestor source, change the source to itself.
		elsif ( $#{@{$block_starts{$start}}} == 0 ) { $smidln2source{"$smid $start"} = "$smid $start"; }
	    }
	}
    }
    @signature_lines = sort {$b <=> $a} &find_line_numbers_of_signature($smid);
    ## Finally, record all of the messages cited by the current message.
    for ($i = 0; $i < $ln; $i++) {
	if ( defined($smidln2source{"$smid $i"}) ) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $i"}  =~ /^(\d+) (\d+)$/;	    
	    ## A source is not counted if the source is the same as the message,
	    ## if the line is a signature line, or if the line follows a signature line.
	    if ( ( $source_smid != $smid ) 
		 and ( not(@signature_lines) or ( $i < $signature_lines[0] ) )
		 and ( not(grep {$_ == $i} @signature_lines) ) ) {
		push(@{$smid2sources{$smid}},$source_smid);
	    }
	}
    }
    ## Remove any duplicates from the entry in %smid2sources
    ## Also, remove any self-citations.
    undef %current_sources;
    foreach $source (@{$smid2sources{$smid}}) { 
	if ($smid2said{$source} != $smid2said{$smid}) {
	    $current_sources{$source} = 1; 
	}
    }
    @{$smid2sources{$smid}} = keys %current_sources;
    ## Invert %smid2sources to create %source2smids
    foreach $source (@{$smid2sources{$smid}}) { 
	if ( not(grep {$_ == $smid} @{$source2smids{$source}}) ) {
	    push(@{$source2smids{$source}},$smid); 
	}
    }	
    ## Keep track of all of the people who cite a given author
    ## (but ignore places where a player cites themself).
    if ( defined($smid2said{$smid}) ) { 
	$said = $smid2said{$smid};
	foreach $source (@{$smid2sources{$smid}}) { 
	    if ( defined($smid2said{$source}) ) {
		$source_said = $smid2said{$source};
		if ( $said != $source_said ) { ${%{$citers{$source_said}}}{$said}++; }
	    }
	}	
    }    
    return;
}


## FIND_POTENTIAL_SIGNATURE_LINES
##
## Input:   Two integers: a simplified author id and the number of 
##          messages posted by that author.
##
## Effects: None
##
## Output:  A list of strings each of which are potentially
##          lines of the author's signature.
##
## Notes:   A line is a potential signature line if it occurs
##          in $messages_posted messages.  Normally, $messages_posted
##          is equal to the total number of messages posted by the
##          author, but may be a lower number if lines which only
##          appear in a significant number (rather than all) of the
##          author's messages are to be considered as potential
##          signature lines.
##
sub find_potential_signature_lines
{
    my($said,$messages_posted) = @_;

    my(%line2smidslns,$line,@possible_signature_lines,@potential_signature_lines);
    my(%smid2lns,$smidln,$smid,$ln,@smids);

    %line2smidslns = %{$said2lines{$said}};
    ## First find all lines with at least $messages_posted appearances.
    foreach $line (keys %line2smidslns) {
	if ( (1 + $#{@{$line2smidslns{$line}}}) >= $messages_posted) {
	    push(@possible_signature_lines,$line);
	}
    }
    ## Next, for each of the possible lines, examine the message ids
    ## associated with it to make sure that every message posted by the
    ## author is represented.  This subset of possible signature lines
    ## is stored in the variable @potential_signature_lines.
    foreach $line (@possible_signature_lines) {
	undef %smid2lns;
	foreach $smidln (@{$line2smidslns{$line}}) {
	    ($smid,$ln) = $smidln =~ /^(\d+) (\d+)$/;
	    push(@{$smid2lns{$smid}},$ln);
	}
	@smids = keys %smid2lns;
	if ( $messages_posted <= (1 + $#smids) ) { 
	    push(@potential_signature_lines,$line); 
	}
    }
    return(@potential_signature_lines);
}


## FIND_SIGNATURE_LINES_IN_FIRST_MESSAGE_OF_AUTHOR
##
## Input:   An integer and a list of strings: an author's simplified
##          id and a list of potential signature lines.
## 
## Output:  A pointer to a hash containing the positions (line numbers)
##          of each of the potential signature lines in the author's
##          first posted message.
##
sub find_signature_lines_in_first_message_of_author
{
    my($said,@potential_signature_lines) = @_;

    my(%line2smidslns,$first_smid,%first_smid_lines,$line,$smidln,$smid,$ln);

    if ( defined($said2lines{$said}) 
	 and defined($said2smids{$said}[0]) ) {
	%line2smidslns = %{$said2lines{$said}};
	$first_smid = $said2smids{$said}[0];
	foreach $line (@potential_signature_lines) {
	    if (defined $line2smidslns{$line}) {
		foreach $smidln (@{$line2smidslns{$line}}) {
		    ($smid,$ln) = $smidln =~ /^(\d+) (\d+)$/;
		    if (  $smid == $first_smid )  {
			$first_smid_lines{$ln} = $line;
		    }
		}
	    }
	}
    }
    return(\%first_smid_lines);
}


## FIND_LINE_NUMBERS_OF_SIGNATURE
##
## Input:   An integer: a simplified message id
##
## Output:  An array of integers: a list of line numbers
##
## Notes:   The signatures line stored under the id of the
##          author of the message are matched against all
##          of the lines of the message.
##
sub find_line_numbers_of_signature
{
    my($smid) = @_;

    my(@signature,@signature_lines,$ln,$sig_index);

    if (defined $line_numbers_of_signature{$smid}) {
	return(@{$line_numbers_of_signature{$smid}});
    }
    $ln = 0;
    $sig_index = 0;
    if ( (defined $smid2said{$smid}) 
	 and (defined $signature{$smid2said{$smid}}) ) { 
	@signature = @{$signature{$smid2said{$smid}}}; 
	while (defined $smidln2line{"$smid $ln"}) {
	    if ( $sig_index > $#signature ) { last; }
	    elsif ( defined($smidln2line{"$smid $ln"}) 
		    and $signature[$sig_index]
		    and ( $smidln2line{"$smid $ln"} eq $signature[$sig_index] ) ) {
		push(@signature_lines,$ln);
		$sig_index++;
	    }
	    else {
		undef @signature_lines;
		$sig_index = 0;
	    }
	    $ln++;
	}
    }
    ## Pad out the signature with a line above and a line below.
    ## This is necessary to cover cases when a line is used
    ## repeatedly but not always by an author.
    if (@signature_lines) {
	unshift(@signature_lines,$signature_lines[0]-1);
	push(@signature_lines,$signature_lines[$#signature_lines]+1);
    }
    @{$line_numbers_of_signature{$smid}} = @signature_lines;
    return(@signature_lines);
}


## ASSERT_SIGNATURE
##
## Input:   An integer: a simplified author id
##
## Effects: The hash %signatures is built: simplified_message_id --> [line of text]*
##          Signature lines are identified through an examination of the 
##          hashes of the hash %said2lines.  The lines posted by each author are 
##          examined to see if any sequence of lines is repeated in every message 
##          posted by the author.  These repeated lines, included with every message, 
##          are identified to be the author's signature.  If there are several such 
##          sequences, then the longest contiguous sequences of repeated lines is 
##          chosen as the author's likely signature.  The signature of the author
##          is stored as an array of the repeated lines.
##
## Output:  None
##
sub assert_signature
{
    my($said) = @_;

    my(%line2smidslns,$messages_posted,$line);
    my($smidln,$smid,$ln,@smids,@potential_signature_lines,$first_smid,@lns);
    my(%first_smid_lines,$first_smid_lines_ptr,%sig_starts,$start,$i,$max_score,$score,$winning_start);    
    my($lines_in_first_message,$lines_in_signature,@signature,@truncated_signature,$longest);

    undef $signature{$said};
    %line2smidslns = %{$said2lines{$said}};
    $messages_posted = 1 + $#{@{$said2smids{$said}}};
    if ( $messages_posted == 1 ) { return; }
    undef @potential_signature_lines;
    @potential_signature_lines = &find_potential_signature_lines($said,$messages_posted);
    ## If no potential signature lines are found, then try to find at least
    ## one line that appears in 40% of the author's messages, if the author
    ## has posted at least 10 messages.
    if ( ( not(@potential_signature_lines) 
	   or not(grep { $_ =~ /\S/ } @potential_signature_lines) )
	 and ( $messages_posted >= 10 ) ) {
	$messages_posted = 0.4 * $messages_posted;
	$longest = 0;
	foreach $line (&find_potential_signature_lines($said,$messages_posted)) {
	    $line =~ s/^\P{IsWord}+//;    ## Delete any whitespace beginning of the line.
	    $line =~ s/\P{IsWord}+$//;    ## Delete any trailing whitespace.
	    if ( length($line) > $longest ) { @{$signature{$said}} = ($line); }
	}
	return;
    }
    $first_smid_lines_ptr = &find_signature_lines_in_first_message_of_author($said,@potential_signature_lines);
    if (not $first_smid_lines_ptr) { return; }
    %first_smid_lines = %{$first_smid_lines_ptr};
    ## Cluster the line numbers of the potential signature lines into continguous groups of lines.
    ## Clusters are indexed in a hash keyed to the line number of the start of the sequence.
    @lns = sort {$a <=> $b} keys %first_smid_lines;
    undef %sig_starts;
    $start = $lns[0];
    for ($i = 1; $i <= $#lns; $i++) {
	if ( $lns[$i] != (1 + $lns[$i-1]) ) { $start = $lns[$i]; }
	push(@{$sig_starts{$start}},$lns[$i]);
    }
    ## For the author's signature, select the longest contiguous sequence of lines that 
    ## contains the greatest number of lines with visible (i.e., non-whitespace) characters.
    $max_score = 0;
    $winning_start = -1;
    foreach $start (keys %sig_starts) {
	$score = 0;
	foreach $ln (@{$sig_starts{$start}}) {
	    if (defined($first_smid_lines{$ln}) and ($first_smid_lines{$ln} =~ /\S/)) { $score++; }
	}
	if ( $score > $max_score ) {
	    $max_score = $score;
	    $winning_start = $start;
	}
    }
    $lines_in_signature = 1 + $#{$sig_starts{$winning_start}};
    $lines_in_first_message = 0;
    $first_smid = $said2smids{$said}[0];
    while (defined $smidln2line{"$first_smid $lines_in_first_message"}) { 
	$lines_in_first_message++;
    }
    ## If a signature is the same length as a message, then the author is
    ## simply re-posting exactly the same message and does not have a
    ## valid signature.
    if ( $lines_in_signature < $lines_in_first_message ) {
	foreach $ln (@{$sig_starts{$winning_start}}) { 
	    push(@{$signature{$said}},$first_smid_lines{$ln});
	}
	## Remove any leading blank lines from the signature
	if (defined $signature{$said}) {
	    @signature = @{$signature{$said}};
	    @truncated_signature = @{$signature{$said}};
	    for ($i = 0; $i <= $#signature; $i++) {
		if ( $signature[$i] =~ /\S/ ) { 
		    @{$signature{$said}} = @truncated_signature;
		    last;
		}
		else { shift(@truncated_signature); }
	    }
	}
	## Remove any trailing blank lines from the signature
	if (defined $signature{$said}) {
	    @signature = @{$signature{$said}};
	    @truncated_signature = reverse(@{$signature{$said}});
	    for ($i = $#signature; $i >= 0; $i--) {
		if ( $signature[$i] =~ /\S/ ) { 
		    @{$signature{$said}} = reverse(@truncated_signature);
		    last;
		}
		else { shift(@truncated_signature); }
	    }
	}
    }
    return;
}


## UPDATE_SCORES_CENTRALITY_BY_PROXIMITY_VERSION
##
## Input:   none
##
## Effects: Update every player's %said2score, and the $min_score, 
##          the $max_score, and $current_winner variables.
## 
## Output:  none
##
## Notes:   Players are scored according to a social network
##          measure based on proximity.  The best score is
##          acheived by the player who is cited by or replied
##          to by every other player _and_ who, reciprocates, by
##          replying to or citing every other player.
##          In the social network, two players are linked if
##          they have reciprocally cited and/or replied to one
##          another.  Counts of reciprocated quotes and replies
##          are recorded and weight the links between actors in
##          in the social networks.  For the purposes of the scoring
##          algorithm, the inverse of the link weight is used.
##          Thus, a player can increase their centrality rating by
##          corresponding frequently with only a few other players,
##          but -- by so doing -- they increase everyone else's score
##          who is connect to them or their correspondent.  Also, since
##          one's global score depends upon the size of one's social 
##          network correspondence with only one other person does not
##          produce a high score.
##
## Ref:     "Centralie et pourvoir," Alain Degenne et Michel Forse,
##          _Les Reseaux Sociaux_ (Paris: Armand Colin, 1994).
##
## NB:      This subroutine has not yet been debugged and may contain 
##          errors.
## 
sub update_scores_centrality_by_proximity_version
{
    my($sni,$graph,@graphs,@graph_sizes,@social_network_indices,$i);
    my($s_said,$e_said,$sssp,$said,$total_distance,@es);

    @social_network_indices = sort { $a <=> $b } keys %sn2saids;
    ## Build a graph for every social network.
    foreach $sni (@social_network_indices) {
	$graph = new Graph::Undirected;
	push(@graph_sizes,scalar(keys %{$sn2saids{$sni}}));
	$graph->add_vertices(keys %{$sn2saids{$sni}});
	foreach $s_said (keys %{$sn2saids{$sni}}) {
	    foreach $e_said (keys %{$sn2saids{$sni}}) {
		if ( defined($link{$s_said,$e_said}) ) {
		    ## A link's distance is inversely proportional to the
		    ## number of exchanges (citations or replies) between
		    ## the two players.
		    $graph->add_weighted_edge($s_said,(1.0/$link{$s_said,$e_said}),$e_said);
		}
	    }
	}
	push(@graphs,$graph);
    }
    ## Score each player in each social network.
    $i = 0;
    foreach $graph (@graphs) {
	foreach $said ($graph->vertices()) {
	    ## Use Dijkstra's algorithm to compute the single-source shortest paths
	    ## from each player in the social network to every other player in
	    ## the network.
	    $sssp = $graph->SSSP_Dijkstra($said);
	    $total_distance = 0;
	    @es = $sssp->edges();
	    while (@es) { 
		($s_said,$e_said,@es) = @es;
		$total_distance += $sssp->get_attribute('weight',$s_said,$e_said);
	    }
	    ## Multiply the total distance my the size of the social network so that
	    ## centrality measures from the different social networks can be compared
	    ## and so that centrality in a large network counts for more than centrality
	    ## in a small network.
	    if ( $total_distance > 0 ) {
		$said2score{$said} = ( ( ($graph_sizes[$i] * $graph_sizes[$i]) - $graph_sizes[$i] ) / $total_distance );
	    }
	    else { $said2score{$said} = 0; } 
	    if ( $said2score{$said} > $max_score ) { 
		$max_score = $said2score{$said}; 
		$current_winner = $said;
	    }
	    if ( $said2score{$said} < $min_score ) { $min_score = $said2score{$said}; }
	}
	$i++;
    }    
}


## UPDATE_SCORES_CENTRALITY_BY_DEGREE_VERSION
##
## Input:   none
##
## Effects: Update every player's %said2score, and the $min_score, 
##          the $max_score, and $current_winner variables.
## 
## Output:  none
##
## Notes:   This version computes the simplest version of
##          centrality for social networks: centrality is
##          computed as the mnumber of other players each
##          player is connected to.  Consequently, with this
##          version, players win points by corresponding
##          with new people in the discussion, but gain no
##          points for continuing to correspond with people
##          they have already been talking with.
##          Actually, this implementation is a bit more
##          nuanced.  The first time a pair of players
##          cites and/or replies to one another, they
##          received full credit; the second time they do so,
##          they receive an additional 90% of the initial
##          points; the third time, 80% of the initial points;
##          etc., until the tenth and subsequent exchanges
##          yield them no points.  However, since posts to the
##          group will "fade out" after N (e.g., 500) new 
##          subsequent posts have been made, a pair of players
##          will need to sustain their exchange in order to
##          keep all of their possible points.
##
sub update_scores_centrality_by_degree_version
{
    my($sni,@social_network_indices,$s_said,$e_said,$i,$said,@saids_with_max_score);

    undef %said2score;
    @social_network_indices = sort { $a <=> $b } keys %sn2saids;
    ## Find the degree of every player.  Record the degree as the player's
    ## score.  Note that the use of the link weight in calculation (as
    ## explained in the comments above.)
    foreach $sni (@social_network_indices) {
	foreach $s_said (keys %{$sn2saids{$sni}}) {
	    $said2score{$s_said} = 0;
	    foreach $e_said (keys %{$sn2saids{$sni}}) {
		if ( defined($link{$s_said,$e_said}) ) {
		    $link_weight = $link{$s_said,$e_said};
		    for ( $i = 10; $i > 0; $i--) {
			if ( $link_weight > 0 ) {
			    $said2score{$s_said} += $i;
			    $link_weight--;
			}
			else { last; }
		    }
		}
	    }
	    if ( $said2score{$s_said} > $max_score ) { 
		$max_score = $said2score{$s_said}; 
		$current_winner = $s_said;
	    }
	    if ( $said2score{$s_said} < $min_score ) { $min_score = $said2score{$s_said}; }
	}
    }
    ## If several players are tied for the winning score, pick the player 
    ## who was first on the field (i.e., the one with the lowest integer as
    ## a simplified author id).
    @saids_with_max_score = ();
    foreach $said (keys %said2score) {
	if ( $said2score{$said} == $max_score ) { push(@saids_with_max_score,$said); }
    }
    @saids_with_max_score = sort { $a <=> $b } @saids_with_max_score;    
    $current_winner = $saids_with_max_score[0];
}


## UPDATE_SCORES
##
## Input:   none
##
## Effects: Update every player's %said2score, and the $min_score, 
##          the $max_score, and $current_winner variables.
##          Also, record every player's previous_score before
##          updating the %said2score hash.  Convert all scores to
##          percentages and save in the hash %said2percentage_score.
## 
## Output:  none
##
## Notes:   This subroutine is a wrapper for the actual
##          score calculating subroutines.  The design is meant
##          to facilitate the experimentation with different
##          scoring algorithms.  Currently, two possible
##          scoring algorithms based on the measurement of
##          "centrality" in the social networks literature
##          have been implemented.
##
sub update_scores
{
    my($said,$score_range,$log_score_range);
    
    ## Save the previous scores for each author.
    undef %said2previous_score;
    foreach $said (keys %said2smids) { 
	if ( defined($said2score{$said}) ) { $said2previous_score{$said} = $said2score{$said}; }
	else { $said2previous_score{$said} = 0; }
    }
    ## Calculate the new scores.
    undef %said2score;
    $max_score = 0;
    $min_score = 1;
    $previous_winner = $current_winner;
    $previous_winning_theme = $current_winning_theme;
    ## If desired, replace this with a call to a different scoring subroutine.
    &update_scores_centrality_by_degree_version();
    ## Save the previous percentage scores for each author.
    undef %said2previous_percentage_score;
    foreach $said (keys %said2score) { 
	if (defined($said2percentage_score{$said})) {
	    $said2previous_percentage_score{$said} = $said2percentage_score{$said}; 
	}
    }
    ## Convert each score to a percentage.  Scores are plotted in a log scale
    ## to allow for the usual Zipf distribution of citations and replies (i.e.,
    ## only a handful of people are frequently cited and/or replied to and the
    ## majority are rarely cited or replied to; cf., Lodka's Law in the information
    ## science literature.
    undef %said2percentage_score;
    $score_range = 1 + $max_score - $min_score;
    $log_score_range = log($score_range);
    foreach $said (sort {$a <=> $b} keys %said2smids) {
	if ( $log_score_range != 0 ) {
#	    $said2percentage_score{$said} = int( 100 * ( log(1 + $said2score{$said}) / $log_score_range) );
	    $said2percentage_score{$said} = int( 100 * ( $said2score{$said} / $score_range ) );
	}
	else { $said2percentage_score{$said} = 0; }
    }
    return;
}


## ANGLE_DIFFERENCE
##
sub angle_difference
{
    my($angle1,$angle2) = @_;

    ## It is assumed that a circle has $radius_of_circle degrees.  Find the
    ## least number of degrees between the two angles.  This
    ## entails deciding whether to go clockwise or counter-
    ## clockwise.
    if ( ( $angle1 - $angle2 ) > ( ( $angle2 + $radius_of_circle ) - $angle1 ) ) {
	return(($angle2 + $radius_of_circle) - $angle1);
    }
    else { return($angle1 - $angle2); }
}


## ANGLE_SUBTRACT
##
sub angle_subtract
{
    my($angle1,$angle2) = @_;
    
    if ( $angle1 >= $angle2 ) { return($angle1-$angle2); }
    else { return(($angle1-$angle2)+$radius_of_circle); }
}


## ANGLE_ADD
##
sub angle_add
{
    my($angle1,$angle2) = @_;

    return(($angle1+$angle2) % $radius_of_circle);
}


## WRITE_HTML_HEADER_FOR_MESSAGE
##
sub write_html_header_for_message
{
    my($smid) = @_;
    
    my($html,$from,$newsgroups,$date,$subject,$mid,$parent,$parent_smid,@replies,$replies);
    my($reply_smid,@cites,$cites,$cite_smid,@cited_bys,$cited_bys,$cited_by_smid);

    ## HTML HEAD
    $html = '<HTML>'."\n\t".'<HEAD>'."\n\t\t".'<TITLE>message '.$smid.'</TITLE>';
    $html .= "\n\t\t".'<LINK type="text/css" rel="stylesheet" href="message_style.css">'."\n\t".'</HEAD>';
    $html .= "\n\t".'<BODY>'."\n";
    $html .= "\n\t".'<SCRIPT>window.resizeTo((screen.availWidth/2),screen.availHeight);</SCRIPT>'."\n";
    ## Table containing fields of message header
    $html .= "\n\t".'<TABLE>'."\n\t\t".'<TBODY>';
    ## Subject
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>SUBJECT:&nbsp;</B></TD>';
    if (defined $smid2subject{$smid}) {
	$subject = $smid2subject{$smid};
	$subject = HTML::Entities::encode_entities($subject);
	$subject =~ s/\s/\&nbsp\;/g;
    }
    else { $subject = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$subject.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## From
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>SENDER:&nbsp;</B></TD>';
    if ( (defined $smid2said{$smid}) 
	 and (defined $said2ana{$smid2said{$smid}}) ) {
	$from = $said2ana{$smid2said{$smid}};
	$from = HTML::Entities::encode_entities($from);
	$from =~ s/\s/\&nbsp\;/g;
    }
    else { $from = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$from.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## Newsgroups
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>NEWSGROUPS:&nbsp;</B></TD>';
    if (defined $smid2newsgroups{$smid}) {
	$newsgroups = $smid2newsgroups{$smid};
	$newsgroups = HTML::Entities::encode_entities($newsgroups);
	$newsgroups =~ s/\s/\&nbsp\;/g;
    }
    else { $newsgroups = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$newsgroups.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## Date
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>DATE:&nbsp;</B></TD>';
    if (defined $smid2date{$smid}) {
	$date = $smid2date{$smid};
	$date = HTML::Entities::encode_entities($date);
	$date =~ s/\s/\&nbsp\;/g;
    }
    else { $date = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$date.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## Message-ID
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>ID:&nbsp;</B></TD>';
    if (defined $smid2mid{$smid}) {
	$mid = $smid2mid{$smid};
	$mid = HTML::Entities::encode_entities($mid);
	$mid =~ s/\s/\&nbsp\;/g;
	$mid = '&#91;'.$smid.'&#93; '.$mid;
    }
    else { $mid = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$mid.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## References
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>PREVIOUS:&nbsp;</B></TD>';
    if ( (defined $smid2spid{$smid}) and $smid2spid{$smid} ) {
	$parent_smid = $smid2spid{$smid};
	$parent = '&#91;'.$parent_smid.'&#93;';
	$parent = '<A href="'.$parent_smid.'.html">'.$parent.'</A>';
    }
    else { $parent = ''; }
    $html .= "\n\t\t\t".'<TD class=headertext>'.$parent.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## Replies
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>NEXT:&nbsp;</B></TD>';
    undef @replies;
    if (defined $spid2smids{$smid}) {
	foreach $reply_smid (sort { $a <=> $b } @{$spid2smids{$smid}}) {
	    push(@replies,'<A href="'.$reply_smid.'.html">&#91;'.$reply_smid.'&#93;</A>');
	}
    }
    $replies = join '&#44;&nbsp;',@replies;
    $html .= "\n\t\t\t".'<TD class=headertext>'.$replies.'</TD>';
    $html .= "\n\t\t".'</TR>';
    ## Cites
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>CITES:&nbsp;</B></TD>';
    undef @cites;
    if (defined $smid2sources{$smid}) {
	foreach $cite_smid (sort { $a <=> $b } @{$smid2sources{$smid}}) {
	    push(@cites,'<A href="'.$cite_smid.'.html">&#91;'.$cite_smid.'&#93;</A>');
	}
    }
    $cites = join '&#44;&nbsp;',@cites;
    $html .= "\n\t\t\t".'<TD class=headertext>'.$cites.'</TD>';
    $html .= "\n\t\t".'</TR>';

    ## Cited-by
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>CITED BY:&nbsp;</B></TD>';
    undef @cited_bys;
    if (defined $source2smids{$smid}) {
	foreach $cited_by_smid (sort { $a <=> $b } @{$source2smids{$smid}}) {
	    push(@cited_bys,'<A href="'.$cited_by_smid.'.html">&#91;'.$cited_by_smid.'&#93;</A>');
	}
    }
    $cited_bys = join '&#44;&nbsp;',@cited_bys;
    $html .= "\n\t\t\t".'<TD class=headertext>'.$cited_bys.'</TD>';
    $html .= "\n\t\t".'</TR>';

    ## Indices
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext><B>MESSAGES BY:&nbsp;</B></TD>';
    $html .= "\n\t\t\t".'<TD class=headertext>';
    $html .= '<A href="http://'.$cgi_url.'/Agonistics/list_messages_by_id.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'">id</A>&#44;&nbsp;';
    $html .= '<A href="http://'.$cgi_url.'/Agonistics/list_messages_by_date.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'">date</A>&#44;&nbsp;';
    $html .= '<A href="http://'.$cgi_url.'/Agonistics/list_messages_by_subject.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'">subject</A>&#44;&nbsp;';
    $html .= '<A href="http://'.$cgi_url.'/Agonistics/list_messages_by_sender.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'">sender</A>&#44;&nbsp;';
    $html .= '<A href="http://'.$cgi_url.'/Agonistics/list_messages_by_thread.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'">thread</A>';
    $html .= '</TD>'."\n\t\t".'</TR>';
    
    ## Reply button for newsgroup discussions
    if ( $nntp_server ) {
	$html .= "\n\t".'<FORM METHOD="POST" ACTION="http://'.$cgi_url.'/Agonistics/write_message.pl" ENCTYPE="multipart/form-data" TARGET="reply_to_message_'.$smid.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="url" VALUE="'.$documents_url.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="cgi_url" VALUE="'.$cgi_url.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="archive" VALUE="'.$archive_name.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="smid" VALUE="'.$smid.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="id" VALUE="'.$smid2mid{$smid}.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="subject" VALUE="'.$smid2subject{$smid}.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="news_group" VALUE="'.$news_group.'">';
	$html .= "\n\t\t".'<INPUT TYPE="hidden" NAME="language_locale" VALUE="'.$language_locale.'">';
    }
    ## Reply button for email-based discussions
    ## This should also contain in-reply-to and subject fields, but Mozilla
    ## seems to have a hard time making sense of these other fields.
    else { $html .= "\n\t".'<FORM ACTION="mailto:'.$mailing_list_address.'">';  }
    $html .= "\n\t\t".'<TR align="left">';
    $html .= "\n\t\t\t".'<TD class=headerfieldtext>';
    my $button_value = '';
    if ( $language_locale =~ /^EN/ ) { $button_value = "REPLY"; }
    elsif ( $language_locale =~ /^FR/ ) { $button_value = "R\x{c9}PONDRE"; }
    elsif ( $language_locale =~ /^DE/ ) { $button_value = "ANTWORTEN"; }
    $html .= "\n\t\t\t\t".'<INPUT name="reply_to_message" class=buttonstyle type="SUBMIT" value="'.$button_value.'">';
    $html .= "\n\t\t\t".'</TD>';
    $html .= "\n\t\t\t".'<TD>&nbsp;</TD>';
    $html .= "\n\t\t".'</TR>';
    $html .= "\n\t".'</FORM>';

    ## End of table
    $html .= "\n\t\t".'</TBODY>'."\n\t".'</TABLE>'."\n\t".'<BR><BR>';

    return($html);
}


## WRITE_HTML_FOR_MESSAGE
##
sub write_html_for_message
{
    my($smid) = @_;

    my($ln,$header,$body,$next_line,$line_prefix,$end);
    my(@signature_lines,$first);

    ## First, find which lines are signature lines.
    @signature_lines = sort {$a <=> $b } &find_line_numbers_of_signature($smid);
    ## Correct for the padding done in &find_line_numbers_of_signature;
    pop(@signature_lines);
    shift(@signature_lines);
    ## Next, write the head and body of the message as HTML.
    $ln = 0;
    $header = &write_html_header_for_message($smid);
    $body = '';
    while (defined $smidln2line{"$smid $ln"}) {
	$next_line = $smidln2line{"$smid $ln"};
	$next_line = HTML::Entities::encode_entities($next_line);
	$next_line =~ s/\s/\&nbsp\;/g;
	## Every line of every message gets an anchor name so that
	## it can be linked to, if necessary, from another message.
	$line_prefix = '<A name="'.$ln.'"';
	## If a line is a signature line, then it should be highlighted,
	## but not hyperlinked.
	if ( ($#signature_lines >= 0) and ( $ln == $signature_lines[0] ) ) {
	    $line_prefix .= ' class=signaturetext';
	    ($first,@signature_lines) = @signature_lines;
	}
	## Else if a line is not its own source, then it is a quote from elsewhere
	## and will be hyperlinked.
	elsif ( defined($smidln2source{"$smid $ln"})
		and ( $smidln2source{"$smid $ln"} ne "$smid $ln" ) ) {
	    ($source_smid,$source_ln) = $smidln2source{"$smid $ln"} =~ /^(\d+) (\d+)$/;
	    if ( $source_smid or $source_ln ) {
		$line_prefix .= ' class=linktext';
		$line_prefix .= ' href="'.$source_smid.'.html#'.$source_ln.'"';
	    }
	}
	## Otherwise, the line is a line of text original to the message.
	else { $line_prefix .= ' class=messagetext'; }
	$line_prefix .= '>';
	$next_line = "\n\t".$line_prefix.$next_line.'</A><BR>';
	$body .= $next_line;
	$ln++;
    }
    $end = "\n\t".'</BODY>'."\n".'</HTML>';
    open(OUT,'>'.$message_directory.$slash.$smid.'.html');
    print OUT $header.$body.$end;
    close(OUT);
    return;
}


## FIND_ADJACENT_SOCIAL_NETWORK_FOR_INTERFACE
##
## Input:   A list of simplified message ids (integers) that represent 
##          either the messages quoted or replied to by the current 
##          message.  
##
## Output:  An integer: a social network index.  If the message posted
##          by the member of the new social network, is a message that
##          replies to or cites the message of an author already in
##          another social network, return the index of the other
##          social network; otherwise, return 0.
##
## Notes:   The purpose of this function is to try to find a likely
##          place to position the player in a new social network in
##          the interface.  The player in the new social network is
##          assumed to be the network's sole member and is the
##          author of the message just posted to the newsgroup.
##
sub find_adjacent_social_network_for_interface
{
    my(@source_smids) = @_;

    my($source_smid,$sn);

    ## The following is necessary because some authors don't
    ## belong to social networks because they are authors
    ## of messages that are referenced, but not included
    ## in the archive being analyzed.
    $sn = -1;
    foreach $source_smid (@source_smids) {
	if ( defined($smid2said{$source_smid})
	     and defined($said2sn{$smid2said{$source_smid}}) ) {
	    $sn = $said2sn{$smid2said{$source_smid}};
	    last;
	}
    }
    return($sn);
}


## EVENLY_MIX
##
sub evenly_mix
{
    my($ptr_to_part1,$ptr_to_part2,$ptr_to_whole) = @_;

    my(@part1,@part2,@whole,@new_whole,$w,$p1,$p2,$i,$j,$part1_after_part2);
    
    @part1 = @{$ptr_to_part1};
    @part2 = @{$ptr_to_part2};
    if ( scalar(@part2) > scalar(@part1) ) {
	@part1 = @{$ptr_to_part2}; ## assumed to be more numerous
	@part2 = @{$ptr_to_part1};
    }
    @whole = @{$ptr_to_whole};
    if ( scalar(@part2) <= 1 ) { return(@whole); }
    ## Rotate the members of part1 until the one that used to 
    ## follow the first elements of part2 is at the front
    ## of @part1.
    ## First, rotate @whole until the first element of @part2
    ## is at the front.
    while ( $whole[0] != $part2[0] ) {
	$w = shift @whole;
	push(@whole,$w);
    }
    $part1_after_part2 = 'undefined';
    ## Second, find the element of @part1 that comes
    ## after the first element opf @part2
    for ($i = 1; $i <= $#whole; $i++) {
	if ( grep { $_ == $whole[$i] } @part1 ) {
	    $part1_after_part2 = $whole[$i];
	    last;
	}
    }
    ## Third, rotate @part1 around until the
    ## element following the first of @part2
    ## is at its head.
    if ( $part1_after_part2 ne 'undefined' ) {
	while ( $part1[0] != $part1_after_part2 ) {
	    $p1 = shift @part1;
	    push(@part1,$p1);
	}
    }	    
    ## Now, interlace @part1 and @part2 
    $increment = int(scalar(@part1) / scalar(@part2));
    @new_whole = ();
    foreach $p2 (@part2) {
	push(@new_whole,$p2);
	for ( $j = 0; $j < $increment; $j++ ) {
	    if ( @part1 ) { 
		push(@new_whole,$part1[0]);
		shift @part1;
	    }
	}
    }
    if ( @part1 ) { push(@new_whole,@part1); }
    return(@new_whole);
}


## REDISTRIBUTE_LARGE_SOCIAL_NETWORKS
##
sub redistribute_large_social_networks 
{
    my(@social_network_indices) = @_;

    my($large,@large,@small,@invisible,@small_and_invisible,$is_visible);
    my($sni,$said);

    $large = 2;  ## any network with two or more players is considered "large."
    @large = ();
    @small = ();
    @invisible = ();
    foreach $sni (@social_network_indices) {
	$is_visible = 0;
	foreach $said (keys %{$sn2saids{$sni}}) {
	    if ( &player_is_visible($said) ) {
		$is_visible = 1;
		last;
	    }
	}
	if ( $is_visible ) {
	    if ( scalar(keys %{$sn2saids{$sni}}) >= $large ) { push(@large,$sni); }
	    else { push(@small,$sni); }
	}
	else { push(@invisible,$sni); }
    }
    ## Mix the invisibles into the smalls
    @small_and_invisible = &evenly_mix(\@small,\@invisible,\@social_network_indices);
    ## Now mix the larges into the small and invisible
    return(&evenly_mix(\@small_and_invisible,\@large,\@social_network_indices));
}


## INSERT_NEW_SOCIAL_NETWORKS_FOR_INTERFACE
##
## Input:  None
##
## Output: A list of sorted social network indices with any new social
##         networks inserted into the list.
##
## Notes:  Insert any new social networks into the ordered list of networks.  
##         There should only be one, at the most, in principle, such new 
##         network after one new message posted to the newsgroup.
##
sub insert_new_social_networks_for_interface
{
    my(@social_network_indices,$sni,$esn,$i);

    ## Produce a list of continuing social networks that does not include
    ## any of the disappearing (i.e., old) social networks.  The old
    ## social networks have been taken out of the %said2sn
    ## and sn2saids hashes, but are still keys in the
    ## %sn2theta hash.  Take them out of this hash too.
    @social_network_indices = ();
    foreach $sni (keys %sn2saids) {
	if ( grep { $_ == $sni } (keys %old_social_network) ) { 
	    delete($sn2theta{$sni});
	    next;
	}
	if ( grep { $_ == $sni } (keys %new_social_network) ) { next; }
	else { push(@social_network_indices,$sni); }
    }
    ## Sort the existing social networks into counter-clockwise order.
    @social_network_indices = sort { $sn2theta{$a} <=> $sn2theta{$b} } @social_network_indices;
    ## Re-sort the indices so that the large social networks are equally distributed.
    @social_network_indices = &redistribute_large_social_networks(@social_network_indices);
    ## If there is currently only one social network, return it.
    if ( not(@social_network_indices) ) { return(keys %new_social_network); }
    ## Otherwise, insert any new social network indices into this ordered list using
    ## &find_adjacent_social_network_for_interface to find appropriate insertion points.
    foreach $new_social_network (keys %new_social_network) {
	@source_smids = ();
	foreach $said (keys %{$sn2saids{$new_social_network}}) {
	    foreach $smid (@{$said2smids{$said}}) {
		if (defined($smid2sources{$smid}) ) {
		    push(@source_smids,@{$smid2sources{$smid}});
		}
		if (defined($smid2spid{$smid})) { push(@source_smids,$smid2spid{$smid}); }
	    }
	}
	$esn = &find_adjacent_social_network_for_interface(@source_smids); 
	## No insertion point found because current message is neither a 
	## reply to nor does it include a citation from another message.
	if ( $esn == -1 ) { push(@social_network_indices,$new_social_network); }  
	## Otherwise, rotate to position the $esn at the start of the ordered list
	## and insert the new social network next to it by pushing onto the
	## end of the list (remember the order is circular).
	else {
	    $i = 0;
	    while ( $social_network_indices[0] != $esn ) {
		if ( $i > $#social_network_indices ) {
		    print STDOUT "\nError: Problem with the social network indices!\n";
		    last;
		}
		$sni = shift @social_network_indices;
		push(@social_network_indices,$sni);
		$i++;
	    }
	    push(@social_network_indices,$new_social_network);
	}
    }
    return(@social_network_indices);
}

## WRITE_HTML_FOR_LEADERS_LIST
##
sub write_html_for_leaders_list
{
    my($current_smid) = @_;

    my($html,$said,@saids,$messagesCGIScript,$addr);

    $messagesCGIScript = 'http://'.$cgi_url.'/Agonistics/index_for_player.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'&msg='.$current_smid.'&id=';
    $html .= "\n\t".'<TABLE align="right" style="width: 20%; text-align: right;">'."\n\t\t".'<TBODY>';
    $html .= "\n\t\t".'<TR align="right">';
    if ( $language_locale =~ /^EN/ ) { 
	$html .= "\n\t\t\t".'<TD align="right" class=headerfieldtext><B>c&nbsp;u&nbsp;r&nbsp;r&nbsp;e&nbsp;n&nbsp;t&nbsp;&nbsp;&nbsp;l&nbsp;e&nbsp;a&nbsp;d&nbsp;e&nbsp;r&nbsp;s</B></TD>';
    }
    elsif ( $language_locale =~ /^FR/ ) { 
	$html .= "\n\t\t\t".'<TD align="right" class=headerfieldtext><B>c&nbsp;l&nbsp;a&nbsp;s&nbsp;s&nbsp;e&nbsp;m&nbsp;e&nbsp;n&nbsp;t</B></TD>';
    }
    elsif ( $language_locale =~ /^DE/ ) { 
	$html .= "\n\t\t\t".'<TD align="right" class=headerfieldtext><B>L&nbsp;i&nbsp;s&nbsp;t&nbsp;e&nbsp;&nbsp;&nbsp;d&nbsp;e&nbsp;r&nbsp;&nbsp;&nbsp;S&nbsp;p&nbsp;i&nbsp;e&nbsp;l&nbsp;e&nbsp;r</B></TD>';
    }
    $html .= "\n\t\t".'</TR>';
    @saids = sort { if ( $said2score{$a} == $said2score{$b} ) { lc($said2addr{$a}) cmp lc($said2addr{$b}); } else { $said2score{$b} <=> $said2score{$a} } } keys %said2score;
#    if ( $#saids >= 9 ) { @saids = @saids[0..9]; }
    foreach $said (@saids) {
	if ( defined($said2addr{$said}) and defined($said2score{$said}) ) {
	    $addr = HTML::Entities::encode_entities($said2addr{$said});
	    $html .= "\n\t\t".'<TR align="right">';
	    $html .= "\n\t\t\t".'<TD align="right" id="'.$said.'" class=scorelisttext onMouseOver="turnLabelOnFromLeadersList(this,event);"><B><A href="'.$messagesCGIScript.$said.'" target="messages_posted_by_'.$said2addr{$said}.'">'.$addr.'</A>&nbsp;'.$said2score{$said}.'</B></TD>';
	    $html .= "\n\t\t".'</TR>';
	}
    }
    $html .= "\n\t\t".'</TBODY>';
    $html .= "\n\t".'</TABLE>';
    return($html);
}


## WRITE_HTML_FOR_WINNING_SENTENCES
##
sub write_html_for_winning_sentences
{
    my($current_smid) = @_;

    my($html,@stems,$stem,$theme,$ssid,@winning_ssids,$i,$j,$ws,$said,@saids);
    my($messagesCGIScript,$messagesURL,$addr,$sentence_text);

    $messagesCGIScript = 'http://'.$cgi_url.'/Agonistics/index_for_player.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'&msg='.$current_smid.'&id=';
    $messagesURL = 'http://'.$documents_url.'/Agonistics/'.$archive_name.'/Interface/Messages/';
    $html .= "\n\t".'<TABLE align="left" style="width: 20%; text-align: left;" >'."\n\t\t".'<TBODY>';
    $html .= "\n\t\t".'<TR align="left">';
    if ( $language_locale =~ /^FR/ ) { 
	$html .= "\n\t\t\t".'<TD align="left" class=headerfieldtext><B>t&nbsp;h&nbsp;&egrave;&nbsp;m&nbsp;e&nbsp;s&nbsp;&nbsp;&nbsp;f&nbsp;r&nbsp;&eacute;&nbsp;q&nbsp;u&nbsp;e&nbsp;n&nbsp;t&nbsp;s</B></TD>';
    }
    elsif ( $language_locale =~ /^EN/ ) { 
	$html .= "\n\t\t\t".'<TD align="left" class=headerfieldtext><B>s&nbsp;t&nbsp;a&nbsp;t&nbsp;e&nbsp;m&nbsp;e&nbsp;n&nbsp;t&nbsp;s&nbsp;&nbsp;&nbsp;a&nbsp;b&nbsp;o&nbsp;u&nbsp;t<BR>f&nbsp;r&nbsp;e&nbsp;q&nbsp;u&nbsp;e&nbsp;n&nbsp;t&nbsp;&nbsp;&nbsp;t&nbsp;h&nbsp;e&nbsp;m&nbsp;e&nbsp;s</B></TD>';
    }
    elsif ( $language_locale =~ /^DE/ ) { 
	$html .= "\n\t\t\t".'<TD align="left" class=headerfieldtext><B>T&nbsp;h&nbsp;e&nbsp;m&nbsp;e&nbsp;n&nbsp;&nbsp;&nbsp;d&nbsp;e&nbsp;r<BR>D&nbsp;i&nbsp;s&nbsp;k&nbsp;u&nbsp;s&nbsp;s&nbsp;i&nbsp;o&nbsp;n</B></TD>';
    }
    $html .= "\n\t\t".'</TR>';
    @stems = sort { $#{@{$winning_sentences{$b}}} <=> $#{@{$winning_sentences{$a}}} } keys %winning_sentences;
    @saids = sort { $said2score{$b} <=> $said2score{$a} } keys %said2score;
    $i = 0;
    foreach $stem (@stems) {
	if ( $i > 2 ) { last; }
	@winning_ssids = ();
	$j = 0;
	foreach $said (@saids) {
	    @{$winning_sentences{$stem}} = sort { $smid2sdt{$ssid2smid{$b}} <=> $smid2sdt{$ssid2smid{$a}} } @{$winning_sentences{$stem}};
	    foreach $ws (@{$winning_sentences{$stem}}) {
		if ($smid2said{$ssid2smid{$ws}} == $said) { 
		    push(@winning_ssids,$ws); 
		    $j++;
		}
		if ( $j >= 10 ) { last; }
	    }
	    if ( $j >= 10 ) { last; }	    
	}
	if ( not(@winning_ssids) ) { next; }
	$i++;
	if ( defined($stem2tokens{$stem}) ) { $theme = ${@{$stem2tokens{$stem}}}[0]; }
	else { $theme = $stem; }
	$theme = HTML::Entities::encode_entities($theme);
	$html .= "\n\t\t".'<TR align="left">';
	$html .= "\n\t\t\t".'<TD valign="top" align="left" class=pastthemetext><B>'.$theme.'</B></TD>';
	$html .= "\n\t\t".'</TR>';
	foreach $ssid (@winning_ssids) {
	    $html .= "\n\t\t".'<TR align="left">';
	    if ( defined($ssid2smid{$ssid})
		 and defined($smid2said{$ssid2smid{$ssid}})
		 and defined($said2addr{$smid2said{$ssid2smid{$ssid}}})
		 and defined($ssid2sentence{$ssid}) ) {
		$addr = HTML::Entities::encode_entities($said2addr{$smid2said{$ssid2smid{$ssid}}});
		$sentence_text = HTML::Entities::encode_entities($ssid2sentence{$ssid});
		$html .= "\n\t\t\t".'<TD align="left"><B class=pastspeakertext><A href="'.$messagesCGIScript.$smid2said{$ssid2smid{$ssid}}.'" target="'.$said2addr{$smid2said{$ssid2smid{$ssid}}}.'">'.$addr.'</A>:</B>&nbsp;<B class=pastsentencetext><A href="'.$messagesURL.$ssid2smid{$ssid}.'.html#'.$ssid2start{$ssid}.'" target="message '.$ssid2smid{$ssid}.'">'.$sentence_text.'</A></B></TD>';
	    }
	    else { $html .= "\n\t\t\t".'<TD>&nbsp;</TD>'; }
	    $html .= "\n\t\t".'</TR>';
	}
    }
    $html .= "\n\t\t".'</TBODY>';
    $html .= "\n\t".'</TABLE>';
    return($html);
}


## ASSIGN_THETAS_FOR_INTERFACE
##
sub assign_thetas_for_interface
{
    my (@social_network_indices) = @_;

    my($i,$sni,$last_angle,$index_of_largest_network_with_theta);
    my($size_of_largest_network_with_theta);

    ## If there is only one social network currently, then assign
    ## position it at 0.
    if ( $#social_network_indices == 0 ) { $sn2theta{$social_network_indices[0]} = 0; }
    ## Otherwise, first rotate the @social_network_indices until a social network
    ## with an existing theta angle is found.  Find the largest social network with
    ## an assigned theta to be the anchor point.
    else {
	$index_of_largest_network_with_theta = -1;
	$size_of_largest_network_with_theta = -1;
	for ($i = 0; $i <= $#social_network_indices; $i++) {
	    $sni = $social_network_indices[$i];
	    if ( defined($social_network_indices[$i])
		 and defined($sn2theta{$sni})
		 and defined($sn2saids{$sni})
		 and (keys %{$sn2saids{$sni}})
		 and ( scalar(keys %{$sn2saids{$sni}}) > $size_of_largest_network_with_theta ) ) {
		$size_of_largest_network_with_theta = scalar(keys %{$sn2saids{$sni}});
		$index_of_largest_network_with_theta = $sni;
	    }
	}		
	if ( $index_of_largest_network_with_theta != -1 ) {
	    $sni = -1;
	    while ( $sni != $index_of_largest_network_with_theta ) {
		$sni = shift @social_network_indices;
		push(@social_network_indices,$sni);
	    }
	    $last_angle = $sn2theta{$index_of_largest_network_with_theta};
	}
	else { $last_angle = 0; }
	$angle_increment = ( $radius_of_circle / scalar(@social_network_indices) );
	undef %sn2theta;
	foreach $sni (@social_network_indices) {
	    $sn2theta{$sni} = $last_angle;
	    $last_angle = &angle_add($last_angle,$angle_increment);
	}
    }
    return;
}


## DISTRIBUTE_PLAYERS_FOR_INTERFACE
##
sub distribute_players_for_interface
{
    my(@social_network_indices) = @_;

    foreach $sni (@social_network_indices) {
	## Update the theta and rho positions for each player.
	foreach $said (keys %{$sn2saids{$sni}}) {
	    if ( not(defined($said2theta{$said})) ) { $said2previous_theta{$said} = $sn2theta{$said}; }
	    else { $said2previous_theta{$said} = $said2theta{$said}; }
	    $said2theta{$said} = $sn2theta{$sni};
	    $said2theta_movement{$said} = &angle_subtract($said2theta{$said},$said2previous_theta{$said});
	    if ( not(defined($said2rho{$said})) ) { $said2previous_rho{$said} = $said2percentage_score{$said}; }
	    else { $said2previous_rho{$said} = $said2rho{$said}; }
	    $said2rho{$said} = $said2percentage_score{$said};
	    if ( defined($said2rho{$said}) and defined($said2previous_rho{$said}) ) {
		$said2rho_movement{$said} = $said2rho{$said} - $said2previous_rho{$said};
	    }
	    else { $said2rho_movement{$said} = 0; }
	}
    }
    return;
}


## SELECT_FACES_AND_PROFILES_FOR_INTERFACE
##
sub select_faces_and_profiles_for_interface
{
    my($current_smid) = @_;

    my($pi,$speaker_said,%x_pos,$said);

    $speaker_said = $smid2said{$current_smid};
    ## Determine the horizontal position of the author of the current_smid
    $pi = atan2(1,1) * 4;
    $x_pos{$speaker_said} = ( ( $radius_of_circle - $said2rho{$speaker_said} ) 
			      * cos(($said2theta{$speaker_said}/$radius_of_circle) * 2 * $pi) );
    ## To pick an image and a profile for players, determine whether they are to the left or the
    ## right of the speaker.
    foreach $said (sort {$a <=> $b} keys %said2smids) {
	if ( defined($said2rho{$said}) and defined($said2theta{$said}) ) {
	    $x_pos{$said} = ( ( $radius_of_circle - $said2rho{$said} ) 
			      * cos(($said2theta{$said}/$radius_of_circle) * 2 * $pi) );
	}
	else { $x_pos{$said} = $x_pos{$speaker_said}; }
	if ( $x_pos{$said} == $x_pos{$speaker_said} ) { $said2profile{$said} = ''; }
	elsif ( $x_pos{$said} > $x_pos{$speaker_said} ) { $said2profile{$said} = 'l'; }
	else { $said2profile{$said} = 'r'; }
	## Assign each new image id randomly.
	if (not defined($said2image_id{$said})) {
	    $said2image_id{$said} = int(rand($number_of_faces));
	    if ( $said2image_id{$said} == 0 ) { $said2image_id{$said} = 1; }
	}
    }
}


## PLAYER_IS_VISIBLE
##
sub player_is_visible
{
    my($said) = @_;
    
    my(@smids);

    @smids = @{$said2smids{$said}};
    if ( $smids[$#smids] >= ( $current_simplified_message_id - $recency ) ) { return(1); }
    ## Even if the current winner has not posted within $recency messages, keep the winner visible.
    if ( $current_winner == $said ) { return(1); }
    else { return(0); }
}


## WRITE_HTML_FOR_INTERFACE
##
sub write_html_for_interface
{
    my($current_smid,@source_smids) = @_;

    my(@social_network_indices,$html_front_divs,$html_profile_divs,$html_label_divs,$html);
    my(@player_defs,$player_def,$said,$speaker_said,@js_code,$js_code,$sni);
    my($html_theme_divs,$theme_defs,$i,$subject_line_div,$shared_stem,@shared_stems);
    my($sentence_div,$sentence_text,$subject_line_text,$start_date,$end_date,$newsgroup_html);
    my($newsgroup_div,$newsgroup_def,$start_date_html,$start_date_div,$start_date_def);
    my($end_date_html,$end_date_div,$end_date_def,$post_button_label);

    @social_network_indices = &insert_new_social_networks_for_interface();
    undef %new_social_network;
    undef %old_social_network;
    &assign_thetas_for_interface(@social_network_indices);
    &distribute_players_for_interface(@social_network_indices);
    &select_faces_and_profiles_for_interface($current_smid,@social_network_indices);

    ## Now write the HTML
    $html = '<HTML>
<HEAD>
<META http-equiv="Refresh" content="'.$pause_between_frames.'; URL=agon'.($frame_number+1).'.html">
<TITLE>agonistics</TITLE>
<LINK type="text/css" rel="stylesheet" href="interface_style.css">
</HEAD>
<BODY>';
    $html .= &write_html_for_leaders_list($current_smid);
    $html .= &write_html_for_winning_sentences($current_smid);
    $html_front_divs = '';
    $html_profile_divs = '';
    $html_label_divs = '';
    $html_theme_divs = '';
    @player_defs = ();
    $speaker_said = $smid2said{$current_smid};
    $speaker_sni = $said2sn{$speaker_said};
    foreach $said (sort {$said2theta{$a} <=> $said2theta{$b}} keys %said2smids) {
	if ( not(&player_is_visible($said)) ) { next; }
	if ( $said == $speaker_said ) { $said_is_speaker{$said} = 1; }
	else { $said_is_speaker{$said} = 0; }
	$html_front_divs .= '<DIV id="'.$said.'_front" style="position:absolute; visibility:hidden" onMouseDown="beginDrag(this, event);" onMouseOver="turnLabelOnFromProfile(this, event);"></DIV>'."\n";
	$html_profile_divs .= '<DIV id="'.$said.'_profile" style="position:absolute; visibility:hidden"></DIV>'."\n";
	$html_label_divs .= '<DIV id="'.$said.'_label" style="position:absolute; visibility:hidden"></DIV>'."\n";
	if ( defined($said2addr{$said}) ) {
	    $player_def = '{ addr:"'.$said2addr{$said}.'"';
	    $player_def .= ', initialRho:'.$said2previous_rho{$said}.', finalRho:'.$said2rho{$said};
	    $player_def .= ', rhoMovement:'.$said2rho_movement{$said}; 
	    $player_def .= ', initialTheta:'.$said2previous_theta{$said}.', finalTheta:'.$said2theta{$said};
	    $player_def .= ', thetaMovement:'.$said2theta_movement{$said}; 
	    $player_def .= ', profile:"'.$said2profile{$said}.'", imageID:'.$said2image_id{$said};
	    $player_def .= ', said:'.$said.', isSpeaker:'.$said_is_speaker{$said};
	    $player_def .= ', score:'.$said2score{$said}.', previousScore:'.$said2previous_score{$said};
	    if ( $said == $current_winner ) { $player_def .= ', isWinner: 1 }'; }
	    else { $player_def .= ', isWinner: 0 }'; }
	    push(@player_defs,$player_def);
	}
    }
    $subject_line_div = '<DIV id="subject_line_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $sentence_div = '<DIV id="sentence_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $newsgroup_html = '<B class=labeltext>n&nbsp;e&nbsp;w&nbsp;s&nbsp;g&nbsp;r&nbsp;o&nbsp;u&nbsp;p&nbsp;&nbsp;&nbsp;</B>';
    $newsgroup_html .= '<B class=scorelisttext>'.$news_group.'&nbsp;&nbsp;&nbsp;</B>';
    $newsgroup_div = '<DIV id="newsgroup_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $newsgroup_def = 'var newsgroupHTML = "'.$newsgroup_html.'";'."\n";
    $end_date_html = '<B class=labeltext>t&nbsp;&nbsp;i&nbsp;m&nbsp;e&nbsp;&nbsp;&nbsp;</B>';
    $end_date_html .= '<B class=scorelisttext>'.&code_the_date($smid2date{$current_smid}).'</B>';
    $end_date_div = '<DIV id="end_date_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $end_date_def = 'var endDateHTML = "'.$end_date_html.'";'."\n";
    ## add forward and rewind buttons
    $rewind_button_html = '<A href="http://'.$cgi_url.'/Agonistics/rewind_to_first_frame.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'"><IMG src="rewind.gif"></A>';
    $rewind_button_div = '<DIV id="rewind_button_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $rewind_button_def = "var rewindButtonHTML = '".$rewind_button_html."';\n";
    $forward_button_html = '<A href="http://'.$cgi_url.'/Agonistics/forward_to_latest_frame.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'"><IMG src="forward.gif"></A>';
    $forward_button_div = '<DIV id="forward_button_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $forward_button_def = "var forwardButtonHTML = '".$forward_button_html."';\n";
    ## add help and post buttons
    if ( $language_locale =~ /^EN/ ) {
	$help_button_html = '<FORM ACTION="http://'.$documents_url.'/Agonistics/'.$archive_name.'/Interface/help-en.html" TARGET="agonistics / help">';  
	$help_button_html .= '<INPUT class=buttonstyle type="SUBMIT" value="help&nbsp;"></FORM>';
	$post_button_label = "write";
    }
    elsif ( $language_locale =~ /^FR/ ) {
	$help_button_html = '<FORM ACTION="http://'.$documents_url.'/Agonistics/'.$archive_name.'/Interface/help-fr.html" TARGET="agonistics / aide">';  
	$help_button_html .= '<INPUT class=buttonstyle type="SUBMIT" value="aide&nbsp;"></FORM>';
	$post_button_label = "&eacute;crire";
    }
    elsif ( $language_locale =~ /^DE/ ) {
	$help_button_html = '<FORM ACTION="http://'.$documents_url.'/Agonistics/'.$archive_name.'/Interface/help-de.html" TARGET="agonistics / hilfe">';  
	$help_button_html .= '<INPUT class=buttonstyle type="SUBMIT" value="hilfe&nbsp;&nbsp;&nbsp;&nbsp;"></FORM>';
	$post_button_label = "schreiben";
    }
    $help_button_div = '<DIV id="help_button_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $help_button_def = "var helpButtonHTML = '".$help_button_html."';\n";
    ## Post button for newsgroup discussions
    if ( $nntp_server ) {
	$post_button_html = '<FORM METHOD="POST" ACTION="http://'.$cgi_url.'/Agonistics/write_message.pl" ENCTYPE="multipart/form-data" TARGET="write_a_message">';
	$post_button_html .= '<INPUT TYPE="hidden" NAME="url" VALUE="'.$documents_url.'">';
	$post_button_html .= '<INPUT TYPE="hidden" NAME="cgi_url" VALUE="'.$cgi_url.'">';
	$post_button_html .= '<INPUT TYPE="hidden" NAME="archive" VALUE="'.$archive_name.'">';
	$post_button_html .= '<INPUT TYPE="hidden" NAME="news_group" VALUE="'.$news_group.'">';
	$post_button_html .= '<INPUT TYPE="hidden" NAME="language_locale" VALUE="'.$language_locale.'">';
    }
    ## Post button for email-based discussions
    else { $post_button_html = '<FORM ACTION="mailto:'.$mailing_list_address.'">';  }
    $post_button_html .= '<INPUT name="post_message" class=buttonstyle type="SUBMIT" value="'.$post_button_label.'">';
    $post_button_div = '<DIV id="post_button_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $post_button_def = "var postButtonHTML = '".$post_button_html."';\n";
    $def_button_html = '';
    $def_button_div = '<DIV id="def_button_div" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $def_button_def = "var defButtonHTML = '".$def_button_html."';\n"; 
    ## Player definitions
    $player_defs = join ','."\n",@player_defs;
    $player_defs = 'var players = [  '.$player_defs.'];';
    $current_winning_theme = HTML::Entities::encode_entities($current_winning_theme);
    $theme_defs = '{ network:'.$speaker_sni.', text:"'.$current_winning_theme.'", theta:'.$sn2theta{$speaker_sni}.' }';
    $theme_defs = 'var networkThemes = [ '.$theme_defs.' ];';
    $html_theme_divs .= '<DIV id="theme_'.$speaker_sni.'" style="position:absolute; visibility:hidden"></DIV>'."\n";
    $html .= $html_front_divs."\n\n".$html_profile_divs."\n\n".$html_label_divs."\n\n".$subject_line_div."\n\n".$sentence_div."\n\n".$html_theme_divs."\n\n".$newsgroup_div."\n\n".$end_date_div."\n\n".$rewind_button_div."\n\n".$forward_button_div."\n\n".$help_button_div."\n\n".$post_button_div."\n\n".$def_button_div."\n\n".'<SCRIPT>'."\n".$player_defs."\n\n".$theme_defs."\n\n".$newsgroup_def."\n\n".$end_date_def."\n\n".$rewind_button_def."\n\n".$forward_button_def."\n\n".$help_button_def."\n\n".$post_button_def."\n\n".$def_button_def."\n\n";
    $html .= 'var dir = "Messages";'."\n";
    $html .= 'var messagesCGIScript = "http://'.$cgi_url.'/Agonistics/index_for_player.pl?url='.$documents_url.'&path='.$web_server_directory.'&archive='.$archive_name.'&msg='.$current_smid.'";'."\n";
    $subject_line_text = HTML::Entities::encode_entities($smid2nsl{$current_smid});
    $html .= 'var subjectLineText = "'.$subject_line_text.'";'."\n";
    if ( $current_winning_sentence ) {
	$current_winning_sentence =~ s/"//g;
	$current_winning_sentence = HTML::Entities::encode_entities($current_winning_sentence);
	$html .= 'var sentenceText = "'.$current_winning_sentence.'";'."\n";
    }
    else { $html .= 'var sentenceText = "";'."\n"; }
    if ( defined($said2addr{$current_winner}) ) { $html .= 'var currentWinner = "'.$said2addr{$current_winner}.'";'."\n"; }
    else { $html .= 'var currentWinner = "";'."\n"; }
    if ( defined($said2addr{$speaker_said}) ) { $html .= 'var currentSpeaker = "'.$said2addr{$speaker_said}.'";'."\n"; }
    else { $html .= 'var currentSpeaker = "";'."\n"; }
    open(JS,'Resources'.$slash.'javascript_interface_code.txt');
    @js_code = <JS>;
    close(JS);
    $js_code = join '',@js_code;
    $html .= $js_code;
    open(AGON,'>'.$interface_directory.$slash.'agon'.$frame_number.'.html');
    print AGON $html;
    close(AGON);
}


## CODE_THE_DATE
##
sub code_the_date
{
    my($date) = @_;
    
    return(HTML::Entities::encode_entities(lc(UnixDate($date,"%a %b %e %H:%M:%S %Y"))));
}


## WRITE_MESSAGE_LIST_ID_ENTRY
##
sub write_message_list_id_entry
{
    my($smid,$is_hyperlinked) = @_;

    if ( $is_hyperlinked ) { $smid = '<A href="'.$smid.'.html">'.$smid.'</A>'; }
    return("\n\t\t\t".'<TD class=listtext>'.$smid.'</TD>');
}


## WRITE_MESSAGE_LIST_DATE_ENTRY
##
sub write_message_list_date_entry
{
    my($smid,$is_hyperlinked) = @_;

    my($date);

    if (defined $smid2date{$smid}) {
	$date = $smid2date{$smid};
	$date = UnixDate($date,"%d %b %y at %H:%M");
	$date = HTML::Entities::encode_entities($date);
	$date =~ s/\s/\&nbsp\;/g;
    }
    else { $date = ''; }
    if ( $is_hyperlinked ) { $date = '<A href="'.$smid.'.html">'.$date.'</A>'; }
    return("\n\t\t\t".'<TD class=listtext>'.$date.'</TD>');
}


## WRITE_MESSAGE_LIST_SUBJECT_ENTRY
##
sub write_message_list_subject_entry
{
    my($smid,$is_hyperlinked) = @_;

    my($subject);

    if (defined $smid2subject{$smid}) {
	$subject = $smid2subject{$smid};
	$subject = HTML::Entities::encode_entities($subject);
	$subject =~ s/\s/\&nbsp\;/g;
    }
    else { $subject = ''; }
    if ( length($subject) > 60 ) { $subject = substr($subject,0,60); }
    if ( $is_hyperlinked ) { $subject = '<A href="http://'.$documents_url.'/Agonistics/'.$archive_name.'/Interface/Messages/'.$smid.'.html">'.$subject.'</A>'; }
    return("\n\t\t\t".'<TD class=listtext>'.$subject.'</TD>');
}


## WRITE_MESSAGE_LIST_SENDER_ENTRY
##
sub write_message_list_sender_entry
{
    my($smid,$is_hyperlinked) = @_;

    my($sender);

    if ( (defined $smid2said{$smid}) and (defined $said2ana{$smid2said{$smid}} ) ) {
	$sender = $said2ana{$smid2said{$smid}};
	$sender = HTML::Entities::encode_entities($sender);
	$sender =~ s/\s/\&nbsp\;/g;
    }
    else { $sender = ''; }
    if ( $is_hyperlinked ) { $sender = '<A href="'.$smid.'.html">'.$sender.'</A>'; }
    return("\n\t\t\t".'<TD class=listtext>'.$sender.'</TD>');
}


## WRITE_ENTRY_IN_MESSAGE_INDEX
##
sub write_entry_in_message_index
{
    my($smid) = @_;

    my($html,$entry);

    if ( defined($smid2sdt{$smid})
	 ## Do not include references to messages that are not in the archive.
	 and ( $smid2sdt{$smid} != -1 )
	 and defined($smid2said{$smid})
	 and defined($said2addr{$smid2said{$smid}})
	 and defined($smid2nsl{$smid}) ) {
	$entry = $smid."\t".$said2addr{$smid2said{$smid}}."\t".$smid2sdt{$smid}."\t".$smid2nsl{$smid}."\t";
	if ( ( $smid2spid{$smid} ) 
	     and ( $smid2sdt{$smid2spid{$smid}} != -1 )
	     and defined($smid2said{$smid2spid{$smid}})
	     and defined($said2addr{$smid2said{$smid2spid{$smid}}})
	     and defined($smid2nsl{$smid2spid{$smid}}) ) {
	    $entry .= $smid2spid{$smid}."\n"; 
	}
	else { $entry .= "no_parent\n"; }
	$html .= "\t\t".'<TR align="left">';
	$html .= &write_message_list_id_entry($smid,0);
	$html .= &write_message_list_date_entry($smid,0);
	$html .= &write_message_list_subject_entry($smid,1);
	$html .= &write_message_list_sender_entry($smid,0);
	$html .= "\n\t\t".'</TR>'."\n";
	$entry .= $html;
	$message_index_for_said = $message_index_file_name.'_for_'.$smid2said{$smid}.'.txt';
	## Initialize the file for the player if this is the player's first message posted.
	if ( not(-e $message_index_for_said) ) {
	    open(MSGINDEX,'>'.$message_index_for_said);
	    close(MSGINDEX);
	}
	open(MSGINDEX,'>>'.$message_index_for_said);
	flock(MSGINDEX, LOCK_EX);  ## exclusive lock, use LOCK_SH to read from the file
	print MSGINDEX $entry;
	close(MSGINDEX);
    }
}

## CLEAN_RAW_MESSAGE
##
sub clean_raw_message
{
    my($next_line,$next_msg,$msg_header,$msg_body,%fields_and_values,$pos_next_msg,$boundary);
    my($temp_filename);

    ## Get rid of any extras carriage returns or line feeds.  Windows, UNIX, and Apple
    ## formats differ according to how newlines are encoded.  This normalizes those
    ## differences.  Also, some newline characters get translated into =20 or =,
    ## so these extra characters are discarded here.
    $/ = "\n";
    open(RAW,$raw_messages_file) || die "Can't open messages file\, $raw_messages_file\, $! \n";
    my $random_number = int(rand(10000));
    $temp_filename = 'temp_file_'.$random_number.'.txt';
    open(TEMP,'>'.$temp_filename);
    while ($next_line = <RAW>) { 
	if ( $next_line !~ /^From\s/ ) { ## Get rid of any leading "From " that is not an actual header.
	    chomp($next_line);
	    for ($next_line) {
		s/^(.*)\s+$/$1/;
		s/^(.*)\=20$/$1/;
		s/^(.*)\=$/$1/;
	    }
	    print TEMP $next_line."\n"; 
	}
    }
    close(RAW);
    close(TEMP);
    $/ = $end_of_message_marker;
    open(TEMP,$temp_filename);
    ## Now strip out any HTML formatting in the messages and simplify the headers.
    open(ARCHIVEFILE,'>>'.$archive_file_name) || die "Can't open file for output, $archive_file_name: $!\n";
    while ($next_msg = <TEMP>) { 
	undef %fields_and_values;
	%fields_and_values = &extract_fields_and_values_from_message($next_msg);
	## separate message header and body
	if ( $next_msg =~ m/\n\n/g ) { 
	    $pos_next_msg = pos($next_msg);
	    $msg_header = substr($next_msg,0,$pos_next_msg); 
	    $msg_body = substr($next_msg,$pos_next_msg,length($next_msg)-$pos_next_msg);
	    $next_msg = 'newsgroups: '.$news_group."\n";
	    foreach $field (sort keys %fields_and_values) {
		$next_msg .= $field.' '.$fields_and_values{$field}."\n";
	    }
	    ## If the message has been sent in plain text and HTML, strip out the HTML.
	    if ( (exists $fields_and_values{'content-type:'})
		 and ( $fields_and_values{'content-type:'} =~ /multipart/i ) ) {
		$boundary = &get_boundary_from_content_type($fields_and_values{'content-type:'});
		$next_msg .= "\n".&remove_html_from_message_body($msg_body,$boundary);
	    }
	    else { $next_msg .= "\n".$msg_body; }
	}
	print ARCHIVEFILE $next_msg;
    }
    $/ = "\n";
    close(ARCHIVEFILE);
    close(TEMP);
    unlink($temp_filename);
}


## GET_BOUNDARY_FROM_CONTENT_TYPE
##
sub get_boundary_from_content_type
{
    my($content_type) = @_;

    if ( $content_type =~ /boundary\=\"([^\"]+)\"/ ) { return quotemeta($1); }
}


## REMOVE_HTML_FROM_MESSAGE_BODY
##
## It might be better to do this with the MIME::Parser module, especially
## if the message attachments are to be saved and analyzed, but the
## function below simply looks for the part of the message that is encoded
## as plain/text and discards all of the other parts of the message.
##
sub remove_html_from_message_body
{
    my($msg_body,$top_level_boundary) = @_;

    my($inputrecordseparator,@msg_body,$boundary,$save_lines_p);
    my($msg_body_part,@lines,$line);

    $inputrecordseparator = $/;
    $/ = "\n";
    @msg_body = &split_message_body_at_boundary($top_level_boundary,$msg_body);
    $boundary = '__this_is_a_highly_unlikely_mime_message_boundary__';
    $save_lines_p = 0;
    foreach $msg_body_part (@msg_body) {
	@msg_body_part = split /\n/,$msg_body_part;
	undef @lines;
	foreach $line (@msg_body_part) {
	    if ( not($save_lines_p) ) {
		if ( $line =~ /boundary\=\"([^\"]+)\"/ ) { $boundary = quotemeta($1); }
		elsif ( $line =~ /^content-type\:\s*text\/plain/mi ) { $save_lines_p = 1; }
	    }
	    else {
		if ( $line =~ /$boundary/ ) { last; }
		elsif ( ( $line !~ /charset\=/i ) 
			and ( $line !~ /content\-transfer\-encoding/i ) )  { 
		    push(@lines,$line); 
		}
	    }
	}
	if ( @lines ) { 
	    $plain_text = join "\n",@lines; 
	    last;
	}
    }
    $/ = $inputrecordseparator;
    return($plain_text."\n".$end_of_message_marker);
}


## SPLIT_MESSAGE_BODY_AT_BOUNDARY
##
sub split_message_body_at_boundary
{
    my($boundary,$msg_body) = @_;
    my(@msg_body,$line,@msg_parts,$msg_part);

    $msg_part = '';
    @msg_body = split /\n/,$msg_body;
    foreach $line (@msg_body) {
	if ( $line =~ /$boundary/ ) { 
	    push(@msg_parts,$msg_part);
	    $msg_part = '';
	}
	else { $msg_part .= "\n".$line; }
    }
    if ( $msg_part ) { push(@msg_parts,$msg_part); }
    return @msg_parts;
}


## PRINT_TO_LOG
##
sub print_to_log
{
    my($status_message_string) = @_;
    
    my($time);

    open(LOG,'>>'.$log_file);
    $time = localtime;
    print LOG $time.' >> ';
    print LOG $status_message_string."\n";
    close(LOG);
    print STDOUT $time.' >> ';
    print STDOUT $status_message_string."\n";
}


## CHECK_YAHOO_MAIL
##
sub check_yahoo_mail
{
    my(@ids,$id,$msg,$msg_header);

    my($yahoo_client);

    $new_messages_downloaded = 0;
    &print_to_log("Logging in...");
    eval {
	$yahoo_client = Mail::Client::Yahoo->login(username => $yahoo_uid,
						   password => $yahoo_password,
						   secure   => 0
						  )
	};
    if ( $@ ) {
	&print_to_log("ERROR: Failed login...");
	sleep($pause);
	return;
    }
    &print_to_log("Checking Inbox...");
    eval { $yahoo_client->select_folder('Inbox') };
    if ( $@ ) {
	&print_to_log("ERROR: Couldn't find Inbox...");
	eval { $yahoo_client->logout() };
	if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
 	sleep($pause);
	return;
    }
    eval { @ids = $yahoo_client->message_list() };
    if ( $@ ) {
	&print_to_log("ERROR: Couldn't find message ids in Inbox...");
	eval { $yahoo_client->logout() };
	if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
	sleep($pause);
	return;
    }
    foreach $id (@ids) { 
	if ( defined($yahooid{$id}) ) { next; }
	else {
	    $yahooid{$id} = 1;
	    eval { $msg = $yahoo_client->message($id) };
	    if ( $@ ) {
		&print_to_log("ERROR: Couldn't get message $id...");
		eval { $yahoo_client->logout() };
		if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
		sleep($pause);
		return;
	    }
	    $msg_header = $msg->head();
	    ## Messages from $news_group are selected by examining the
	    ## subject lines.
	    if ( $msg_header->get('Subject') =~ /$news_group/i ) {
		open(RAW,'>'.$raw_messages_file);
		$msg->print(\*RAW);
		print RAW "\n".$end_of_message_marker."\n";
		close(RAW);
		&clean_raw_message();
		$new_messages_downloaded++;
		&print_to_log("Downloaded $id");
		eval { $yahoo_client->move_message($id,$yahoo_outbox) };
		if ( $@ ) { 
		    &print_to_log("ERROR: Unable to move message out of Inbox into $yahoo_outbox..."); 
		    sleep($pause);
		    return;
		}
	    }
	}
    }
    eval { $yahoo_client->logout() };
    if ( $@ ) {	&print_to_log("ERROR: Failed logout..."); }
    if ( $new_messages_downloaded == 0 ) { &print_to_log("No new messages found."); }
    sleep($pause);
}



## CHECK_NEWSGROUP
##
sub check_newsgroup
{
    my($since) = @_;

    my($now,$msgid,$message_ids_ptr,$since_date,$article_lines_ptr,$article);

    $new_messages_downloaded = 0;
    $now = $since; # or, should this be, time() ?  this seems to depend on the NNTP server and the machine
    ## Get a pointer to a list of message ids posted since server was last checked ($since).
    $since_date = &ParseDateString("epoch $since");
    $since_date = &UnixDate($since_date,"%d %b %y at %H:%M");
    &print_to_log("Accessing ".$nntp_server." to download ids of messages posted to ".$news_group." since ".$since_date." GMT");
    eval { $message_ids_ptr = $nntp->newnews($since,$news_group) };
    if ( $@ ) {
	&print_to_log("ERROR: Couldn't get message ids from NNTP server...");
	sleep($pause);
	return($since);
    }
    foreach $msgid (@{$message_ids_ptr}) {
	chomp($msgid);
	if ( exists($nntpid{$msgid}) ) { next; }
	$nntpid{$msgid} = 1;
	eval { $article_lines_ptr = $nntp->article($msgid) };
	if ( $@ ) {
	    &print_to_log("ERROR: Couldn't get message $msgid from NNTP server...");
	    sleep($pause);
	    return($since);
	}
	$article = join '',@{$article_lines_ptr};
	open(RAW,'>'.$raw_messages_file);
	print RAW $article."\n".$end_of_message_marker."\n";
	close(RAW);
	&clean_raw_message();
	$new_messages_downloaded++;
	&print_to_log("Downloaded $msgid");
    }
    if ( $new_messages_downloaded == 0 ) { &print_to_log("No new messages found."); }
    sleep($pause);
    return($now);
}


## CHECK_IMAP_MAIL
##
## This is not debugged.
##
sub check_imap_mail
{
    my(@ids,$id,$msg,$msg_header);

    my($imap_client,$imapid,$number_of_messages,$lines_ref,$lines,$line,$from_mailing_list_p);

    $new_messages_downloaded = 0;
    &print_to_log("Connecting to IMAP server $imap_server...");
    eval { $imap_client = new Net::IMAP::Simple::SSL($imap_server) }; ## frequently hangs; cause: unknown
    if ( $@ ) {
	&print_to_log("ERROR: Failed to connect to server...");
	sleep($pause);
	return;
    }
    &print_to_log("Logging in...");
    eval { $imap_client->login( $imap_uid => $imap_password ) };
    if ( $@ ) {
	&print_to_log("ERROR: Failed log in...");
	sleep($pause);
	return;
    }
    &print_to_log("Checking Inbox...");
    eval { $number_of_messages = $imap_client->select( 'Inbox' ) };
    if ( $@ ) {
	&print_to_log("ERROR: Couldn't find Inbox...");
	eval { $imap_client->quit() };
	if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
 	sleep($pause);
	return;
    }
    foreach $imapid ( 1..$number_of_messages ) {
	if ( not($imap_client->seen($imapid)) ) {
	    # get the message, returned as a reference to an array of lines
	    $lines_ref = $imap_client->get($imapid);
	    $from_mailing_list_p = 0;
	    foreach $line (@{$lines_ref}) {
		if ( $line =~ /^subject:\s+$news_group/i ) {
		    $from_mailing_list_p = 1;
		    last;
		}
	    }
	    if ( $from_mailing_list_p ) {
		$lines = join '',@{$lines_ref};
		open(RAW,'>'.$raw_messages_file);
		print RAW $lines;
		print RAW "\n".$end_of_message_marker."\n";
		close(RAW);
		&clean_raw_message();
		$new_messages_downloaded++;
		&print_to_log("Downloaded $imapid");
		eval { $imap_client->copy($imapid,$imap_outbox) };
		if ( $@ ) { 
		    &print_to_log("ERROR: Unable to move message out of Inbox into $imap_outbox..."); 
		    eval { $imap_client->quit() };
		    if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
		    sleep($pause);
		    return;
		}
		eval { $imap_client->delete($imapid) };
		if ( $@ ) { 
		    &print_to_log("ERROR: Unable to delete message from Inbox after moving it to $imap_outbox..."); 
		    eval { $imap_client->quit() }; 
		    if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
		    sleep($pause);
		    return;
		}
	    }
	}
    }
    eval { $imap_client->quit() };
    if ( $@ ) { &print_to_log("ERROR: Failed logout..."); }
    if ( $new_messages_downloaded == 0 ) { &print_to_log("No new messages found."); }
    sleep($pause);
    return;
    sleep($pause);
}


## YESTERDAY_IN_SECONDS
##
sub yesterday_in_seconds
{
    my($yesterday,@yesterday_as_strings,@yesterday);
    my($elem,$sec,$min,$hour,$mday,$mon,$year,$time);

    ## Use ParseDate to find the date and time of yesterday.
    $yesterday = ParseDate('24 hours ago');
    ## Simplify and format the date and time into a list of strings.
    @yesterday_as_strings = UnixDate($yesterday,"%S","%M","%k","%e","%m","%y");
    ## Convert the strings into integers.
    @yesterday = ();
    foreach $elem (@yesterday_as_strings) {
	$elem += 0;
	push(@yesterday,$elem);
    }
    ($sec,$min,$hour,$mday,$mon,$year) = @yesterday;
    ## Decrement the month ($mon) to accord with the gmtime format
    ## (wherein the month runs from 0..11).
    $mon--;
    ## Calculate the number of seconds between yesterday and 1970.
    $time = timegm($sec,$min,$hour,$mday,$mon,$year);
    return($time);
}


## FETCH_NEW_MESSAGES_FROM_SERVER
##
## This function simply calls a more specific function.
## It is a placeholder for a function that periodically
## fetches messages off of a server.  The function called
## here must address the following issues:
##
## (1) Timing: Some form of wait function should be built into the
##     function so that it is not constantly pinging the
##     server for new messages but, rather, polls from the
##     server only once in a while.  The time period to
##     wait will depend on the server.
## 
## (2) Message format:  The fetching function must write the
##     fetched messages into the file named $archive_file_name.
##     These messages should be appended into that file so that
##     more messages can be downloaded even if the system is not
##     done processing the current messages.  The messages should
##     be rendered in plain text (i.e., for example, HTML should
##     be stripped out of the messages).  The messages should be 
##     formatted according to RFC-1036 conventions.  One field of
##     the messages must be 'newsgroups:' and must be filled
##     with the name of the newsgroup specified in the agonistics
##     configuration file.  Finally, after the end of the
##     text of each message, on a separate line, the 
##     $end_of_message_marker must be printed.
##
## (3) Log: It is preferrable, but not necessary, for the fetching
##     function to log its errors (and successes) in the $log_file
##     using the &print_to_log function.  
##
## See &check_yahoo_mail and the associated functions 
## (e.g., &clean_raw_message) for an example implementation of 
## these criteria.
##     
sub fetch_new_messages_from_server
{
    print STDOUT "\nDownloading messages...\n";
    ## Download messages from a Yahoo email account; or,
    if ( $yahoo_uid and $yahoo_password ) { &check_yahoo_mail(); }
    ## download messages from a Usenet newsgroup server; or, 
    elsif ( $nntp_server ) { 
	## If no messages have been downloaded yet, download all message posted since yesterday.
	if ( $time_of_last_nntp_server_check == 0 ) {
	    $time_of_last_nntp_server_check = &yesterday_in_seconds();
	}
	$time_of_last_nntp_server_check = &check_newsgroup($time_of_last_nntp_server_check);
    }
    ## download messages from an IMAP email server.
    elsif ( $imap_server and $imap_uid and $imap_password ) { &check_imap_mail(); }
    else { die "No Yahoo, NNTP or IMAP server was specified for the interactive downloading of messages."; }
}


## REASSEMBLE_MESSAGE
##
sub reassemble_message
{
    my($smid) = @_;

    $message_header = '';
    if ( $smid2newsgroups{$smid} ) { $message_header = 'newsgroups: '.$smid2newsgroups{$smid}."\n"; }
    if ( $smid2date{$smid} ) { $message_header .= 'date: '.$smid2date{$smid}."\n"; }
    if ( $smid2mid{$smid} ) { 
	$message_header .= 'message-id: '.$smid2mid{$smid}."\n"; 
	if ( $mid2pid{$smid2mid{$smid}} ) { $message_header .= 'references: '.$mid2pid{$smid2mid{$smid}}."\n"; }
    }
    if ( $smid2said{$smid} and $said2ana{$smid2said{$smid}} ) { $message_header .= 'from: '.$said2ana{$smid2said{$smid}}."\n"; }
    if ( $smid2subject{$smid} ) { $message_header .= 'subject: '.$smid2subject{$smid}."\n"; }
    $message_body = '';
    $ln = 0;
    while ( defined($smidln2line{"$smid $ln"}) ) {
	$message_body .= $smidln2line{"$smid $ln"}."\n";
	$ln++;
    }
    return($message_header."\n".$message_body);
}


## REORGANIZE_ARCHIVE
##
sub reorganize_archive
{
    my(@smids,$mid,$last_batch_archive_file_name);

    ## sort the messages by date
    @smids = ();
    foreach $mid (keys %analyzed_p) { push(@smids,$mid2smid{$mid}); }
    @smids = sort { $smid2sdt{$a} <=> $smid2sdt{$b} } @smids;
    ## store the last batch of messages
    $last_batch_archive_file_name = $archive_file_name;
    $last_batch_archive_file_name =~ s/\./\_$batch_of_messages\./;
    if ( $last_batch_archive_file_name eq $archive_file_name ) { die "archive file name not renamed!"; }
    ## Save the archive into a new file and delete the existing archive file
    copy($archive_file_name,$last_batch_archive_file_name);
    unlink($archive_file_name);
    ## print the last ( $max_frames / 2 ) into the (now empty) archive; and,
    ## update %analyzed_p so that it only contains message ids from messages not in the new archive
    open(ARCHIVEFILE,'>'.$archive_file_name);
    $first_smid_index = $#smids / 2;
    foreach $smid (@smids[$first_smid_index..$#smids]) {
	$message = &reassemble_message($smid);
	print ARCHIVEFILE $message.$end_of_message_marker."\n";
    }
    close(ARCHIVEFILE);
}


## CREATE_BLANK_PAGES
##
sub create_blank_pages
{
    my(@interface_files,$fn,$n,$html);

    opendir(ID,$interface_directory);
    @interface_files = readdir(ID);
    closedir(ID);
    $html = '<HTML>
<HEAD>
<META http-equiv="Refresh" content="1; URL=agon2.html">
<TITLE>agonistics</TITLE>
<LINK type="text/css" rel="stylesheet" href="interface_style.css">
</HEAD>
<BODY>
</BODY>
</HTML>';
    open(AGON,'>'.$interface_directory.$slash.'agon1.html');
    print AGON $html;
    close(AGON);
    $html =~ s/agon2/agon1/;
    foreach $fn (@interface_files) {
	if ( $fn =~ /^agon([0-9]+)\.html/ ) {
	    $n = $1;
	    if ( $n != 1 ) {
		open(AGON,'>'.$interface_directory.$slash.'agon'.$n.'.html');
		print AGON $html;
		close(AGON);
	    }
	}
    }
    return;
}


## INTERACTIVE_ANALYSIS
##
## Messages are periodically pulled from a server and analyzed.
##
sub interactive_analysis
{
    my($l,$small_pause);

    while (1) {
	if ( $new_messages_downloaded <= 0 ) { &fetch_new_messages_from_server(); }
	if ( $new_messages_downloaded > 0 ) {
	    $frame_number++;
	    $file_pointer_in_messages_archive = &process_next_message($file_pointer_in_messages_archive);
	    $new_messages_downloaded--;
	    print STDOUT 'frame: '.$frame_number."\t";
	}
	# write another version of the last page and have it point back to the
	# current page.
	if (-e $interface_directory.$slash.'agon'.$frame_number.'.html') {
	    open(PREVIOUSAGON,$interface_directory.$slash.'agon'.$frame_number.'.html');
	    @current_page_contents = <PREVIOUSAGON>;
	    close(PREVIOUSAGON);
	    open(NEXTAGON,'>'.$interface_directory.$slash.'agon'.($frame_number + 1).'.html');
	    foreach $l (@current_page_contents) {
		if ( $l =~ /content\=\"$pause_between_frames\;\s+URL\=agon([0-9]+)/ ) {
		    $l =~ s/agon[0-9]+/agon$frame_number/i;
		    $small_pause = int($pause/2);
		    $l =~ s/content\=\"$pause_between_frames\;/content\=\"$small_pause\;/i;
		}
		print NEXTAGON $l;
	    }
	    close(NEXTAGON);
	}
	if ( $frame_number > $max_frames ) { last; }
    }
    ## If max frames reached, restart the analysis
    &analyze_messages();
}


## STATIC_ANALYSIS
##
## Messages are analyzed from a static archive of messages.
##
sub static_analysis
{
    my($l,$small_pause,@interface_files,$fn,$n);

    while ($file_pointer_in_messages_archive < (-s $archive_file_name) ) {
	$frame_number++;
	$file_pointer_in_messages_archive = &process_next_message($file_pointer_in_messages_archive);
	print STDOUT 'frame: '.$frame_number."\t";
    }
    # write another version of the last page and have it point back to the
    # current page.
    if (-e $interface_directory.$slash.'agon'.$frame_number.'.html') {
	open(PREVIOUSAGON,$interface_directory.$slash.'agon'.$frame_number.'.html');
	@current_page_contents = <PREVIOUSAGON>;
	close(PREVIOUSAGON);
	open(NEXTAGON,'>'.$interface_directory.$slash.'agon'.($frame_number + 1).'.html');
	foreach $l (@current_page_contents) {
	    if ( $l =~ /content\=\"$pause_between_frames\;\s+URL\=agon([0-9]+)/ ) {
		$l =~ s/agon[0-9]+/agon$frame_number/i;
		$small_pause = int($pause/2);
		$l =~ s/content\=\"$pause_between_frames\;/content\=\"$small_pause\;/i;
	    }
	    print NEXTAGON $l;
	}
	close(NEXTAGON);
    }
    ## Erase any blank interface pages still remaining.
    opendir(ID,$interface_directory);
    @interface_files = readdir(ID);
    closedir(ID);
    foreach $fn (@interface_files) {
	if ( $fn =~ /^agon([0-9]+)\.html/ ) {
	    $n = $1;
	    if ( $n > ($frame_number + 1) ) {
		unlink($interface_directory.$slash.'agon'.$n.'.html');
	    }
	}
    }
    ## If max frames reached, restart the analysis
    if ( $frame_number > $max_frames ) { &analyze_messages(); }
}


## ANALYZE_MESSAGES
##
sub analyze_messages
{
    &initialize_global_variables();
    $batch_of_messages++;
    ## Make sure that clients viewing interface pages are moved to a blank page
    ## before any pages are deleted.
    sleep($pause_between_frames+2);
    ## If the archive does not yet exist, create it.
    if ( not(-e $archive_file_name) ) {
	open(ARCHIVEFILE,'>'.$archive_file_name);
	close(ARCHIVEFILE);
    }
    $file_pointer_in_messages_archive = 0;
    ## case 1: The messages to be analyzed are to be fetched periodically from
    ## a server.  This is the interactive mode.  
    ## case 2: Messages are analyzed from a static archive of messages.  
    ## case 3: There is already an archive messages to be analyzed and then -- 
    ## after the archive is analyzed -- messages will be pulled from the server and 
    ## analyzed interactively.
    if ( (-e $archive_file_name) and ( $is_interactive_p eq 'YES' ) ) {
	&static_analysis();
	&interactive_analysis();
    }
    elsif ( $is_interactive_p eq 'YES' ) { &interactive_analysis(); }
    else { &static_analysis(); }
}


&analyze_messages();




