#!/usr/bin/perl

# Copyright (C) 2004  Joshua Hoblitt
#
# $Id: buildIndex.pl,v 1.1 2005-01-25 21:48:54 jhoblitt Exp $

use 5.008;

use strict;
use warnings;

our $VERSION = '0.01';

use File::Basename qw();
use File::Find::Rule;
##use HTML::Strip;
use HTML::TreeBuilder;
use Plucene::Index::Writer;
use Plucene::Plugin::Analyzer::PorterAnalyzer;

use Getopt::Long qw( GetOptions :config auto_help auto_version );
use Pod::Usage qw( pod2usage );

my ($verbose, $help, $path, $indexDir);
GetOptions(
    'verbose' => \$verbose,
    'path=s'  => \$path,
    'index=s' => \$indexDir,
    )
    or pod2usage(2);

pod2usage(-msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
pod2usage(-msg => "Required option: --path|-p", -exitval => 2) unless $path;

$indexDir = $path . "/.index" unless defined $indexDir;

##my $stripper = HTML::Strip->new;

my $writer = Plucene::Index::Writer->new(
    $indexDir,
    Plucene::Plugin::Analyzer::PorterAnalyzer->new(),
    1    # Create the index from scratch
);

my $doc = Plucene::Document->new;

foreach my $filename (File::Find::Rule->file->name(qr/^msg\d+\.html/)->in($path)) {
    print "indexing: $filename\n" if $verbose;
    $writer->add_document(parseMhonarcHtml($filename));
}

sub parseMhonarcHtml
{
    my $filename = shift;

    my $tree = HTML::TreeBuilder->new_from_file($filename);
    my $doc  = Plucene::Document->new;

    $doc->add(Plucene::Document::Field->Keyword(file_name => File::Basename::basename($filename),));

    # get the header values for to, cc, from, date, and subject

    # find the <div> tag labeled header
    my $headerTag = $tree->look_down(_tag => "div", id => "header");

    my %headers;

    foreach my $rowNode ($headerTag->look_down(_tag => "tr")) {

        # process each row of the header

        # get name of header field
        my $nameNode = $rowNode->look_down(_tag => "td");
        my $name = lc($nameNode->look_down(_tag => "em")->as_text);
        $nameNode->delete;

        # get content of header field
        my $contentNode = $rowNode->look_down(_tag => "td");
        my $content     = $contentNode->as_text;

        # index "cc" as part of "to"
        $name =~ s/^cc/to/;

        # index header fields
        if ($name =~ /^(to|from|subject|date)/i) {
            ##$content = $stripper->parse( $content );
            print "\tadding to index $name: $content\n" if $verbose;
            $doc->add(Plucene::Document::Field->Text($name => $content));
        }
    }

    # find the <div> tag labeled content
    my $contentTag = $tree->root->look_down(_tag => "div", id => "content");

    # index message body
    print "\tadding to index content\n" if $verbose;
    $doc->add(Plucene::Document::Field->UnStored(content => $contentTag->as_text));

    return $doc;
}

__END__

=pod

=head1 NAME

buildIndex.pl - generate Plucene/Lucene index of MHonarc archives

=head1 SYNOPSIS

    buildIndex.pl --path dir [--index dir] [--verbose]

    or

    buildIndex.pl -p dir [-i dir] [-v]

    or

    buildIndex.pl [--help | -h | -? | --version]

=head1 DESCRIPTION

Parses directories of MHonarc generated HTML files and builds a searchable
index.  The parser requires the HTML files to be in a specifc format which is
signifigantly modified from default MHonarc format.  The indexer creates a
directory of index files in the Plucene/Lucene format.

=head1 OPTIONS

=over 4

=item * --path dir | -p dir

Directory of MHonarc generated HTML files to process.

=item * --index dir | -i dir

Directory to write the Plucene/Lucene index files to.  Defaults to C<--path> + '.index'.

=item * --verbose | -v

Print status information while processing.

=item * --help | -h | -?

=item * --version

=back

=head1 DEVELOPER NOTES

The parser extracts the C<to>, C<cc>, C<from>, C<subject>, and C<date> headers
along with the message body.  The HTML document is required to contain tags in
this format.

    <div id="header">
        <table>
            <tr>
                <td> <em>[ header ]</em>:</td><td>[ value ]</td>
            </tr>
            .
            .

        </table>
    </div>

    <div id="content">
        [ message body ]
    </div>

The parser is insenstive to the ordering of the C<header> and C<content>
C<E<lt>divE<gt>> tags.  All other tags in the document are ignored.

=head1 CREDITS

Just me, myself, and I.

=head1 SUPPORT

Please contact the author directly via e-mail.

=head1 AUTHOR

Joshua Hoblitt <jhoblitt@cpan.org>

=head1 COPYRIGHT

Copyright (C) 2004  Joshua Hoblitt.  All rights reserved.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE.  See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place - Suite 330, Boston, MA  02111-1307, USA.

The full text of the license can be found in the L<perlgpl> Pod as supplied
with Perl 5.8.1 and later.

=head1 SEE ALSO

L<Plucene::Index::Writer>,
L<Plucene::Plugin::Analyzer::PorterAnalyzer>

=cut
