Index: /trunk/doc/misc/buildIndex.pl =================================================================== --- /trunk/doc/misc/buildIndex.pl (revision 2471) +++ /trunk/doc/misc/buildIndex.pl (revision 2471) @@ -0,0 +1,211 @@ +#!/usr/bin/perl + +# Copyright (C) 2004 Joshua Hoblitt +# +# $Id: buildIndex.pl,v 1.1 2004/04/20 22:16:17 jhoblitt Exp $ + +use strict; +use warnings; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use File::Basename qw(); +use File::Find::Rule; +##use HTML::Strip; +use HTML::TreeBuilder; +use Plucene::Index::Writer; +use Plucene::Plugin::Analyzer::PorterAnalyzer; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($verbose, $help, $path, $index_dir); +GetOptions( + 'verbose' => \$verbose, + 'path=s' => \$path, + 'index=s' => \$index_dir, + ) + or pod2usage(2); + +pod2usage(-msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; +pod2usage(-msg => "Required option: --path|-p", -exitval => 2) unless $path; + +$index_dir = $path . "/.index" unless defined $index_dir; + +##my $stripper = HTML::Strip->new; + +my $writer = Plucene::Index::Writer->new( + $index_dir, + Plucene::Plugin::Analyzer::PorterAnalyzer->new(), + 1 # Create the index from scratch +); + +my $doc = Plucene::Document->new; + +foreach my $file_name (File::Find::Rule->file->name(qr/^msg\d+\.html/)->in($path)) { + print "indexing: $file_name\n" if $verbose; + $writer->add_document(parse_mhonarc_html($file_name)); +} + +sub parse_mhonarc_html +{ + my $file_name = shift; + + my $tree = HTML::TreeBuilder->new_from_file($file_name); + my $doc = Plucene::Document->new; + + $doc->add(Plucene::Document::Field->Keyword(file_name => File::Basename::basename($file_name),)); + + # get the header values for to, cc, from, date, and subject + + # find the
tag labeled header + my $header_tag = $tree->look_down(_tag => "div", id => "header"); + + my %headers; + + foreach my $row_node ($header_tag->look_down(_tag => "tr")) { + + # process each row of the header + + # get name of header field + my $name_node = $row_node->look_down(_tag => "td"); + my $name = lc($name_node->look_down(_tag => "em")->as_text); + $name_node->delete; + + # get content of header field + my $content_node = $row_node->look_down(_tag => "td"); + my $content = $content_node->as_text; + + # index "cc" as part of "to" + $name =~ s/^cc/to/; + + # index header fields + if ($name =~ /^(to|from|subject|date)/i) { + ##$content = $stripper->parse( $content ); + print "\tadding to index $name: $content\n" if $verbose; + $doc->add(Plucene::Document::Field->Text($name => $content)); + } + } + + # find the
tag labeled content + my $content_tag = $tree->root->look_down(_tag => "div", id => "content"); + + # index message body + print "\tadding to index content\n" if $verbose; + $doc->add(Plucene::Document::Field->UnStored(content => $content_tag->as_text)); + + return $doc; +} + +__END__ + +=pod + +=head1 NAME + +build_index.pl - generate Plucene/Lucene index of MHonarc archives + +=head1 SYNOPSIS + + build_index.pl --path dir [--index dir] [--verbose] + + or + + build_index.pl -p dir [-i dir] [-v] + + or + + build_index.pl [--help | -h | -? | --version] + +=head1 DESCRIPTION + +Parses directories of MHonarc generated HTML files and builds a searchable +index. The parser requires the HTML files to be in a specifc format which is +signifigantly modified from default MHonarc format. The indexer creates a +directory of index files in the Plucene/Lucene format. + +=head1 OPTIONS + +=over 4 + +=item * --path dir | -p dir + +Directory of MHonarc generated HTML files to process. + +=item * --index dir | -i dir + +Directory to write the Plucene/Lucene index files to. Defaults to C<--path> + '.index'. + +=item * --verbose | -v + +Print status information while processing. + +=item * --help | -h | -? + +=item * --version + +=back + +=head1 DEVELOPERS + +The parser extracts the C, C, C, C, and C headers +along with the message body. The HTML document is required to contain tags in +this format. + + + +
+ [ message body ] +
+ +The parser is insenstive to the ordering of the C
and C +CdivE> tags. All other tags in the document are ignored. + +=head1 CREDITS + +Just me, myself, and I. + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2004 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the LICENSE file included with +this module, or in the L Pod as supplied with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, +L + +=cut Index: /trunk/doc/misc/perlCodeConventions.tex =================================================================== --- /trunk/doc/misc/perlCodeConventions.tex (revision 2470) +++ /trunk/doc/misc/perlCodeConventions.tex (revision 2471) @@ -1,3 +1,3 @@ -%%% $Id: perlCodeConventions.tex,v 1.34 2004-11-25 01:51:32 jhoblitt Exp $ +%%% $Id: perlCodeConventions.tex,v 1.35 2004-11-25 02:02:33 jhoblitt Exp $ \documentclass[panstarrs]{panstarrs} @@ -1180,8 +1180,8 @@ \label{SourceExample} -An example of \file{parseErrorCode.pl} that conforms with the Perl code +An example of \file{buildIndex.pl} that conforms with the Perl code convention. -\verbatiminput{parseErrorCodes.pl} +\verbatiminput{buildIndex.pl} %------------------------------------------------------------------------------