Index: unk/doc/misc/.perltidyrc =================================================================== --- /trunk/doc/misc/.perltidyrc (revision 3091) +++ (revision ) @@ -1,52 +1,0 @@ -# $Id: .perltidyrc,v 1.6 2004-12-01 22:34:22 jhoblitt Exp $ - -# check the syntax with perl --syn --pscf=-wc - -# enable perltiday warnings --w - -# use only \n as a line ending --ole=unix - -# indent 4 spaces --i=4 - -# continued statements get indented 4 spaces --ci=4 - -# maximum line length --l=110 - -# edit in place but backup the file first --b - -# cuddled elses --ce - -# Cish tight containers --bt=2 --pt=2 --sbt=2 - -# place the brace on the right after a multi-line expression --bar - -# don't indent closing tokens --cti=0 - -# no spaces before semicolons in Cish for loops --nsfs - -# no outdenting long lines --noll - -# don't outdent labels --nola - -# treat ## as commented code and not a comment --sbc - -# opening sub brace on a new line --sbl Index: unk/doc/misc/buildIndex.pl =================================================================== --- /trunk/doc/misc/buildIndex.pl (revision 3091) +++ (revision ) @@ -1,212 +1,0 @@ -#!/usr/bin/perl - -# Copyright (C) 2004 Joshua Hoblitt -# -# $Id: buildIndex.pl,v 1.1 2004/04/20 22:16:17 jhoblitt Exp $ - -use 5.008; - -use strict; -use warnings; - -our $VERSION = '0.01'; - -use File::Basename qw(); -use File::Find::Rule; -##use HTML::Strip; -use HTML::TreeBuilder; -use Plucene::Index::Writer; -use Plucene::Plugin::Analyzer::PorterAnalyzer; - -use Getopt::Long qw( GetOptions :config auto_help auto_version ); -use Pod::Usage qw( pod2usage ); - -my ($verbose, $help, $path, $indexDir); -GetOptions( - 'verbose' => \$verbose, - 'path=s' => \$path, - 'index=s' => \$indexDir, - ) - or pod2usage(2); - -pod2usage(-msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; -pod2usage(-msg => "Required option: --path|-p", -exitval => 2) unless $path; - -$indexDir = $path . "/.index" unless defined $indexDir; - -##my $stripper = HTML::Strip->new; - -my $writer = Plucene::Index::Writer->new( - $indexDir, - Plucene::Plugin::Analyzer::PorterAnalyzer->new(), - 1 # Create the index from scratch -); - -my $doc = Plucene::Document->new; - -foreach my $filename (File::Find::Rule->file->name(qr/^msg\d+\.html/)->in($path)) { - print "indexing: $filename\n" if $verbose; - $writer->add_document(parseMhonarcHtml($filename)); -} - -sub parseMhonarcHtml -{ - my $filename = shift; - - my $tree = HTML::TreeBuilder->new_from_file($filename); - my $doc = Plucene::Document->new; - - $doc->add(Plucene::Document::Field->Keyword(file_name => File::Basename::basename($filename),)); - - # get the header values for to, cc, from, date, and subject - - # find the
tag labeled header - my $headerTag = $tree->look_down(_tag => "div", id => "header"); - - my %headers; - - foreach my $rowNode ($headerTag->look_down(_tag => "tr")) { - - # process each row of the header - - # get name of header field - my $nameNode = $rowNode->look_down(_tag => "td"); - my $name = lc($nameNode->look_down(_tag => "em")->as_text); - $nameNode->delete; - - # get content of header field - my $contentNode = $rowNode->look_down(_tag => "td"); - my $content = $contentNode->as_text; - - # index "cc" as part of "to" - $name =~ s/^cc/to/; - - # index header fields - if ($name =~ /^(to|from|subject|date)/i) { - ##$content = $stripper->parse( $content ); - print "\tadding to index $name: $content\n" if $verbose; - $doc->add(Plucene::Document::Field->Text($name => $content)); - } - } - - # find the
tag labeled content - my $contentTag = $tree->root->look_down(_tag => "div", id => "content"); - - # index message body - print "\tadding to index content\n" if $verbose; - $doc->add(Plucene::Document::Field->UnStored(content => $contentTag->as_text)); - - return $doc; -} - -__END__ - -=pod - -=head1 NAME - -buildIndex.pl - generate Plucene/Lucene index of MHonarc archives - -=head1 SYNOPSIS - - buildIndex.pl --path dir [--index dir] [--verbose] - - or - - buildIndex.pl -p dir [-i dir] [-v] - - or - - buildIndex.pl [--help | -h | -? | --version] - -=head1 DESCRIPTION - -Parses directories of MHonarc generated HTML files and builds a searchable -index. The parser requires the HTML files to be in a specifc format which is -signifigantly modified from default MHonarc format. The indexer creates a -directory of index files in the Plucene/Lucene format. - -=head1 OPTIONS - -=over 4 - -=item * --path dir | -p dir - -Directory of MHonarc generated HTML files to process. - -=item * --index dir | -i dir - -Directory to write the Plucene/Lucene index files to. Defaults to C<--path> + '.index'. - -=item * --verbose | -v - -Print status information while processing. - -=item * --help | -h | -? - -=item * --version - -=back - -=head1 DEVELOPER NOTES - -The parser extracts the C, C, C, C, and C headers -along with the message body. The HTML document is required to contain tags in -this format. - - - -
- [ message body ] -
- -The parser is insenstive to the ordering of the C
and C -CdivE> tags. All other tags in the document are ignored. - -=head1 CREDITS - -Just me, myself, and I. - -=head1 SUPPORT - -Please contact the author directly via e-mail. - -=head1 AUTHOR - -Joshua Hoblitt - -=head1 COPYRIGHT - -Copyright (C) 2004 Joshua Hoblitt. All rights reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place - Suite 330, Boston, MA 02111-1307, USA. - -The full text of the license can be found in the L Pod as supplied -with Perl 5.8.1 and later. - -=head1 SEE ALSO - -L, -L - -=cut Index: /trunk/doc/misc/perlCodeConventions.tex =================================================================== --- /trunk/doc/misc/perlCodeConventions.tex (revision 3091) +++ /trunk/doc/misc/perlCodeConventions.tex (revision 3092) @@ -1,3 +1,3 @@ -%%% $Id: perlCodeConventions.tex,v 1.45 2005-01-25 21:44:01 jhoblitt Exp $ +%%% $Id: perlCodeConventions.tex,v 1.46 2005-01-25 21:48:54 jhoblitt Exp $ \documentclass[panstarrs]{panstarrs} @@ -1584,5 +1584,5 @@ convention. -\verbatiminput{buildIndex.pl} +\verbatiminput{perlCodeExamples/buildIndex.pl} @@ -1603,5 +1603,5 @@ Example \file{.perltidyrc} file to achieve the Perl code convention: -\verbatiminput{.perltidyrc} +\verbatiminput{perlCodeExamples/.perltidyrc} Index: /trunk/doc/misc/perlCodeExamples/.perltidyrc =================================================================== --- /trunk/doc/misc/perlCodeExamples/.perltidyrc (revision 3092) +++ /trunk/doc/misc/perlCodeExamples/.perltidyrc (revision 3092) @@ -0,0 +1,52 @@ +# $Id: .perltidyrc,v 1.1 2005-01-25 21:48:54 jhoblitt Exp $ + +# check the syntax with perl +-syn +-pscf=-wc + +# enable perltiday warnings +-w + +# use only \n as a line ending +-ole=unix + +# indent 4 spaces +-i=4 + +# continued statements get indented 4 spaces +-ci=4 + +# maximum line length +-l=110 + +# edit in place but backup the file first +-b + +# cuddled elses +-ce + +# Cish tight containers +-bt=2 +-pt=2 +-sbt=2 + +# place the brace on the right after a multi-line expression +-bar + +# don't indent closing tokens +-cti=0 + +# no spaces before semicolons in Cish for loops +-nsfs + +# no outdenting long lines +-noll + +# don't outdent labels +-nola + +# treat ## as commented code and not a comment +-sbc + +# opening sub brace on a new line +-sbl Index: /trunk/doc/misc/perlCodeExamples/buildIndex.pl =================================================================== --- /trunk/doc/misc/perlCodeExamples/buildIndex.pl (revision 3092) +++ /trunk/doc/misc/perlCodeExamples/buildIndex.pl (revision 3092) @@ -0,0 +1,212 @@ +#!/usr/bin/perl + +# Copyright (C) 2004 Joshua Hoblitt +# +# $Id: buildIndex.pl,v 1.1 2005-01-25 21:48:54 jhoblitt Exp $ + +use 5.008; + +use strict; +use warnings; + +our $VERSION = '0.01'; + +use File::Basename qw(); +use File::Find::Rule; +##use HTML::Strip; +use HTML::TreeBuilder; +use Plucene::Index::Writer; +use Plucene::Plugin::Analyzer::PorterAnalyzer; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($verbose, $help, $path, $indexDir); +GetOptions( + 'verbose' => \$verbose, + 'path=s' => \$path, + 'index=s' => \$indexDir, + ) + or pod2usage(2); + +pod2usage(-msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; +pod2usage(-msg => "Required option: --path|-p", -exitval => 2) unless $path; + +$indexDir = $path . "/.index" unless defined $indexDir; + +##my $stripper = HTML::Strip->new; + +my $writer = Plucene::Index::Writer->new( + $indexDir, + Plucene::Plugin::Analyzer::PorterAnalyzer->new(), + 1 # Create the index from scratch +); + +my $doc = Plucene::Document->new; + +foreach my $filename (File::Find::Rule->file->name(qr/^msg\d+\.html/)->in($path)) { + print "indexing: $filename\n" if $verbose; + $writer->add_document(parseMhonarcHtml($filename)); +} + +sub parseMhonarcHtml +{ + my $filename = shift; + + my $tree = HTML::TreeBuilder->new_from_file($filename); + my $doc = Plucene::Document->new; + + $doc->add(Plucene::Document::Field->Keyword(file_name => File::Basename::basename($filename),)); + + # get the header values for to, cc, from, date, and subject + + # find the
tag labeled header + my $headerTag = $tree->look_down(_tag => "div", id => "header"); + + my %headers; + + foreach my $rowNode ($headerTag->look_down(_tag => "tr")) { + + # process each row of the header + + # get name of header field + my $nameNode = $rowNode->look_down(_tag => "td"); + my $name = lc($nameNode->look_down(_tag => "em")->as_text); + $nameNode->delete; + + # get content of header field + my $contentNode = $rowNode->look_down(_tag => "td"); + my $content = $contentNode->as_text; + + # index "cc" as part of "to" + $name =~ s/^cc/to/; + + # index header fields + if ($name =~ /^(to|from|subject|date)/i) { + ##$content = $stripper->parse( $content ); + print "\tadding to index $name: $content\n" if $verbose; + $doc->add(Plucene::Document::Field->Text($name => $content)); + } + } + + # find the
tag labeled content + my $contentTag = $tree->root->look_down(_tag => "div", id => "content"); + + # index message body + print "\tadding to index content\n" if $verbose; + $doc->add(Plucene::Document::Field->UnStored(content => $contentTag->as_text)); + + return $doc; +} + +__END__ + +=pod + +=head1 NAME + +buildIndex.pl - generate Plucene/Lucene index of MHonarc archives + +=head1 SYNOPSIS + + buildIndex.pl --path dir [--index dir] [--verbose] + + or + + buildIndex.pl -p dir [-i dir] [-v] + + or + + buildIndex.pl [--help | -h | -? | --version] + +=head1 DESCRIPTION + +Parses directories of MHonarc generated HTML files and builds a searchable +index. The parser requires the HTML files to be in a specifc format which is +signifigantly modified from default MHonarc format. The indexer creates a +directory of index files in the Plucene/Lucene format. + +=head1 OPTIONS + +=over 4 + +=item * --path dir | -p dir + +Directory of MHonarc generated HTML files to process. + +=item * --index dir | -i dir + +Directory to write the Plucene/Lucene index files to. Defaults to C<--path> + '.index'. + +=item * --verbose | -v + +Print status information while processing. + +=item * --help | -h | -? + +=item * --version + +=back + +=head1 DEVELOPER NOTES + +The parser extracts the C, C, C, C, and C headers +along with the message body. The HTML document is required to contain tags in +this format. + + + +
+ [ message body ] +
+ +The parser is insenstive to the ordering of the C
and C +CdivE> tags. All other tags in the document are ignored. + +=head1 CREDITS + +Just me, myself, and I. + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2004 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, +L + +=cut