Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 28154) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 28181) @@ -37,5 +37,5 @@ } -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update, +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -48,4 +48,7 @@ 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database + 'minidvodb' => \$minidvodb, # use minidvodb? + 'minidvodb_name|w=s'=> \$minidvodb_name, # miniDVO database name + 'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group 'image-only' => \$image_only, # Print to stdout 'verbose' => \$verbose, # Print to stdout @@ -58,5 +61,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot", + -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", -exitval => 3, ) unless @@ -65,6 +68,9 @@ defined $outroot and defined $camroot and + defined $dvodb and defined $camera; - +if ($minidvodb && !defined($minidvodb_group)) { + my_die( "missing minidvodb_group", $add_id, 3 ); + } my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -103,6 +109,37 @@ my $dtime_addstar = 0; +if (defined $dvodbReal) { + if ($minidvodb) { + + if (!defined($minidvodb_name)) { + #take the active one, if it's not defined on the command line + #reverts would have this already set, for example. + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + + } + # tack on the minidvodb part to the db. + $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; + + } unless ($no_op) { - if (defined $dvodbReal) { + print $dvodbReal; ## addstar can either save the full set of detections, or just @@ -142,6 +179,7 @@ $fpaCommand .= " -path_base $outroot"; $fpaCommand .= " -dtime_addstar $dtime_addstar"; +$fpaCommand .= " -dvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; - +print $fpaCommand; # Add the result into the database unless ($no_update) { @@ -171,4 +209,5 @@ $command .= " -addprocessedexp"; $command .= " -fault $exit_code"; + $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal; $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); Index: trunk/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 28181) +++ trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 28181) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $dvodb,$minidvodb_interval, $minidvodb_group, $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'dvodb|w=s' => \$dvodb, # output DVO database + 'interval|w=s' => \$minidvodb_interval, #interval between creation of minidvodbs (default = 1day) + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --camera --dvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + defined $camera and + defined $outroot and + defined $dvodb; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_group , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +# XXX This is now not used: do we still need it? + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} else { + warn("dvodb undefined:\n"); + exit(4); +} + +if (!defined $minidvodb_interval) { + $minidvodb_interval = 1; +} + + +unless ($no_op) { + + + +#see if there is already one in new state + my $fpaCommand1 = "$addtool -listminidvodbrun"; + $fpaCommand1 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand1 .= " -state 'new'"; + $fpaCommand1 .= " -dbname $dbname" if defined $dbname; + + +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand1, verbose => $verbose); + + + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to list minidvodb database: $error_code\n"); + exit($error_code); + } + + if (scalar(@{$stdout_buf})) { + $error_code = 3; + warn("Unwilling to create minidvodb, one already exists in new state: $error_code\n"); + exit($error_code); + } + + my $creation_date; + my $addRun_count; + my $minidvodb_name; +#find the active one's date, and find out if it has more than 1 addRun in it + + my $fpaCommand2 = "$addtool -listminidvodbrun"; + $fpaCommand2 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand2 .= " -state 'active'"; + $fpaCommand2 .= " -limit 1"; + $fpaCommand2 .= " -dbname $dbname" if defined $dbname; + +#print $fpaCommand2; + + + my ( $success2, $error_code2, $full_buf2, $stdout_buf2, $stderr_buf2 ) = + run(command => $fpaCommand2, verbose => $verbose); + &my_die( "Unable to get listminidvodbrun",$minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success2; + # if it didn't list something in active state (what?) then we definitely need to create a new one + if (defined(@$stdout_buf2)) { + my $metadata2 = $mdcParser->parse(join "", @$stdout_buf2) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components2 = parse_md_list($metadata2) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp2 = $$components2[0]; + $minidvodb_name = $comp2->{minidvodb_name}; + $creation_date = $comp2->{creation_date}; + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + if (!defined($creation_date)) { + &my_die("Unable to parse creation_date", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + } else { + $create_new = 1; #this is to force it to make a new one + } + + #find the number of add_ids that have been proccessed + my $fpaCommand3 = "$addtool -checkminidvodbrunaddrun"; + $fpaCommand3 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand3 .= " -state 'active'"; + $fpaCommand3 .= " -minidvodb_name '$minidvodb_name'" if defined $minidvodb_name; + $fpaCommand3 .= " -limit 1"; + $fpaCommand3 .= " -dbname $dbname" if defined $dbname; + + +my ( $success3, $error_code3, $full_buf3, $stdout_buf3, $stderr_buf3 ) = + run(command => $fpaCommand3, verbose => $verbose); + &my_die( "Unable to get checkminidvodbunaddrun", $minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success3; + + if (defined(@$stdout_buf3)) { #checkminidvodb returns nothing IF there have been no addruns added to the db yet + my $metadata3 = $mdcParser->parse(join "", @$stdout_buf3) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components3 = parse_md_list($metadata3) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp = $$components3[0]; + $addRun_count = $comp->{addRun_count}; + } + if (!defined($addRun_count)) { + ## there's nothing to parse if there's nothing + $addRun_count = 0; + } + + + if ($addRun_count > 30000) { + #it's too big, create_new + $create_new = 1; + } + if ($create_new == 0) { + my $parser = DateTime::Format::Strptime->new( pattern => '%Y-%m-%dT%H:%M:%S', time_zone => "HST" ); + my $creation_dt = $parser->parse_datetime( $creation_date )->mjd; + if ($mjd_start- $creation_dt > $minidvodb_interval && $addRun_count > 0 ) { + #db is old and has stuff in it, want to create_new + $create_new = 1; + } + } + + +} +#create the minidvodb entry (well, the command for it) + my $fpaCommand = "$addtool -addminidvodbrun"; + $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + unless ($no_update or !$create_new) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $minidvodb_group = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/minidvodb_merge.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_merge.pl (revision 28181) +++ trunk/ippScripts/scripts/minidvodb_merge.pl (revision 28181) @@ -0,0 +1,240 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_merge; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb, $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb|w=s' => \$mergedvodb, # output DVO database + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $mergedvodb and + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodb, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} + +my $dtime_addstar = 0; +#my $dtime_relphot = 0; + +unless ($no_op) { + if (defined $mergedvodbReal) { + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $mergedvodbReal"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $mergedvodb, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + print "addstar -resort time $dtime_addstar\n"; + } + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $mergedvodbReal"; + + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $mergedvodb, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $command = "$addtool -listminidvodbrun -state mergedsdfsad -minidvodb_group " . $minidvodb_group; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + $this_is_the_first = 0; + } + } + + { + my $merge_command; + if ($this_is_the_first) { + $merge_command = "cp -rp $minidvodb $mergedvodb" + } else { + $merge_command = "$dvomerge $minidvodb into $mergedvodb" + } + + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $mergedvodb, $error_code); + } + + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; + } + + + { + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code); + } + } + + } else { + &my_die("dvodb: $mergedvodb not found", $mergedvodb, $PS_EXIT_UNKNOWN_ERROR); + } +} else { + print "skipping processing for $mergedvodbReal\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + + + + #print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__