Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 33019) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 33030) @@ -30,4 +30,5 @@ my $missing_tools; my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); @@ -37,5 +38,6 @@ } my $minidvodb_path; -my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my $stage_extra1; +my ( $add_id, $camera, $stage, $stage_id, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -43,4 +45,5 @@ 'camera|c=s' => \$camera, # Camera 'stage|s=s' => \$stage, # Camera + 'stage_id|w=s' => \$stage_id, 'multi_num|w=s' => \$multi_num, # the number for a staticskymulti (for finding cmf) 'dbname|d=s' => \$dbname, # Database name @@ -71,4 +74,8 @@ defined $dvodb and defined $camera; +if ($stage =~ /cam/ && !defined $stage_id) { + my_die("cam stage requires -stage_id", $add_id, 3); + +} if ($minidvodb && !defined($minidvodb_group)) { my_die( "missing minidvodb_group", $add_id, 3 ); @@ -108,11 +115,52 @@ # it was PSASTRO.OUTPUT -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($stage =~ /cam/) { + # if it is cam stage we need to be careful when grabbing the filename. + # This breaks down into a few steps: + + #get info about the cam_id + my $magicked; + { + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$camtool -processedexp -cam_id " . $stage_id; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR); + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mparsed = $comp->{magicked}; + + if (!defined($mparsed)) { + &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found a value for magicked:$mparsed\n"; + $magicked = $mparsed; + } + + #is this cam_id magicked or not? + if ($magicked) { + $stage_extra1 = $magicked; + $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + print "cam_id is magicked, using $fpaObjects for the cam smf\n"; + } else { + print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n"; + } + } +} if (($stage =~/staticsky/) || ($stage =~/stack/)) { $fpaObjects =~ s/smf$/cmf/; - -} - +} if ($stage =~/staticsky_multi/) { $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above @@ -120,9 +168,8 @@ my $nice_num = sprintf ("%03d", $multi_num); $fpaObjects =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc - } my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + # convert supplied DVO database name to UNIX filename my $dvodbReal; @@ -131,84 +178,69 @@ $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); } - - - - my $dtime_addstar = 0; - if (defined $dvodbReal) { - if ($minidvodb) { - - #if (!defined($minidvodb_name)) { - #take the active one, if it's not defined on the command line - #reverts would have this already set, for example. - my $command = "addtool -listminidvodbrun "; - $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; - $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; - $command .= " -state 'active' -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - print $command; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - $minidvodb_path = $comp->{minidvodb_path}; - $minidvodb_name = $comp->{minidvodb_name}; - - if (!defined($minidvodb_path)) { - &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); - } - if (!defined($minidvodb_name)) { - &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); - } - #} - # tack on the minidvodb part to the db. -# $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; -#we don't need this now that I fixed the paths - - } - else { - - $minidvodb_path = $dvodbReal; - } - -unless ($no_op) { + if ($minidvodb) { + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_path)) { + &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); + } + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + } + else { + $minidvodb_path = $dvodbReal; + } + + unless ($no_op) { print $dvodbReal; - - ## addstar can either save the full set of detections, or just - ## the image metadata, in the dvodb. this is set in the - ## database table addRun - - # addstar requires the user to have a valid .ptolemyrc which - # in turn points at ippconfig/dvo.site - - # get the names for the camera and the real input file - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); - - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - my $command = "$addstar -update"; # XXX optionally set -update? - $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $minidvodb_path"; - $command .= " $realFile"; - $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + + ## addstar can either save the full set of detections, or just + ## the image metadata, in the dvodb. this is set in the + ## database table addRun + + # addstar requires the user to have a valid .ptolemyrc which + # in turn points at ippconfig/dvo.site + + # get the names for the camera and the real input file + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); + + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + my $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; + $command .= " -D CATDIR $minidvodb_path"; + $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference $command .= " -image" if $image_only; if ($stage =~ /staticsky/) { $command .= " -accept-astrom "; } #careful here - this matches staticsky and staticsky_multi - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script } } @@ -221,4 +253,5 @@ $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; +$fpaCommand .= " -stage_extra1" if defined $stage_extra1; $fpaCommand .= " -dbname $dbname" if defined $dbname; print $fpaCommand; @@ -253,6 +286,8 @@ $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + # $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure) + $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $command .= " -dbname $dbname" if defined $dbname; + print $command; system ($command); } Index: trunk/ippScripts/scripts/mergedvodb_copy.pl =================================================================== --- trunk/ippScripts/scripts/mergedvodb_copy.pl (revision 33030) +++ trunk/ippScripts/scripts/mergedvodb_copy.pl (revision 33030) @@ -0,0 +1,189 @@ +#!/usr/bin/env perl + +#This is essentially the same as minidvodb_copy.pl - the tables are essentially identical, and the task (to rsync a copy) is essentially identical. A new script was created for rsyncing the merges in case they should diverge in the future. + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; + +my $dtime; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $rsync = can_run('rsync') or (warn "Can't find rsync" and $missing_tools = 1); +my $mergedvodbtool = can_run('mergedvodbtool') or (warn "Can't find mergedvodbtool" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $df = can_run('df') or (warn "Can't find df" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my ( $mergedvodbcopy_id, $merge_id, $mergedvodbrun_path, $mergedvodb_rsync_path, $destination_host, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodbcopy_id|w=s' => \$mergedvodbcopy_id, #minidvodb database + 'merge_id|w=s' => \$mergedvodb_id, #minidvodb_id + 'mergedvodbrun_path|w=s' => \$mergedvodbrun_path, #minidvodb_id + 'mergedvodb_rsync_path|w=s' => \$mergedvodb_rsync_path, + 'destination_host|w=s' => \$destination_host, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodbcopy_id --mergedvodb_rsync_path --mergedvodbrun_path --destination_host", + -exitval => 3, + ) unless + defined $mergedvodbcopy_id and + defined $mergedvodb_rsync_path and + defined $mergedvodbrun_path and + defined $destination_host; + + + +my $ipprc = PS::IPP::Config->new(); + +my @df; + + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodbcopy_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +{ #can we rsync? + + my $sizes = `du -sk $mergedvodbrun_path`; + my $total = 0; + for(split /[\r\n]+/,$sizes) # split on one or more newline characters + { + my($number,$file) = split /\t/,$_,2; # split on tab ($file not used here) + $total += $number; + } +#print 'Total: '.$total; +{ + print "Checking available diskspace on $destination_host\n\n"; + + my $command = "$ssh $destination_host df $mergedvodb_rsync_path"; + print "$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform '$ssh $destination_host df $mergedvodb_rsync_path' $error_code", $minidvodbcopy_id, $error_code); + } + + my @fullbuf= split(/\n/, @{$stdout_buf}[0]); + # print $#{$stdout_buf}; +# print "xx $#df xx"; +# print $fullbuf[0]; + if ($#fullbuf ==1 ) { +# print $fullbuf[1]; + @df = split /\s+/, $fullbuf[1]; + if ($#df == 5) { +# + #print $df[3], $df[4]; + + $df[4]=~ s/\%//; + + if ($df[4] < 97 && $df[3]-$total) { + print "\nThere is enough available space ($total < $df[3]) for the rsync\n"; + } else { + &my_die("Not enough available disk space: $df[4]% available, $df[3] on destination, $total free required for copy", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +# + + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nCan't figure out %free or available", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nToo many lines ($#fullbuf) vs 1", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +} +} + +{#do the rsync + my $command = "$rsync -rvuaq"; + $command .= " $mergedvodbrun_path"; + $command .= " $destination_host"; + $command .= ":$mergedvodb_rsync_path"; + print "\nPerforming rsync:\n"; + print "$command\n\n"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform rsync: $error_code", $mergedvodbcopy_id, $error_code); + } + + print "Rsync complete..\n"; + +} + +{#update the database + $dtime = 86400.0*(DateTime->now->mjd - $mjd_start); + + my $command = "$mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_state full -set_dtime $dtime"; + $command .= " -dbname $dbname" if defined $dbname; + print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = 20; + &my_die("Unable to update database - rsync completed without error' $error_code", $mergedvodbcopy_id, $error_code); + } + +} +exit 0; + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $mergedbvodbcopy_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodbcopy_id\n"; + +if (defined $mergedvodb_id ) { + my $command = "mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_fault $exit_code"; + $command .= " -set_dtime $dtime" if defined $dtime; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: trunk/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- trunk/ippScripts/scripts/mergedvodb_merge.pl (revision 33030) +++ trunk/ippScripts/scripts/mergedvodb_merge.pl (revision 33030) @@ -0,0 +1,237 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_merge; +my $dtime_verify; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb_path, $merge_id, $mergedvodb, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb_path|w=s' => \$mergedvodb_path, # output DVO database + 'minidvodb_path|w=s' => \$minidvodb_path, #minidvodb database + 'mergedvodb|w=s' => \$mergedvodb, #mergedvodb + 'merge_id|w=s' => \$merge_id, #minidvodb_id + 'dbname|d=s' => \$dbname, # Database name + 'camera|w=s' => \$camera, #camera name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb_path --mergedvodb --camera", + -exitval => 3, + ) unless + defined $mergedvodb_path and + defined $minidvodb_path and + defined $mergedvodb and + defined $merge_id and + defined $camera; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $merge_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $merge_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename + +unless ($no_op) { + + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -faulted -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + print "looking for faulted:\n\n$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $merge_id, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + } + { + my $command = "$dvoverify -s $minidvodb_path"; + print "$command\n"; + my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); + } + $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script + print "dvoverify time $dtime_verify\n"; + } + #Merge + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; + + if (!defined($mergedvodb_path_tmp)) { + &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 mergedvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + if (-e "$mergedvodb_path/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; + } else { + $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + } + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $merge_id, $error_code); + } + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; +# $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); + } + } + +#} else { +# &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); +} +else { + print "skipping processing for $mergedvodb_path\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $merge_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb_path\n"; + +if (defined $merge_id ) { + + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; + $command .= " -fault $exit_code"; + #$command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + + + + print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: trunk/ippScripts/scripts/mergedvodb_queue.pl =================================================================== --- trunk/ippScripts/scripts/mergedvodb_queue.pl (revision 33030) +++ trunk/ippScripts/scripts/mergedvodb_queue.pl (revision 33030) @@ -0,0 +1,164 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +# Ideally, we want to queue new runs with mergetool -definebyquery. We can't for the survey task, because we use the value in 'site.config' to find the mergedvodb_path. I do not know how to tell pantasks how to do this, but I know how to do it in perl. + + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $minidvodb_group, $mergedvodb , $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); + +# we just need any old camera to be able to read site.config. I'm using SIMPLE, unless it is overridden on the commandline (it makes no difference for this case) + +$camera = 'SIMPLE'; + + +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'mergedvodb|w=s' => \$mergedvodb, # output miniDVODB + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + # defined $camera and + defined $outroot and + defined $mergedvodb; + +print "camera is $camera\n"; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); +print "log file is $logDest\n"; +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} else { + warn("mergedvodb undefined:\n"); + exit(4); +} +if (!defined $mergedvodbReal) { + warn("can't parse out mergedvodb_path\n"); + exit(4); +} + + + +print "mergedvodb_path = $mergedvodbReal\n"; + +unless ($no_op) { + + #create the mergedvodb entry (well, the command for it) + my $fpaCommand = "$mergetool -definebyquery"; + $fpaCommand .= " -mergedvodb $mergedvodb"; + $fpaCommand .= " -minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $mergedvodbReal" ; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + print "running the following: \n\n"; + print "$fpaCommand\n\n"; + + unless ($no_update) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $mergedvodb = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 33019) +++ trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 33030) @@ -226,5 +226,4 @@ $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; $fpaCommand .= " -set_minidvodb_path $minidvodbReal" if defined $minidvodbReal; - $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; Index: trunk/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 33030) +++ trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 33030) @@ -0,0 +1,174 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $minidvodb_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +my $dtime_addstar = 0; + +unless ($no_op) { + + #this is chopped into several parts: addstar, relphot + #addstar + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + #relphot + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $minidvodb_id, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $minidvodb_id, $error_code); + } + } +#} else { +# &my_die("dvodb: minidvodb_id = $minidvodb_id not found", $minidvodb_id, $PS_EXIT_UNKNOWN_ERROR); +#} +} else { + print "skipping processing for minidvodb_id = $minidvodb_id\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + print STDERR "$msg $minidvodb_id\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__