Changeset 40803 for trunk/ippScripts/scripts/addstar_multi_run.pl
- Timestamp:
- Jun 20, 2019, 10:20:26 AM (7 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/addstar_multi_run.pl (modified) (7 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/addstar_multi_run.pl
r39160 r40803 33 33 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); 34 34 my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); 35 35 36 if ($missing_tools) { 36 37 warn("Can't find required tools."); … … 40 41 41 42 my ( $label,$camera, $stage, $stage_id, $use_diff_inv, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, 42 $no_op, $redirect, $save_temps );43 $no_op, $redirect, $save_temps, $limit_number); 43 44 GetOptions( 44 'label=s' => \$label, #addstar label 45 'camera|c=s' => \$camera, # Camera 46 'stage|s=s' => \$stage, # Camera 47 'stage_id|w=s' => \$stage_id, 48 'use_diff_inv' => \$use_diff_inv, #use inv images for diff cmf? 49 'dbname|d=s' => \$dbname, # Database name 50 'outroot|w=s' => \$outroot, # output file base name 51 'stageroot|w=s' => \$stageroot, # stage root name. 52 'reduction=s' => \$reduction, # Reduction class 53 'dvodb|w=s' => \$dvodb, # output DVO database 54 'minidvodb_name|w=s'=> \$minidvodb_name, # miniDVO database name 55 'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group 56 'minidvodb' => \$minidvodb, # use minidvodb? 57 'image-only' => \$image_only, # Print to stdout 58 'verbose' => \$verbose, # Print to stdout 59 'no-update' => \$no_update, # Update the database? 60 'no-op' => \$no_op, # Don't do any operations? 61 'redirect-output' => \$redirect, 62 'save-temps' => \$save_temps, # Save temporary files? 45 'label=s' => \$label, # addstar label 46 'stage=s' => \$stage, # stage (camera, fullforce, etc) 47 'stage_id=s' => \$stage_id, 48 'camera=s' => \$camera, # Camera 49 'outroot=s' => \$outroot, # output file base name 50 'dvodb=s' => \$dvodb, # output DVO database 51 'stageroot=s' => \$stageroot, # stage root name. 52 'limit=i' => \$limit_number, 53 'use_diff_inv' => \$use_diff_inv, # use inv images for diff cmf? 54 'dbname=s' => \$dbname, # Database name 55 'reduction=s' => \$reduction, # Reduction class 56 'minidvodb_name=s' => \$minidvodb_name, # miniDVO database name 57 'minidvodb_group=s' => \$minidvodb_group, # miniDVO database group 58 'minidvodb' => \$minidvodb, # use minidvodb? 59 'image-only' => \$image_only, # Print to stdout 60 'verbose' => \$verbose, # Print to stdout 61 'no-update' => \$no_update, # Update the database? 62 'no-op' => \$no_op, # Don't do any operations? 63 'redirect-output' => \$redirect, 64 'save-temps' => \$save_temps, # Save temporary files? 63 65 ) or pod2usage( 2 ); 64 66 65 67 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; 66 68 pod2usage( 67 -msg => "Required options: --label --stage _id --stage--camera --outroot --dvodb --stageroot",69 -msg => "Required options: --label --stage --stage_id --camera --outroot --dvodb --stageroot", 68 70 -exitval => 3, 69 71 ) unless … … 111 113 #my $temp_file = $outroot.".log.".$label.".".$stage.".".$stage_id.".list"; 112 114 my $temp_file = "/tmp/addstar.list"; 115 # XXX this temp_file is dangerous: multiple instances of this program will overwrite 113 116 114 117 print "using $temp_file for list\n"; … … 116 119 open( TEMPLIST, ">$temp_file") or &my_die( "Can't open $temp_file\n", $stage_id,$stage,$label, $PS_EXIT_UNKNOWN_ERROR); 117 120 121 my @stage_extra1 = (); 122 123 # query database for list of exposures: we need to generate the input list for addstar 118 124 { 119 125 my $mdcParser = PS::IPP::Metadata::Config->new; 120 my $command = "$addtool -pendingexp -stage_id ".$stage_id; 121 $command .= " -stage $stage -label $label"; 122 $command .= " -dbname $dbname" if defined $dbname; 123 124 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 125 run(command => $command, verbose => $verbose); 126 127 my $command = "$addtool -pendingexp"; 128 $command .= " -stage_id $stage_id"; 129 $command .= " -stage $stage"; 130 $command .= " -label $label"; 131 $command .= " -dbname $dbname" if defined $dbname; 132 $command .= " -limit $limit_number" if defined $limit_number; 133 134 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 135 run(command => $command, verbose => $verbose); 126 136 &my_die( "Unable to get info on addRun", $stage_id,$stage, $label, $PS_EXIT_SYS_ERROR) unless $success; 137 127 138 if (scalar @$stdout_buf == 0 ) { 128 #&my_die( "empty addRun", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR); 139 # if there are no entries, just exit (do not update database with a fault) 140 # &my_die( "empty addRun", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR); 129 141 print "empty addRun, exiting... $stage, $stage_id, $label\n"; 130 142 exit 0; 131 } else { 132 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 133 &my_die("Unable to parse metadata config", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 134 #this fails if there is nothing listed. I checked. 135 my $components = parse_md_list($metadata) or 136 &my_die("Unable to parse metadata list", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 137 my $num_components = scalar @$components; 138 print "there are $num_components\n"; 139 foreach my $comp (@$components) { 140 my $mparsed = $comp->{stageroot}; 141 142 if (!defined($mparsed)) { 143 &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 144 } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. 145 print "found a value for stageroot:$mparsed\n"; 146 my $fpaObjects = $mparsed.'.cmf'; 147 my $realFile = $ipprc->file_resolve($fpaObjects); 148 if (!defined($realFile)) { 149 &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7); 150 } 151 printf TEMPLIST "$realFile = $fpaObjects\n"; 152 if ($stage =~ /diff/) { # hardwired for now$use_diff_inv) { 153 print "finding the inv.cmf files\n"; 154 my $fpaObjectsInv = $mparsed.'.inv.cmf'; 155 my $realFileInv = $ipprc->file_resolve($fpaObjectsInv); 156 if (!defined($realFileInv)) { 157 &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7); 158 } 159 printf TEMPLIST "$realFileInv = $fpaObjectsInv\n"; 160 } 161 143 } 144 145 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or 146 &my_die("Unable to parse metadata config", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 147 # this fails if there is nothing listed. I checked. 148 my $components = parse_md_list($metadata) or 149 &my_die("Unable to parse metadata list", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 150 my $num_components = scalar @$components; 151 print "there are $num_components\n"; 152 153 # go through the list of components to create addstar input list 154 foreach my $comp (@$components) { 155 my $mparsed = $comp->{stageroot}; 156 157 if (!defined($mparsed)) { 158 &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 159 } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. 160 print "found a value for stageroot:$mparsed\n"; 161 162 my $fpaObjects = $mparsed . '.cmf'; 163 my $realFile = $ipprc->file_resolve($fpaObjects); 164 if (!defined($realFile)) { 165 &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7); 166 } 167 168 printf TEMPLIST "$realFile = $fpaObjects\n"; 169 if ($stage =~ /diff/) { # hardwired for now: $use_diff_inv) { 170 print "finding the inv.cmf files\n"; 171 my $fpaObjectsInv = $mparsed . '.inv.cmf'; 172 my $realFileInv = $ipprc->file_resolve($fpaObjectsInv); 173 if (!defined($realFileInv)) { 174 &my_die("unable to resolve real file from $mparsed", $stage_id, $stage, $label, 8); 162 175 } 163 164 } 165 } 176 printf TEMPLIST "$realFileInv = $fpaObjectsInv\n"; 177 } 178 179 # save the stage_extra1 values so we can update the database correctly at the end 180 my $stage_extra1_value = $comp->{stage_extra1}; 181 if (!defined($stage_extra1_value)) { 182 &my_die("unable to find stage_extra1 value for $mparsed", $stage_id, $stage, $label, 9); 183 } 184 push @stage_extra1, $stage_extra1_value; 185 } 186 } 187 166 188 close(TEMPLIST); 167 189 print "saved $temp_file here\n"; … … 173 195 $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $stage_id,$stage,$label, $PS_EXIT_CONFIG_ERROR); 174 196 } 197 175 198 my $dtime_addstar = 0; 176 199 if (defined $dvodbReal) { … … 200 223 &my_die("Unable to parse minidvodb_name", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR); 201 224 } 202 } 203 else { 225 } else { 204 226 $minidvodb_path = $dvodbReal; 205 227 } 206 228 207 229 unless ($no_op) { 208 print $dvodbReal; 209 210 ## addstar can either save the full set of detections, or just 211 ## the image metadata, in the dvodb. this is set in the 212 ## database table addRun 213 214 # addstar requires the user to have a valid .ptolemyrc which 215 # in turn points at ippconfig/dvo.site 216 217 # get the names for the camera and the real input file 218 my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar 219 220 # my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR); 221 222 # most cameras use CHIP_HEADER. other exceptions: 223 # gpc1 @ camera : PHU_HEADER 224 # gpc1 @ staticsky : NOMINAL (is uncalibrated) 225 226 # temporary hard-wired info 227 my $zeroPointOption = "CHIP_HEADER"; 228 if (($camdir =~ /gpc1/) && ($stage =~ /staticsky/)) { 229 $zeroPointOption = "NOMINAL"; 230 } 231 if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) { 232 $zeroPointOption = "PHU_HEADER"; 233 } 234 235 # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) 236 my $command = "$addstar -update"; # XXX optionally set -update? 237 $command .= " -D CAMERA $camdir"; 238 $command .= " -D CATDIR $minidvodb_path"; 239 $command .= " -D ZERO_POINT_OPTION $zeroPointOption"; 240 $command .= " -list $temp_file"; 241 $command .= " -image" if $image_only; 242 if ($stage =~ /staticsky/) { 243 $command .= " -accept-astrom -quick-airmass"; 244 } #careful here - this matches staticsky and staticsky_multi 245 if ($stage =~ /skycal/) { 246 $command .= " -quick-airmass"; 247 } #careful here - this matches staticsky and staticsky_multi 248 if ($stage =~ /diff/) { 249 $command .= " -accept-astrom"; 250 if ($use_diff_inv) { 251 $command .= " -diff-inv"; 252 } 230 print $dvodbReal; 231 232 ## addstar can either save the full set of detections, or just 233 ## the image metadata, in the dvodb. this is set in the 234 ## database table addRun 235 236 # addstar requires the user to have a valid .ptolemyrc which 237 # in turn points at ippconfig/dvo.site 238 239 # get the names for the camera and the real input file 240 my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar 241 242 # most cameras use CHIP_HEADER. other exceptions: 243 # gpc1 @ camera : PHU_HEADER 244 # gpc1 @ staticsky : NOMINAL (is uncalibrated) 245 246 # temporary hard-wired info 247 my $zeroPointOption = "CHIP_HEADER"; 248 if (($camdir =~ /gpc1/) && ($stage =~ /staticsky/)) { 249 $zeroPointOption = "NOMINAL"; 250 } 251 if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) { 252 $zeroPointOption = "PHU_HEADER"; 253 } 254 255 # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) 256 my $command = "$addstar -update"; # XXX optionally set -update? 257 $command .= " -D CAMERA $camdir"; 258 $command .= " -D CATDIR $minidvodb_path"; 259 $command .= " -D ZERO_POINT_OPTION $zeroPointOption"; 260 $command .= " -list $temp_file"; 261 $command .= " -image" if $image_only; 262 if ($stage =~ /staticsky/) { 263 $command .= " -accept-astrom -quick-airmass"; 264 } #careful here - this matches staticsky and staticsky_multi 265 if ($stage =~ /skycal/) { 266 $command .= " -quick-airmass"; 267 } #careful here - this matches staticsky and staticsky_multi 268 if ($stage =~ /diff/) { 269 $command .= " -accept-astrom"; 270 if ($use_diff_inv) { 271 $command .= " -diff-inv"; 253 272 } 254 255 if ($stage =~ /fullforce/) { 256 $command .= " -accept-astrom -xrad"; 257 } 258 259 my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script 260 261 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 262 run(command => $command, verbose => $verbose); 263 unless ($success) { 264 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 265 &my_die("Unable to perform addstar: $error_code", $stage_id,$stage,$label, $error_code); 266 } 267 $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script 268 } 269 } 270 271 272 my $fpaCommand = "$addtool -multiadd -stage_id $stage_id"; 273 } 274 275 if ($stage =~ /fullforce/) { 276 $command .= " -accept-astrom -xrad"; 277 } 278 279 my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script 280 281 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 282 run(command => $command, verbose => $verbose); 283 unless ($success) { 284 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 285 &my_die("Unable to perform addstar: $error_code", $stage_id,$stage,$label, $error_code); 286 } 287 $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script 288 } 289 } 290 291 my $stage_extra1_list = ""; 292 my $Nstage_extra1_list = @stage_extra1; 293 294 for (my $i = 0; $i < $Nstage_extra1_list; $i ++) { 295 $stage_extra1_list .= "$stage_extra1[$i]"; 296 if ($i < $Nstage_extra1_list - 1) { 297 $stage_extra1_list .= ","; 298 } 299 print "STAGE EXTRA1 : $stage_extra1[$i]\n"; 300 } 301 302 # at the end, we need to update the database with the entries we actually processed 303 # these are identified by stage_extra1, listed in the array @stage_extra1 304 my $fpaCommand = "$addtool -addprocessedexp"; 305 $fpaCommand .= " -multiadd"; 273 306 $fpaCommand .= " -stage $stage"; 307 $fpaCommand .= " -stage_id $stage_id"; 274 308 $fpaCommand .= " -multiaddlabel $label"; 275 $fpaCommand .= " -addprocessedexp"; 309 $fpaCommand .= " -dtime_addstar $dtime_addstar"; 310 if ($Nstage_extra1_list > 0) { 311 $fpaCommand .= " -stage_extra1_list $stage_extra1_list"; 312 } 276 313 $fpaCommand .= " -path_base $outroot"; 277 $fpaCommand .= " -dtime_addstar $dtime_addstar";278 314 $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; 279 315 $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; 280 316 $fpaCommand .= " -dbname $dbname" if defined $dbname; 317 281 318 print $fpaCommand; 319 282 320 # Add the result into the database 283 321 unless ($no_update) { … … 314 352 $command .= " -path_base $outroot" if defined $outroot; 315 353 $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); 316 #$command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure)354 # $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure) 317 355 $command .= " -dbname $dbname" if defined $dbname; 318 356 print $command;
Note:
See TracChangeset
for help on using the changeset viewer.
