Index: trunk/ippScripts/scripts/addstar_multi_run.pl
===================================================================
--- trunk/ippScripts/scripts/addstar_multi_run.pl	(revision 39160)
+++ trunk/ippScripts/scripts/addstar_multi_run.pl	(revision 40803)
@@ -33,4 +33,5 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
+
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -40,30 +41,31 @@
 
 my ( $label,$camera, $stage, $stage_id, $use_diff_inv, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
-     $no_op, $redirect, $save_temps);
+     $no_op, $redirect, $save_temps, $limit_number);
 GetOptions(
-    'label=s'          => \$label, #addstar label
-    'camera|c=s'        => \$camera, # Camera
-    'stage|s=s'        => \$stage, # Camera
-    'stage_id|w=s'   => \$stage_id,
-    'use_diff_inv'   => \$use_diff_inv, #use inv images for diff cmf? 
-    'dbname|d=s'        => \$dbname, # Database name
-    'outroot|w=s'       => \$outroot, # output file base name
-    'stageroot|w=s'       => \$stageroot, # stage root name.
-    'reduction=s'       => \$reduction, # Reduction class
-    'dvodb|w=s'         => \$dvodb,  # output DVO database
-    'minidvodb_name|w=s'=> \$minidvodb_name,  # miniDVO database name
-    'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group
-    'minidvodb'         => \$minidvodb,  # use minidvodb?
-    'image-only'        => \$image_only,   # Print to stdout
-    'verbose'           => \$verbose,   # Print to stdout
-    'no-update'         => \$no_update, # Update the database?
-    'no-op'             => \$no_op, # Don't do any operations?
-    'redirect-output'   => \$redirect,
-    'save-temps'        => \$save_temps, # Save temporary files?
+    'label=s'             => \$label, # addstar label
+    'stage=s'             => \$stage, # stage (camera, fullforce, etc)
+    'stage_id=s'          => \$stage_id, 
+    'camera=s'            => \$camera, # Camera
+    'outroot=s'           => \$outroot, # output file base name
+    'dvodb=s'             => \$dvodb,  # output DVO database
+    'stageroot=s'         => \$stageroot, # stage root name.
+    'limit=i'             => \$limit_number, 
+    'use_diff_inv'        => \$use_diff_inv, # use inv images for diff cmf? 
+    'dbname=s'            => \$dbname, # Database name
+    'reduction=s'         => \$reduction, # Reduction class
+    'minidvodb_name=s'    => \$minidvodb_name,  # miniDVO database name
+    'minidvodb_group=s'   => \$minidvodb_group, # miniDVO database group
+    'minidvodb'           => \$minidvodb,  # use minidvodb?
+    'image-only'          => \$image_only,   # Print to stdout
+    'verbose'             => \$verbose,   # Print to stdout
+    'no-update'           => \$no_update, # Update the database?
+    'no-op'               => \$no_op, # Don't do any operations?
+    'redirect-output'     => \$redirect,
+    'save-temps'          => \$save_temps, # Save temporary files?
     ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-          -msg => "Required options: --label --stage_id --stage  --camera --outroot --dvodb --stageroot",
+          -msg => "Required options: --label --stage --stage_id --camera --outroot --dvodb --stageroot",
           -exitval => 3,
           ) unless
@@ -111,4 +113,5 @@
 #my $temp_file = $outroot.".log.".$label.".".$stage.".".$stage_id.".list";
 my $temp_file = "/tmp/addstar.list";
+# XXX this temp_file is dangerous: multiple instances of this program will overwrite
 
 print "using $temp_file for list\n";
@@ -116,52 +119,71 @@
 open( TEMPLIST, ">$temp_file") or &my_die( "Can't open $temp_file\n", $stage_id,$stage,$label, $PS_EXIT_UNKNOWN_ERROR);
 
+my @stage_extra1 = ();
+
+# query database for list of exposures: we need to generate the input list for addstar
 {
     my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $command = "$addtool -pendingexp -stage_id ".$stage_id;
-    $command .= " -stage $stage -label $label";
-    $command .= " -dbname $dbname" if defined $dbname;
-
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
+
+    my $command = "$addtool -pendingexp";
+    $command .= " -stage_id $stage_id";
+    $command .= " -stage $stage";
+    $command .= " -label $label";
+    $command .= " -dbname $dbname"      if defined $dbname;
+    $command .= " -limit $limit_number" if defined $limit_number;
+    
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
     &my_die( "Unable to get info on addRun", $stage_id,$stage, $label, $PS_EXIT_SYS_ERROR) unless $success;
+
     if (scalar @$stdout_buf == 0 ) { 
-	#&my_die( "empty addRun", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR);
+	# if there are no entries, just exit (do not update database with a fault)
+	# &my_die( "empty addRun", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR);
 	print "empty addRun, exiting... $stage,  $stage_id, $label\n";
 	exit 0;
-    } else {
-            my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-                &my_die("Unable to parse metadata config", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
-            #this fails if there is nothing listed. I checked.
-            my $components = parse_md_list($metadata) or
-                &my_die("Unable to parse metadata list", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
-	    my $num_components = scalar @$components;
-	    print "there are $num_components\n";
-            foreach my $comp  (@$components) {
-		my  $mparsed = $comp->{stageroot};
-            
-		if (!defined($mparsed)) {
-		    &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
-		} #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
-		print "found a value for stageroot:$mparsed\n";
-		my $fpaObjects = $mparsed.'.cmf';
-		my $realFile = $ipprc->file_resolve($fpaObjects);
-		if (!defined($realFile)) {
-		    &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
-		}
-		printf TEMPLIST "$realFile = $fpaObjects\n";
-		if ($stage =~ /diff/) { # hardwired  for now$use_diff_inv) {
-		    print "finding the inv.cmf files\n";
-		    my $fpaObjectsInv = $mparsed.'.inv.cmf';
-		    my $realFileInv = $ipprc->file_resolve($fpaObjectsInv);
-		    if (!defined($realFileInv)) {
-			&my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
-		    }
-		    printf TEMPLIST "$realFileInv = $fpaObjectsInv\n";
-		}
-		
+    } 
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata config", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
+    # this fails if there is nothing listed. I checked.
+    my $components = parse_md_list($metadata) or
+	&my_die("Unable to parse metadata list", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
+    my $num_components = scalar @$components;
+    print "there are $num_components\n";
+
+    # go through the list of components to create addstar input list
+    foreach my $comp  (@$components) {
+	my $mparsed = $comp->{stageroot};
+	
+	if (!defined($mparsed)) {
+	    &my_die("Unable to parse stageroot", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
+	} #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
+	print "found a value for stageroot:$mparsed\n";
+
+	my $fpaObjects = $mparsed . '.cmf';
+	my $realFile = $ipprc->file_resolve($fpaObjects);
+	if (!defined($realFile)) {
+	    &my_die("unable to resolve real file from $mparsed",$stage_id, $stage, $label, 7);
+	}
+
+	printf TEMPLIST "$realFile = $fpaObjects\n";
+	if ($stage =~ /diff/) { # hardwired for now: $use_diff_inv) {
+	    print "finding the inv.cmf files\n";
+	    my $fpaObjectsInv = $mparsed . '.inv.cmf';
+	    my $realFileInv = $ipprc->file_resolve($fpaObjectsInv);
+	    if (!defined($realFileInv)) {
+		&my_die("unable to resolve real file from $mparsed", $stage_id, $stage, $label, 8);
 	    }
-
-    }
-}
+	    printf TEMPLIST "$realFileInv = $fpaObjectsInv\n";
+	}
+
+	# save the stage_extra1 values so we can update the database correctly at the end
+	my $stage_extra1_value = $comp->{stage_extra1};
+	if (!defined($stage_extra1_value)) {
+	    &my_die("unable to find stage_extra1 value for $mparsed", $stage_id, $stage, $label, 9);
+	}
+	push @stage_extra1, $stage_extra1_value;
+    }
+}
+
 close(TEMPLIST);
 print "saved $temp_file here\n";
@@ -173,4 +195,5 @@
     $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $stage_id,$stage,$label, $PS_EXIT_CONFIG_ERROR);
 }
+
 my $dtime_addstar = 0;
 if (defined $dvodbReal) {
@@ -200,84 +223,99 @@
 	    &my_die("Unable to parse minidvodb_name", $stage_id,$stage,$label, $PS_EXIT_PROG_ERROR);
 	}
-    }
-    else {
+    } else {
 	$minidvodb_path = $dvodbReal;
     }
     
     unless ($no_op) {
-            print $dvodbReal;
-	    
-	    ## addstar can either save the full set of detections, or just
-	    ## the image metadata, in the dvodb.  this is set in the
-	    ## database table addRun
-
-	    # addstar requires the user to have a valid .ptolemyrc which
-	    # in turn points at ippconfig/dvo.site
-
-	    # get the names for the camera and the real input file
-	    my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
-	   
-#	    my $realFile = $ipprc->file_resolve($fpaObjects) or  &my_die("Unable to resolve $fpaObjects", $stage_id,$stage,$label, $PS_EXIT_SYS_ERROR);
-
-	    # most cameras use CHIP_HEADER.  other exceptions:
-	    # gpc1 @ camera : PHU_HEADER
-	    # gpc1 @ staticsky : NOMINAL (is uncalibrated)
-
-	    # temporary hard-wired info
-	    my $zeroPointOption = "CHIP_HEADER";
-	    if (($camdir =~ /gpc1/) && ($stage =~ /staticsky/)) {
-		$zeroPointOption = "NOMINAL";
-	    }		
-	    if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) {
-		$zeroPointOption = "PHU_HEADER";
-	    }		
-
-	    # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
-	    my $command  = "$addstar -update"; # XXX optionally set -update?
-	    $command .= " -D CAMERA $camdir";
-	    $command .= " -D CATDIR $minidvodb_path";
-	    $command .= " -D ZERO_POINT_OPTION $zeroPointOption";
-	    $command .= " -list $temp_file";
-	    $command .= " -image" if $image_only;
-	    if ($stage =~ /staticsky/) {
-		$command .= " -accept-astrom -quick-airmass";
-	    }  #careful here - this matches staticsky and staticsky_multi
-	    if ($stage =~ /skycal/) {
-		$command .= " -quick-airmass";
-	    }  #careful here - this matches staticsky and staticsky_multi
-	    if ($stage =~ /diff/) {
-		$command .= " -accept-astrom";
-		if ($use_diff_inv) {
-		    $command .= " -diff-inv";
-		}
+	print $dvodbReal;
+	
+	## addstar can either save the full set of detections, or just
+	## the image metadata, in the dvodb.  this is set in the
+	## database table addRun
+
+	# addstar requires the user to have a valid .ptolemyrc which
+	# in turn points at ippconfig/dvo.site
+
+	# get the names for the camera and the real input file
+	my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
+	
+	# most cameras use CHIP_HEADER.  other exceptions:
+	# gpc1 @ camera : PHU_HEADER
+	# gpc1 @ staticsky : NOMINAL (is uncalibrated)
+
+	# temporary hard-wired info
+	my $zeroPointOption = "CHIP_HEADER";
+	if (($camdir =~ /gpc1/) && ($stage =~ /staticsky/)) {
+	    $zeroPointOption = "NOMINAL";
+	}		
+	if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) {
+	    $zeroPointOption = "PHU_HEADER";
+	}		
+
+	# require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
+	my $command  = "$addstar -update"; # XXX optionally set -update?
+	$command .= " -D CAMERA $camdir";
+	$command .= " -D CATDIR $minidvodb_path";
+	$command .= " -D ZERO_POINT_OPTION $zeroPointOption";
+	$command .= " -list $temp_file";
+	$command .= " -image" if $image_only;
+	if ($stage =~ /staticsky/) {
+	    $command .= " -accept-astrom -quick-airmass";
+	}  #careful here - this matches staticsky and staticsky_multi
+	if ($stage =~ /skycal/) {
+	    $command .= " -quick-airmass";
+	}  #careful here - this matches staticsky and staticsky_multi
+	if ($stage =~ /diff/) {
+	    $command .= " -accept-astrom";
+	    if ($use_diff_inv) {
+		$command .= " -diff-inv";
 	    }
-	    
-	    if ($stage =~ /fullforce/) {
-		$command .= " -accept-astrom -xrad"; 
-	    }
-
-	    my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
-
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform addstar: $error_code", $stage_id,$stage,$label, $error_code);
-	    }
-	    $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
-    }
-}
-
-
-my $fpaCommand = "$addtool -multiadd -stage_id $stage_id";
+	}
+	
+	if ($stage =~ /fullforce/) {
+	    $command .= " -accept-astrom -xrad"; 
+	}
+
+	my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
+
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform addstar: $error_code", $stage_id,$stage,$label, $error_code);
+	}
+	$dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
+    }
+}
+
+my $stage_extra1_list = "";
+my $Nstage_extra1_list = @stage_extra1;
+
+for (my $i = 0; $i < $Nstage_extra1_list; $i ++) {
+    $stage_extra1_list .= "$stage_extra1[$i]";
+    if ($i < $Nstage_extra1_list - 1) {
+	$stage_extra1_list .= ",";
+    }
+    print "STAGE EXTRA1 : $stage_extra1[$i]\n";
+}
+
+# at the end, we need to update the database with the entries we actually processed
+# these are identified by stage_extra1, listed in the array @stage_extra1
+my $fpaCommand = "$addtool -addprocessedexp";
+$fpaCommand .= " -multiadd";
 $fpaCommand .= " -stage $stage";
+$fpaCommand .= " -stage_id $stage_id";
 $fpaCommand .= " -multiaddlabel $label";
-$fpaCommand .= " -addprocessedexp";
+$fpaCommand .= " -dtime_addstar $dtime_addstar";
+if ($Nstage_extra1_list > 0) {
+    $fpaCommand .= " -stage_extra1_list $stage_extra1_list";
+}
 $fpaCommand .= " -path_base $outroot";
-$fpaCommand .= " -dtime_addstar $dtime_addstar";
 $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
 $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
 $fpaCommand .= " -dbname $dbname" if defined $dbname;
+
 print $fpaCommand;
+
 # Add the result into the database
 unless ($no_update) {
@@ -314,5 +352,5 @@
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
-   #     $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure)
+	# $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure)
         $command .= " -dbname $dbname" if defined $dbname;
 	print $command;