Index: /branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in (revision 32851) @@ -60,4 +60,5 @@ $(DESTWWW)/czartool_getplot.php \ $(DESTWWW)/czartool_exposures.php \ +$(DESTWWW)/czartool_ipptopsps.php \ $(DESTWWW)/histogram.php \ $(DESTWWW)/show_and_delete_image.php \ Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl (revision 32851) @@ -27,7 +27,7 @@ my $nebulous = undef; my $savingToFile = undef; -my $analysis = undef; my $exposureId = undef; my $log = undef; +my $psps = undef; GetOptions ( @@ -45,8 +45,8 @@ "nebulous|n" => \$nebulous, "rate|r" => \$rate, - "analysis|a" => \$analysis, "timeseries|t" => \$timeSeries, "verbose|v" => \$verbose, "log|g" => \$log, + "psps|u" => \$psps, ); @@ -58,7 +58,4 @@ print "* UNKNKOWN: option @ARGV\n"; } -if ($analysis && !$stage) { - $quit = 1; - print "* REQUIRED: choose a stage for analsis -s (default=none)\n";} if (!$histogram) { print "* OPTIONAL: plot histogram -h (default=off)\n";} @@ -71,7 +68,4 @@ if (!$rate) { print "* OPTIONAL: plot histogram of rate -r (default=off)\n";} -if (!$analysis) { - $analysis = 0; - print "* OPTIONAL: include analysis -a (default=$analysis)\n";} if (!$log) { $log = 0; @@ -97,4 +91,6 @@ print "* OPTIONAL: choose an output location -o (default=none)\n"; } +if (!$psps) { + print "* OPTIONAL: plot ippToPsps stages -u (default=off)\n";} print "*\n*******************************************************************************\n"; @@ -104,5 +100,5 @@ # default values -if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;} +if (!$rate && !$magicMask && !$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1;} if (!$verbose) {$verbose = 0;} if (!$save_temps) {$save_temps = 0;} @@ -123,4 +119,6 @@ $plotter = czartool::Plotter->new_display($config, $save_temps); } + +if ($psps) {$plotter->setIppToPspsMode();} # if a single day has been chosen @@ -177,7 +175,2 @@ } -if($analysis) { - - my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end); - if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();} -} Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl (revision 32851) @@ -37,6 +37,28 @@ -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); - +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); timePoll($period); @@ -244,5 +266,29 @@ print "* Generating plots\n"; + # get ippToPsps labels + my $ippToPspsLabels = undef; + $czarDb->getCurrentIppToPspsLabels($begin, $end, \$ippToPspsLabels); + $plotter->setIppToPspsMode(); + + # create ippToPsps labels + foreach $stage (@ippToPspsStages) { + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + chomp($label); + $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1); + } + } + + # create plots for each label for all stages for ippToPsps + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); + } + + # create plots for each label for each stage + $plotter->setStandardMode(); foreach $stage (@stages) { foreach $row ( @{$rows} ) { Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm (revision 32851) @@ -8,5 +8,28 @@ use File::Temp; -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); # TODO put elsewhere +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); # TODO put elsewhere + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); use base 'czartool::MySQLDb'; @@ -33,5 +56,28 @@ ########################################################################### # -# Gets current_labels table +# Gets current ippToPsps labels +# +########################################################################### +sub getCurrentIppToPspsLabels { + my ($self, $begin, $end, $labels) = @_; + + my $query = $self->{_db}->prepare(<= '$begin' + AND timestamp <= '$end' +SQL + + if (!$query->execute) {return 0;} + + ${$labels} = $query->fetchall_arrayref(); + + return 1; +} + +########################################################################### +# +# Gets current labels from the current_labels table # ########################################################################### @@ -672,4 +718,6 @@ # times $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); + if ($timeSep == 0) {$timeSep = 1;}; # HACK to prevent divide-by-zero + $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp); $formattedMiddleTimestamp = $self->getFormattedDate($middleTime); @@ -681,5 +729,5 @@ # added, and store value to subtract from linear value below. If > 1000 # processed per hour, then it must be wrong - if (($thisProcessed - $lastProcessed)/(($timeSep/3600)+1) > 1000) { + if (($thisProcessed - $lastProcessed)/($timeSep/3600) > 1000) { $positiveJump = $thisProcessed - $lastProcessed; } @@ -701,7 +749,7 @@ # calculate first derivative in units of images per hour - $derivPending = $diffPending/($timeSep/3600 +1); - $derivFaults = $diffFaults/($timeSep/3600 +1); - $derivProcessed = $diffProcessed/($timeSep/3600 +1); + $derivPending = $diffPending/($timeSep/3600); + $derivFaults = $diffFaults/($timeSep/3600); + $derivProcessed = $diffProcessed/($timeSep/3600); } @@ -1107,5 +1155,5 @@ my $stage = undef; $totalDeleted = 0; - foreach $stage (@stages) { + foreach $stage (@allStages) { if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;} @@ -1213,5 +1261,5 @@ my $stage = undef; - foreach $stage (@stages) { + foreach $stage (@allStages) { $self->optimizeTable($stage); Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm (revision 32851) @@ -10,4 +10,5 @@ "addstarlap", "cleanup", + "deepstack", "detrend", "distribution", Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm (revision 32851) @@ -10,5 +10,27 @@ use czartool::Config; -my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); +my @allStages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @allIppToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + ########################################################################### @@ -75,4 +97,26 @@ ########################################################################### # +# Sets plotter to use standard IPP mode, i.e. chip, cam etc +# +########################################################################### +sub setStandardMode { + my ($self) = @_; + + $self->{allStages} = \@allStages; +} + +########################################################################### +# +# Sets plotter to use ippToPsps mode, i.e. processed, loaded_to_datastore etc +# +########################################################################### +sub setIppToPspsMode { + my ($self) = @_; + + $self->{allStages} = \@allIppToPspsStages; +} + +########################################################################### +# # Some initialisation code common to all constructors # @@ -83,4 +127,5 @@ $self->{_gpc1Db} = $self->{_config}->getGpc1Instance(); $self->{_czarDb} = $self->{_config}->getCzarDbInstance(); + $self->setStandardMode(); } @@ -140,5 +185,5 @@ my $stages = undef; if (!$selectedStage) { - $stages = \@allStages; + $stages = \@{$self->{allStages}}; $selectedStage = 'all_stages'; } @@ -203,10 +248,10 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures processed per $interval\";" . + "set ylabel \"Items processed per $interval\";" . "set boxwidth 0.75;" . "plot "; my $first = 1; - foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { @@ -226,5 +271,5 @@ ########################################################################### # -# Plots a time series for all stages for this label +# Plots a time series for all or one stage for this label # ########################################################################### @@ -238,5 +283,5 @@ my $stages = undef; - if (!$selectedStage) {$stages = \@allStages;} + if (!$selectedStage) {$stages = \@{$self->{allStages}};} else {$stages = ["$selectedStage"]}; @@ -297,5 +342,5 @@ $timeDiff, "", - "Exposures"); + "Items"); } @@ -314,5 +359,5 @@ $timeDiff, "", - "Log(Exposures)"); + "Log(Items)"); } @@ -331,5 +376,5 @@ $timeDiff, "", - "Exposures per hour"); + "Items per hour"); } } @@ -351,5 +396,5 @@ my $stage = undef; my $pendingMinusFaults = undef; - foreach $stage (@allStages) { + foreach $stage (@{$self->{allStages}}) { if (!$self->{_czarDb}->countProcessedPendingAndFaults( @@ -381,5 +426,5 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures\";" . + "set ylabel \"Items\";" . "set boxwidth 0.75;" . "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . @@ -520,5 +565,6 @@ my $firstIn = 1; # loop through stages array so that they are ordered properly (not maintained in hash) - foreach my $stage (@allStages) { + #foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { if(!$gnuplotFiles->{$stage}) {next;} @@ -620,4 +666,5 @@ "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . "set style fill solid 1.00 border -1;" . + "set key center top;" . "set style histogram rowstacked;" . "set style data histograms;" . Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 32851) @@ -0,0 +1,405 @@ + 0) $selectedLabel = $labels[0]; + else $selectedLabel = "no labels"; +} +if ($selectedStage == "") { $selectedStage = "all_stages"; } +if ($plotType == "") { $plotType = "linear"; } + +// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later +if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode); + +// tell selected server to stop or run +if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer, $serverCmd); + +// tell all servers to stop or run +if ($allServerCmd) { + + foreach ($servers as &$server) serverStopRun($czardb, $server, $allServerCmd); +} + + +$debug = 0; +$table = "\n"; + + +if ($debug) { + echo "prog returned $status, and output:
"; + for ($i = 0; $i < count($distLabels); $i++) { + echo "$distLabels[$i]
"; + } +} + +$isUpdate = ($selectedMode == "update"); + +// set up the form +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +echo "
"; + +echo ""; +echo ""; +echo ""; + +echo ""; +echo ""; +echo "\n"; +echo ""; +echo ""; + +echo "
"; +echo "

Czartool

"; +echo "

IPP to PSPS Monitoring

"; +echo "
"; + # time series plot +echo "
"; + echo "
"; + # rate time series plot +echo "
"; + echo "
"; +echo "		"; +echo ""; + +#echo "
"; + +// status table at top +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; +echo "
"; +echo $table; +createTableTitle("Status and links", 3); +echo "
Time of last update$lastUpdateTime
Documentationhere
\n"; + +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; + + + +echo "
"; + + createLabelsTable($pass, + $proj, + $czardb, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + "new", + $selectedLabel, + $selectedStage, + $plotType); + +echo "
"; + +echo ""; + +menu_end(); + +########################################################################### +# +# Gets labels in use over the last 24 hours +# +########################################################################### +function getLabels($db) { + + # order by descending priority as set in gpc1 database + $sql = "SELECT DISTINCT label FROM processed WHERE timestamp >= now() - INTERVAL 1 DAY ORDER BY label"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + while ($qry->fetchInto($row)) { + + $labels[] = $row[0]; + } + + return $labels; +} + +########################################################################### +# +# Creates table for all labels showing all stages for given 'state' +# +########################################################################### +function createLabelsTable( + $pass, + $proj, + $db, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + $selectedState, + $selectedLabel, $selectedStage, $plotType) { + + $isUpdate = ($selectedMode == "update"); + global $headerColor; + global $columnHeaderColor; + + // setup table + global $table; + echo $table; + createTableTitle("Label status", 13); + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=".$selectedMode + ."&label=".$selectedLabel + ."&stage=all_stages" + ."&plottype=".$plotType; + createFormattedTableCell(($selectedStage != "all_stages"), $link, "All stages", 0, "$columnHeaderColor"); + + #echo "Label (in order of priority)"; + foreach ($stages as &$stage) { + + if ($stage == $selectedStage) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$stage + ."&plottype=".$plotType; + + createFormattedTableCell($showLink, $link, $stage, 0, "$columnHeaderColor"); +# createTableColumnHeader("Dates"); + } + + // setup default link + $defaultlink = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + + $searchState = "new"; + if ($selectedMode == "update") $searchState = "update"; + + // write rows + foreach ($labels as &$thisLabel) { + + + $distributing = false; + $publishing = false; + + // show distribution/publishing status + echo "\n"; + + // create link to label summary page for each label + if ($thisLabel == $selectedLabel) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$thisLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + createFormattedTableCell($showLink, $link, $thisLabel, 0, null); + + $str = ""; + $anyFaults = false; + + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "processed", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_ODM", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merge_worthy", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_dxlayer", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merged", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_local", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + echo "\n"; + } + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + if ($selectedLabel == "all") $showLink = 0; + else $showLink = 1; + $link = "czartool_ipptopsps.php?pass=".$pass."&proj=".$proj."&mode=".$selectedMode."&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + echo "\n"; +} + + +########################################################################### +# +# Creates a table title +# +########################################################################### +function createTableTitle($str, $colSpan) { + + global $headerColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates a table header cell +# +########################################################################### +function createTableColumnHeader($str) { + + global $columnHeaderColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates table cell with a red background if an error +# +########################################################################### +function createFormattedTableCell($showLink, $link, $str, $error, $color) { + + $fontColor = $error ? "black" : "blue"; + + if ($error) echo ""; + else if ($color) echo ""; + else echo ""; + if ($showLink) echo "$str"; + else if ($str) echo "$str"; + else echo " "; +} + + +########################################################################### +# +# Returns state and fault-count (if new) as a string +# +########################################################################### +function getStateAndFaults($db, $label, $state, $stage, &$str, &$anyFaults) { + + $anyFaults = false; + $sql = "SELECT pending, faults FROM $stage WHERE label LIKE '$label' ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + $pending = $row[0]; + $faults = $row[1]; + + if ($pending == 0) + $str = ""; + else + $str = "$pending"; + + if ($state == "new") { + + if ($faults > 0) { + + $str = $str."(".$faults.")"; + $anyFaults = true; + } + } +} + +########################################################################### +# +# Returns time of last czarDb update +# +########################################################################### +function getLastUpdateTime($db) { + + $anyFaults = false; + $sql = "SELECT timestamp FROM processed ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + return $row[0]; +} + + +?> + Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 32851) @@ -27,6 +27,17 @@ menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); -$states=array("full","new","drop","wait"); -$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist", "pub"); +$stages=array( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + $servers=array( "addstar", @@ -150,5 +161,5 @@ echo "
"; echo $table; -#"\n"; + createTableTitle("Status and links", 3); echo "\n"; @@ -180,5 +191,4 @@ $pubLabels, $stages, - $states, "new", $selectedLabel, @@ -349,5 +359,4 @@ $pubLabels, $stages, - $states, $selectedState, $selectedLabel, $selectedStage, $plotType) { Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat (revision 32851) @@ -21,5 +21,6 @@ menutop | menutop | link | simple plot - chip (to remove?) | simplePlot.php menutop | menutop | link | simple plot - cam (to remove?) | simplePlotcam.php -menutop | menutop | link | czartool | czartool_labels.php +menutop | menutop | link | czartool - standard | czartool_labels.php +menutop | menutop | link | czartool - ipptopsps | czartool_ipptopsps.php menutop | menutop | link | mask stats | maskStats.php Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl (revision 32851) @@ -37,5 +37,5 @@ } my $minidvodb_path; -my ( $add_id, $camera, $stage, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -43,5 +43,5 @@ 'camera|c=s' => \$camera, # Camera 'stage|s=s' => \$stage, # Camera - + 'multi_num|w=s' => \$multi_num, # the number for a staticskymulti (for finding cmf) 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name @@ -114,4 +114,13 @@ } + +if ($stage =~/staticsky_multi/) { + $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above + &my_die( "can't find the filter_num for staticsky_multi, giving up.", $add_id, $PS_EXIT_SYS_ERROR) unless (defined $multi_num); + my $nice_num = sprintf ("%03d", $multi_num); + $fpaObjects =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc + +} + my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); @@ -190,7 +199,7 @@ $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference $command .= " -image" if $image_only; - if ($stage = ~/staticsky/) { + if ($stage =~ /staticsky/) { $command .= " -accept-astrom "; - } + } #careful here - this matches staticsky and staticsky_multi my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_filename.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_filename.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_filename.pl (revision 32851) @@ -10,4 +10,10 @@ #print "$ENV{'PATH'}\n"; #print "$ENV{'PERL5LIB'}\n"; + +# CZW: This is a horrible hack, but I don't want to have to debug all of ippMonitor to figure out why it's not working. +unless (defined($ENV{'NEB_SERVER'})) { + $ENV{'NEB_SERVER'} = 'http://nebserver.ipp.ifa.hawaii.edu:80/nebulous'; +} + use PS::IPP::Config; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl (revision 32851) @@ -254,4 +254,5 @@ if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) { $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); + my $skycell_id = $skycell->{skycell_id}; $diff_skycells{$skycell_id} = $skycell_uri; } else { Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/register_exp.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/register_exp.pl (revision 32851) @@ -121,5 +121,6 @@ my ($data_group,$dist_group,$chip_workdir,$reduction); ($label,$data_group,$dist_group,$end_stage, - $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"), + $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-exp_name"), + &value_for_flag($cmdflags,"-dateobs"), &value_for_flag($cmdflags,"-exp_type"), &value_for_flag($cmdflags,"-obs_mode"), @@ -170,4 +171,5 @@ ### Pau. sub advance_decisions { + my $exp_name = shift; my $dateobs = shift; my $exptype = shift; @@ -178,4 +180,5 @@ # The strings come pre-quoted. + $exp_name =~ s/\'//g; $dateobs =~ s/\'//g; $exptype =~ s/\'//g; @@ -188,4 +191,9 @@ my $target; + if ($exp_name =~ /^c/) { + print "Skipping because this is a camera commanded exposure: $exp_name\n"; + return(undef,undef,undef,"reg",undef,undef,undef); + } + if ($exptype ne 'OBJECT') { print "Skipping because exptype doesn't claim to be OBJECT: $exptype\n"; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl (revision 32851) @@ -32,4 +32,5 @@ my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -116,4 +117,5 @@ run(command => $command, verbose => $verbose); unless ($success) { + check_input_file($uri, $vp_id, $class_id, 'nocell'); $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform $command: $error_code", $vp_id, $error_code); @@ -244,4 +246,34 @@ exit 0; +sub check_input_file { + my $uri = shift; + my $vp_id = shift; + my $class_id = shift; + my $cell_id = shift; + my $resolved = $ipprc->file_resolve($uri); + + my $tryrepair = 0; + if (!-e $resolved) { + printf STDERR "instance $resolved for $uri does not exist\n"; + $tryrepair = 1; + } elsif (-s $resolved == 0) { + printf STDERR "instance $resolved for $uri is empty\n"; + $tryrepair = 1; + } + if ($tryrepair) { + my $scheme = file_scheme($uri); + if ($scheme and ($scheme = 'neb')) { + my $command = "$nebrepair $uri"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform nebrepair"; + &my_die("$err_message: $error_code", $vp_id, $error_code); + } + } + } +} + sub my_die { Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/whichimage =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/whichimage (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/whichimage (revision 32851) @@ -60,5 +60,5 @@ my $sign = 1; - if ($h < 0) { + if ($h =~ /-/) { $sign = -1; $h = - $h; @@ -76,5 +76,5 @@ my $sign = 1; - if ($d < 0) { + if ($d =~ /-/) { $sign = -1; $d = - $d; Index: /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro (revision 32851) @@ -26,4 +26,7 @@ active true end + task addstar.exp.load.staticsky_multi + active true + end task addstar.exp.run active true @@ -39,4 +42,7 @@ end task addstar.exp.load.staticsky + active false + end + task addstar.exp.load.staticsky_multi active false end @@ -175,4 +181,5 @@ end end + task addstar.exp.load.staticsky host local @@ -229,4 +236,61 @@ end end + +task addstar.exp.load.staticsky_multi + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + # if ($LABEL:n == 0) break + $run = addtool -pendingexp -stage staticsky_multi + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + # run the addstar script on pending exposures task addstar.exp.run @@ -252,4 +316,5 @@ book getword addPendingExp $pageName exp_tag -var EXP_TAG book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1 book getword addPendingExp $pageName stageroot -var STAGEROOT book getword addPendingExp $pageName stage -var STAGE @@ -285,4 +350,7 @@ sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID end + if ("$STAGE" == "staticsky_multi") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end if ("$STAGE" == "stack") sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID @@ -296,4 +364,7 @@ if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION + end + if ("$STAGE" == "staticsky_multi") + $run = $run --multi_num $STAGE_EXTRA1 end if ("$IMAGE_ONLY" == "T") Index: /branches/eam_branches/ipp-20111122/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/survey.pro (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTasks/survey.pro (revision 32851) @@ -33,5 +33,5 @@ book create SURVEY_WARP_BG book create SURVEY_PUBLISH - book create SURVEY_STATICSKYSINGLE + book create SURVEY_STATICSKYSINGLE $haveSurveyBooks = TRUE end @@ -48,5 +48,4 @@ $SURVEY_PUBLISH_DB = 0 $SURVEY_STATICSKYSINGLE_DB = 0 - $SURVEY_EXEC = 120 $SURVEY_POLL = 10 @@ -839,4 +838,9 @@ $run = $run -uncensored end + if ("$stage" == "staticsky_multi") + #only queue uncensored staticsky (multi filter) + $run = $run -uncensored + end + if ("$stage" == "stack") #only queue uncensored stacks Index: /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am (revision 32851) @@ -10,4 +10,6 @@ addtool_find_sky_id_dvo.sql \ addtool_find_sky_id.sql \ + addtool_find_sky_id_multi_dvo.sql \ + addtool_find_sky_id_multi.sql \ addtool_find_stack_id_dvo.sql \ addtool_find_stack_id.sql \ @@ -16,8 +18,10 @@ addtool_find_pendingexp_stack.sql \ addtool_find_pendingexp_staticsky.sql \ + addtool_find_pendingexp_staticsky_multi.sql \ addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp_cam.sql \ addtool_find_processedexp_stack.sql \ addtool_find_processedexp_staticsky.sql \ + addtool_find_processedexp_staticsky_multi.sql \ addtool_find_minidvodbprocessed.sql \ addtool_find_minidvodbrun.sql \ @@ -27,4 +31,5 @@ addtool_queue_stack_id.sql \ addtool_queue_sky_id.sql \ + addtool_queue_sky_id_multi.sql \ addtool_queue_minidvodbrun.sql \ addtool_revertminidvodbprocessed.sql \ @@ -32,4 +37,5 @@ addtool_revertprocessedexp_stack.sql \ addtool_revertprocessedexp_staticsky.sql \ + addtool_revertprocessedexp_staticsky_multi.sql \ bgtool_advancechip.sql \ bgtool_advancewarp.sql \ @@ -308,4 +314,5 @@ pstamptool_getdependent.sql \ pstamptool_listjob.sql \ + pstamptool_listfile.sql \ pstamptool_pendingcleanup.sql \ pstamptool_pendingdependent.sql \ Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 32851) @@ -0,0 +1,32 @@ +SELECT + addRun.*, + staticskyResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN staticskyRun + ON sky_id = stage_id +JOIN staticskyResult + USING (sky_id) +JOIN staticskyInput + USING (sky_id) +JOIN stackRun + USING (stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + staticskyRun.state = 'full' + AND stage = 'staticsky_multi' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_processedexp_staticsky_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_processedexp_staticsky_multi.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_processedexp_staticsky_multi.sql (revision 32851) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql (revision 32851) @@ -0,0 +1,24 @@ +SELECT + distinct(sky_id) as stage_id, + staticskyRun.*, + staticskyResult.num_inputs +FROM + staticskyResult +join staticskyRun using (sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT sky_id AS added_sky_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN staticskyRun on sky_id = stage_id + ) as foo + ON sky_id = added_sky_id + AND stage = 'staticsky_multi' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 32851) @@ -0,0 +1,12 @@ +SELECT distinct(sky_id) as stage_id, staticskyRun.*, staticskyResult.num_inputs FROM staticskyResult +JOIN staticskyRun USING(sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 + AND sky_id NOT IN (SELECT sky_id + FROM addRun + JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id + JOIN staticskyRun USING(sky_id) + WHERE addRun.stage = 'staticsky_multi' AND %s + ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_cam_id.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_cam_id.sql (revision 32851) @@ -4,4 +4,5 @@ 'cam', -- stage cam_id, -- stage_id + %d, -- stage_extra1 '%s', -- state '%s', -- workdir Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id.sql (revision 32851) @@ -4,4 +4,5 @@ 'staticsky', -- stage sky_id, -- stage_id + %d, -- stage_extra1 '%s', -- state '%s', -- workdir Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql (revision 32851) @@ -0,0 +1,22 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'staticsky_multi', -- stage + sky_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM staticskyRun + WHERE + staticskyRun.state = 'full' + AND staticskyRun.sky_id = %lld Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_stack_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_stack_id.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_stack_id.sql (revision 32851) @@ -4,4 +4,5 @@ 'stack', -- stage stack_id, -- stage_id + %d, --stage_extra1 '%s', -- state '%s', -- workdir Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 32851) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'staticsky_multi' Index: /branches/eam_branches/ipp-20111122/ippTools/share/disttool_listinterests.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/disttool_listinterests.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/disttool_listinterests.sql (revision 32851) @@ -9,4 +9,5 @@ stage, clean, + target_id, distTarget.state as target_state FROM rcInterest Index: /branches/eam_branches/ipp-20111122/ippTools/share/magicdstool_getskycells.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/magicdstool_getskycells.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/magicdstool_getskycells.sql (revision 32851) @@ -5,5 +5,6 @@ diffSkyfile.skycell_id, diffSkyfile.path_base, - diffSkyfile.data_state + diffSkyfile.data_state, + diffSkyfile.quality FROM magicDSRun JOIN magicRun USING(magic_id) @@ -37,5 +38,6 @@ diffSkyfile.skycell_id, diffSkyfile.path_base, - diffSkyfile.data_state + diffSkyfile.data_state, + diffSkyfile.quality FROM magicDSRun JOIN magicRun USING(magic_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_listfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_listfile.sql (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_listfile.sql (revision 32851) @@ -0,0 +1,5 @@ +SELECT + pstampFile.*, + pstampJob.req_id +FROM pstampFile + JOIN pstampJob using(job_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql (revision 32851) @@ -1479,4 +1479,12 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE pstampFile ( + file_id BIGINT AUTO_INCREMENT, + job_id BIGINT NOT NULL, + path VARCHAR(255), + PRIMARY KEY(file_id), + KEY(job_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE pstampWebRequest ( num BIGINT AUTO_INCREMENT, Index: /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 32851) @@ -143,5 +143,11 @@ pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label } - + if (strcmp(stage, "staticsky_multi")== 0) { + + pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); + //no filter here + } if (!psListLength(where->list)) { psFree(where); @@ -237,5 +243,19 @@ } - + if (strcmp(stage,"staticsky_multi") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); + } + } if (!bare_query) { @@ -292,5 +312,7 @@ psStringAppend(&query, " GROUP BY stack_id"); } - + if (strcmp(stage,"staticsky_multi") == 0) { + psStringAppend(&query, " GROUP BY sky_id"); //some reason it needs this + } @@ -405,4 +427,43 @@ } } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + if (num_inputs < 0) { + psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs"); + return false; + } + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb) { //there's no staticsky.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + + psFree(row); + } + } + + + // start a transaction so we don't end up with an exp without any associted @@ -438,4 +499,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -479,4 +541,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -520,4 +583,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -544,4 +608,57 @@ } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + stage_id = row->sky_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + for (int filter_id = 0; filter_id < num_inputs; filter_id++) { + if (!pxaddQueueByCamID(config, + stage, + stage_id, + filter_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + } + psFree(row); + } + } + + + + psFree(output); @@ -585,4 +702,7 @@ } if (strcmp(stage, "staticsky")==0) { + query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + } + if (strcmp(stage, "staticsky_multi")==0) { query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); } @@ -626,5 +746,8 @@ query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); } - + if (strcmp(stage, "staticsky_multi")==0) { + query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql"); + } + if (!query) { @@ -648,5 +771,8 @@ psStringAppend(&query, " GROUP BY %s", "sky_id"); } - + if (strcmp(stage, "staticsky_multi") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); + } // treat limit == 0 as "no limit" if (limit) { @@ -838,4 +964,8 @@ query = pxDataGet("addtool_find_processedexp_staticsky.sql"); } + if (strcmp (stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); + } + if (!query) { @@ -948,4 +1078,10 @@ query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); } + if (strcmp(stage, "staticsky_multi") == 0) { + query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql"); + } + + + if (!query) { // rollback Index: /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 32851) @@ -419,6 +419,10 @@ } - if (!strcmp(stage, "sky") && single) { - psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1"); + if (!strcmp(stage, "sky")) { + if (single) { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1"); + } else { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) > 1"); + } } Index: /branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c (revision 32851) @@ -666,4 +666,5 @@ "cam", row->cam_id, + 0, row->workdir, row->reduction, Index: /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 32851) @@ -56,4 +56,7 @@ static bool revertdependentMode(pxConfig *config); static bool getwebrequestnumMode(pxConfig *config); +static bool addfileMode(pxConfig *config); +static bool listfileMode(pxConfig *config); +static bool deletefileMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -103,4 +106,7 @@ MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode); + MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode); + MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode); + MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode); default: psAbort("invalid option (this should not happen)"); @@ -1476,2 +1482,122 @@ return true; } + +static bool addfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_STR(path, config->args, "-path", true, false); + + if (!pstampFileInsert(config->dbh, + 0, // file_id + job_id, + path + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 file_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", file_id); + + return true; +} + +static bool listfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); + exit (1); + } + + psString query = pxDataGet("pstamptool_listfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool deletefileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + + psString query = NULL; + psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected); + + return true; +} Index: /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h (revision 32851) @@ -50,4 +50,7 @@ PSTAMPTOOL_MODE_REVERTDEPENDENT, PSTAMPTOOL_MODE_GETWEBREQUESTNUM, + PSTAMPTOOL_MODE_ADDFILE, + PSTAMPTOOL_MODE_LISTFILE, + PSTAMPTOOL_MODE_DELETEFILE, } pstamptoolMode; Index: /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 32851) @@ -274,4 +274,21 @@ psMetadata *modes = psMetadataAlloc(); + // -addfile + psMetadata *addfileArgs = psMetadataAlloc(); + psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id", 0, "define job ID for file (required)", 0); + psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path", 0, "define path for file (required)", NULL); + + // -listfile + psMetadata *listfileArgs = psMetadataAlloc(); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id", 0, "select by file ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -deletefile + psMetadata *deletefileArgs = psMetadataAlloc(); + psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID (required)", 0); + PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); @@ -302,4 +319,7 @@ PXOPT_ADD_MODE("-project", "", PSTAMPTOOL_MODE_PROJECT, projectArgs); PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumArgs); + PXOPT_ADD_MODE("-addfile", "", PSTAMPTOOL_MODE_ADDFILE, addfileArgs); + PXOPT_ADD_MODE("-listfile", "", PSTAMPTOOL_MODE_LISTFILE, listfileArgs); + PXOPT_ADD_MODE("-deletefile", "", PSTAMPTOOL_MODE_DELETEFILE, deletefileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 32851) @@ -131,4 +131,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, @@ -159,4 +160,8 @@ psMemSetPersistent(query, true); } + if (strcmp(stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_queue_sky_id_multi.sql"); + psMemSetPersistent(query, true); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -170,4 +175,5 @@ // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, + stage_extra1 ? stage_extra1 : 0, "new", // state workdir ? workdir : "NULL", Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h (revision 32851) @@ -33,4 +33,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, Index: /branches/eam_branches/ipp-20111122/ippTools/src/vptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/vptool.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/vptool.c (revision 32851) @@ -178,13 +178,4 @@ PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); - psMetadata *where = psMetadataAlloc(); - pxchipGetSearchArgs (config, where); // rawExp only - pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label - - if (!psListLength(where->list)) { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); @@ -195,7 +186,17 @@ // default - PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "="); // define using newExp label + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } // find the exp_id of all the exposures that we want to queue up. @@ -213,6 +214,6 @@ psFree(where); - if (unique) { - psStringAppend(&query, "AND vp_id IS NULL"); + if (!rerun) { + psStringAppend(&query, "\n AND vp_id IS NULL\n AND (vpRun.label = '%s' OR vpRun.label IS NULL)", label); } Index: /branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c (revision 32850) +++ /branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c (revision 32851) @@ -50,5 +50,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by newExp label (LIKE comparison)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); @@ -56,5 +56,5 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-set_dest_id", 0, "define destination", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous vpRun", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue exposures even if a previous vpRun exists", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -pendingrun