Index: /branches/eam_branches/ipp-20120405/Nebulous-Server/bin/neb-admin =================================================================== --- /branches/eam_branches/ipp-20120405/Nebulous-Server/bin/neb-admin (revision 33948) +++ /branches/eam_branches/ipp-20120405/Nebulous-Server/bin/neb-admin (revision 33949) @@ -625,31 +625,35 @@ # OFFSET $offset # "); - my $query = $dbh->prepare( - " -SELECT Z.so_id,ext_id,Z.ins_id,Z.uri,substr(Z.uri,-10,5) AS class_id,Z.vol_id,vol_name,vol_host, - instance.vol_id AS second_vol_id,volume.name AS second_name,volume.host AS second_host FROM ( - SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host,MAXins_id FROM ( - select * FROM ( - select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id FROM storage_object - JOIN storage_object_xattr USING(so_id) - RIGHT JOIN instance USING(so_id) - WHERE name = 'user.copies' - AND value >= 2 - AND ext_id LIKE '%ota%fits' - AND so_id >= $so_id_start - AND so_id < $so_id_end - GROUP BY so_id - ) AS V - WHERE MAXins_id != MINins_id - ) AS S - JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) - JOIN volume USING(vol_id) WHERE volume.available = 1 - AND volume.name AS $requested_source - ) AS Z -JOIN instance ON (MAXins_id = instance.ins_id AND Z.so_id = instance.so_id) -JOIN volume ON(instance.vol_id = volume.vol_id) WHERE volume.available = 1 -LIMIT $limit -OFFSET $offset -"); + + print STDERR "This query is broken so I have disabled it to get the MHPCC->ATRC shuffle going.\n"; + exit 0; + my $query; +# my $query = $dbh->prepare( +# " +# SELECT Z.so_id,ext_id,Z.ins_id,Z.uri,substr(Z.uri,-10,5) AS class_id,Z.vol_id,vol_name,vol_host, +# instance.vol_id AS second_vol_id,volume.name AS second_name,volume.host AS second_host FROM ( +# SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host,MAXins_id FROM ( +# select * FROM ( +# select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id FROM storage_object +# JOIN storage_object_xattr USING(so_id) +# RIGHT JOIN instance USING(so_id) +# WHERE name = 'user.copies' +# AND value >= 2 +# AND ext_id LIKE '%ota%fits' +# AND so_id >= $so_id_start +# AND so_id < $so_id_end +# GROUP BY so_id +# ) AS V +# WHERE MAXins_id != MINins_id +# ) AS S +# JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) +# JOIN volume USING(vol_id) WHERE volume.available = 1 +# AND volume.name AS $requested_source +# ) AS Z +# JOIN instance ON (MAXins_id = instance.ins_id AND Z.so_id = instance.so_id) +# JOIN volume ON(instance.vol_id = volume.vol_id) WHERE volume.available = 1 +# LIMIT $limit +# OFFSET $offset +# "); $query->execute(); Index: /branches/eam_branches/ipp-20120405/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- /branches/eam_branches/ipp-20120405/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 33948) +++ /branches/eam_branches/ipp-20120405/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 33949) @@ -750,4 +750,20 @@ ### +CREATE TABLE lost_instances ( + id BIGINT NOT NULL AUTO_INCREMENT, + ins_id BIGINT NOT NULL DEFAULT -1, + so_id BIGINT NOT NULL DEFAULT -1, + vol_id INT NOT NULL DEFAULT -1, + uri CHAR(255) NOT NULL DEFAULT 'undefined', + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + mtime TIMESTAMP, + PRIMARY KEY(id), + KEY(so_id), + KEY(vol_id), + KEY(uri(40)) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +### + CREATE TABLE log ( timestamp TIMESTAMP, Index: /branches/eam_branches/ipp-20120405/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/Makefile.in (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/Makefile.in (revision 33949) @@ -47,4 +47,6 @@ $(DESTWWW)/ipp.dvodb.dat \ $(DESTWWW)/ipp.dvodb.php \ +$(DESTWWW)/ipp.czar.dat \ +$(DESTWWW)/ipp.czar.php \ $(DESTWWW)/phptest.php \ $(DESTWWW)/site.php \ @@ -222,6 +224,12 @@ $(DESTWWW)/minidvodbCopy.php \ $(DESTWWW)/minidvodbProcessed.php \ +$(DESTWWW)/mergedvodbRun.php \ +$(DESTWWW)/mergedvodbCopy.php \ +$(DESTWWW)/mergedvodbProcessed.php \ +$(DESTWWW)/dvodbStatus.php \ +$(DESTWWW)/dvodbFaulted.php \ $(DESTWWW)/addRunRun.php \ $(DESTWWW)/addRunProcessed.php \ +$(DESTWWW)/addRunSummary.php \ $(DESTWWW)/lapSequenceSummary.php \ $(DESTWWW)/lapRuns.php \ Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czarpoll.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czarpoll.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czarpoll.pl (revision 33949) @@ -38,5 +38,4 @@ my @stages = ( - "burntool", "chip", "cam", @@ -44,9 +43,9 @@ "warp", "stack", + "staticsky", "diff", - "magic", - "magicDS", "dist", - "pub"); + "pub", + "chipbackground"); my @ippToPspsStages = ( Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool.pl (revision 33949) @@ -32,5 +32,5 @@ my @publishingLabels = undef; -my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); +my @stages = ("chip", "cam", "fake", "warp", "stack", "staticsky", "diff", "dist", "pub", "chipbackground"); promptPoll(); @@ -155,13 +155,25 @@ chomp($label); - - printf("\n+----------------------------------------------------------------------------------------------------------------------------------------------+\n"); - printf("| %32s |\n", $label); - printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n"); - printf("| state | chip | cam | fake | warp | stack | diff | magic | destreak | dist | publish |\n"); - printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n"); + my $stage; + + printf("\n| %32s \n", $label); + printf("+------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); + printf("| state |"); + foreach $stage (@stages) { + + printf("%14s|", $stage) + } + printf("\n"); + printf("+------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); my $state; - my $stage; my $i=0; foreach $state (@states) { @@ -172,5 +184,5 @@ foreach $stage (@stages) { - printf("| %10s ", getStateAndFaultsString($label, $state, $stage)); + printf("|%14s", getStateAndFaultsString($label, $state, $stage)); } @@ -179,5 +191,9 @@ } - printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n"); + printf("+------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); } @@ -191,9 +207,13 @@ my ($state) = @_; #print time, $/; - printf("\n+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n"); - printf("| %10s (any faults are shown in parentheses) |\n", $state); - printf("+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+\n"); - printf("| no | label | distributing? | publishing? | chip | cam | fake | warp | stack | diff | magic | destreak | dist | publish |\n"); - printf("+-----+----------------------------------+---------------+---------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n"); + my $stage; + + printf("|\n"); + printf("+-----+----------------------------------+---------------+---------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); + printf("| no | label | distributing? | publishing? |"); my $stdsLabel; @@ -202,6 +222,16 @@ my $distributing; my $publishing; - my $stage; my $i=1; + foreach $stage (@stages) { + + printf("%14s|", $stage) + } + printf("\n"); + printf("+-----+----------------------------------+---------------+---------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); + foreach $stdsLabel (@stdscienceLabels) { @@ -225,5 +255,5 @@ foreach $stage (@stages) { - printf("| %10s ", getStateAndFaultsString($stdsLabel, $state, $stage)); + printf("|%14s", getStateAndFaultsString($stdsLabel, $state, $stage)); } @@ -232,5 +262,9 @@ } - printf("+-----+----------------------------------+---------------+---------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n"); + printf("+-----+----------------------------------+---------------+---------------+"); + foreach $stage (@stages) { + print "--------------+" + } + printf("\n"); } Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/CzarDb.pm (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/CzarDb.pm (revision 33949) @@ -9,5 +9,4 @@ my @stages = ( - "burntool", "chip", "cam", @@ -16,8 +15,7 @@ "stack", "diff", - "magic", - "magicDS", "dist", - "pub"); # TODO put elsewhere + "pub", + "chipbackground"); # TODO put elsewhere my @ippToPspsStages = ( Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/DayMetrics.pm =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/DayMetrics.pm (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/DayMetrics.pm (revision 33949) @@ -40,5 +40,5 @@ $self->{begin} = "$yesterday " . $self->{config}->getMetricsStartTime(); $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY"); - $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR"); + $self->{chipEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR"); # create path, dir and html file @@ -67,8 +67,8 @@ my $htmlFile = $self->{htmlFile}; - # summit and burntool exposures + # summit and chip exposures my $summitExposures = $self->{gpc1Db}->countScienceExposuresFromLastNight($self->{day}); - my $burntoolMetrics = new czartool::StageMetrics("burntool", "all_stdscience_labels", $self->{begin}, $self->{burntoolEnd}); - $self->{czarDb}->runAnalysis($burntoolMetrics); + my $chipMetrics = new czartool::StageMetrics("chip", "all_stdscience_labels", $self->{begin}, $self->{chipEnd}); + $self->{czarDb}->runAnalysis($chipMetrics); my $previousDay = $self->{czarDb}->subtractInterval($self->{day}, "1 DAY"); @@ -78,6 +78,6 @@ print $htmlFile " \< previous day | all | next day \>
\n"; print $htmlFile "Measured from $self->{begin} to $self->{end} HST
\n"; - printf ( $htmlFile "%d exposures taken on summit last night, %d through burntool today\n", - $summitExposures, $burntoolMetrics->getProcessed() ? $burntoolMetrics->getProcessed() : 0 ); + printf ( $htmlFile "%d exposures taken on summit last night, %d through chip today\n", + $summitExposures, $chipMetrics->getProcessed() ? $chipMetrics->getProcessed() : 0 ); print $htmlFile "\n"; print $htmlFile "\n"; @@ -98,5 +98,14 @@ my %labelTables; - my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); + my @stages = ( + "chip", + "cam", + "fake", + "warp", + "stack", + "staticsky", + "diff", + "dist", + "chipbackground"); @@ -126,5 +135,5 @@ foreach my $stage ( @stages ){ - my $stageMetrics = new czartool::StageMetrics($stage, $label, $self->{begin}, ($stage eq "burntool") ? $self->{burntoolEnd} : $self->{end}); + my $stageMetrics = new czartool::StageMetrics($stage, $label, $self->{begin}, ($stage eq "chip") ? $self->{chipEnd} : $self->{end}); if (!$self->{czarDb}->runAnalysis($stageMetrics)) {next;} @@ -253,5 +262,5 @@ print $htmlFile " \n"; print $htmlFile " Survey\n"; - print $htmlFile " Started burntool\n"; + print $htmlFile " Started chip\n"; print $htmlFile " Finished distribution\n"; print $htmlFile " Time taken\n"; @@ -260,5 +269,5 @@ # OSS survey my ($started, $finished, $timeTaken) = undef; - $self->{czarDb}->getDayTimings("OSS.nightlyscience", "burntool", "diff", $self->{begin}, \$started, \$finished, \$timeTaken); + $self->{czarDb}->getDayTimings("OSS.nightlyscience", "chip", "diff", $self->{begin}, \$started, \$finished, \$timeTaken); $self->printSurveyDetails("OSS", $started, $finished, $timeTaken); @@ -285,5 +294,5 @@ foreach $mdf (@mdfs) { - if ($self->{czarDb}->getDayTimings($mdf, "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken)) { + if ($self->{czarDb}->getDayTimings($mdf, "chip", "dist", $self->{begin}, \$started, \$finished, \$timeTaken)) { if ($started && $self->{czarDb}->isBefore($started, $earliest)) {$earliest = $started; $haveStart = 1;} @@ -299,5 +308,5 @@ $self->printSurveyDetails("MDF", $earliest, $latest, $timeTaken); - $self->{czarDb}->getDayTimings("all_stdscience_labels", "burntool", "dist", $self->{begin}, \$started, \$finished, \$timeTaken); + $self->{czarDb}->getDayTimings("all_stdscience_labels", "chip", "dist", $self->{begin}, \$started, \$finished, \$timeTaken); $self->printSurveyDetails("All", $started, $finished, $timeTaken); Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Gpc1Db.pm =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 33949) @@ -24,4 +24,5 @@ elsif ($stage eq "warp") {${$joinTable}="warpSkyfile";} elsif ($stage eq "stack") {${$joinTable}="stackSumSkyfile";} + elsif ($stage eq "staticsky") {${$id} = "sky_id"; ${$joinTable}="staticskyResult";} elsif ($stage eq "diff") {${$joinTable}="diffSkyfile";} elsif ($stage eq "magic") {${$joinTable}="magicNodeResult";} @@ -29,4 +30,5 @@ elsif ($stage eq "dist") {${$joinTable}="distComponent";} elsif ($stage eq "pub") {${$joinTable}="publishDone";} + elsif ($stage eq "chipbackground") {${$id}="chip_bg_id"; ${$joinTable}="chipBackgroundImfile";} else { @@ -51,4 +53,5 @@ elsif ($stage eq "warp" ) {return "warpRun";} elsif ($stage eq "stack" ) {return "stackRun";} + elsif ($stage eq "staticsky" ) {return "staticskyRun";} elsif ($stage eq "diff" ) {return "diffRun";} elsif ($stage eq "magic" ) {return "magicRun";} @@ -56,4 +59,5 @@ elsif ($stage eq "dist" ) {return "distRun";} elsif ($stage eq "pub" ) {return "publishRun";} + elsif ($stage eq "chipbackground" ) {return "chipBackgroundRun";} return "ERROR"; Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Plotter.pm (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/czartool/Plotter.pm (revision 33949) @@ -11,5 +11,4 @@ my @allStages = ( - "burntool", "chip", "cam", @@ -17,9 +16,9 @@ "warp", "stack", + "staticsky", "diff", - "magic", - "magicDS", "dist", - "pub"); + "pub", + "chipbackground"); my @allIppToPspsStages = ( Index: /branches/eam_branches/ipp-20120405/ippMonitor/czartool/roboczar.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/czartool/roboczar.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/czartool/roboczar.pl (revision 33949) @@ -21,5 +21,14 @@ my $czarDb = $config->getCzarDbInstance(); -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks +my @stages = ( + "chip", + "cam", + "fake", + "warp", + "stack", + "staticsky", + "diff", + "dist", + "chipbackground"); # TODO get from Pantasks my @interestedServers = $config->getRoboczarInterestedServers(); Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunRun.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunRun.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunRun.d (revision 33949) @@ -9,19 +9,18 @@ # field size format name show link to extras -FIELD distinct(addRun.add_id), 5, %d, add ID +FIELD addRun.add_id, 5, %d, addID FIELD addRun.stage, 5, %s, stage -FIELD addRun.stage_id, 5, %d, stage ID -FIELD addRun.add_id, 5, %d, add ID +FIELD addRun.stage_id, 5, %d, stageID +FIELD addRun.stage_extra1, 5, %d, stageExtra1 +#FIELD addRun.add_id, 5, %d, addID FIELD addRun.state, 5, %s, state -FIELD addRun.workdir, 5, %s, workdir -FIELD addRun.reduction, 5, %s, reduction +FIELD addRun.workdir, 5, %s, workdir +FIELD addRun.reduction, 5, %s, reduction FIELD addRun.label, 5, %s, label -FIELD addRun.data_group, 5, %s, data_group +FIELD addRun.data_group, 5, %s, data_group FIELD addRun.dvodb, 5, %s, dvodb -FIELD addRun.note, 5, %s, note -FIELD addRun.image_only, 5, %d, image_only -FIELD addRun.minidvodb, 5, %d, minidvodb -FIELD addRun.minidvodb_group, 5, %s, minidvodb_group -FIELD addRun.minidvodb_name, 5, %s, minidvodb_name - - +FIELD addRun.note, 5, %s, note +FIELD addRun.image_only, 5, %d, image_only +FIELD addRun.minidvodb, 5, %d, minidvodb +FIELD addRun.minidvodb_group, 5, %s, minidvodb_group +FIELD addRun.minidvodb_name, 5, %s, minidvodb_name Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunSummary.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunSummary.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/addRunSummary.d (revision 33949) @@ -0,0 +1,26 @@ +TABLE addRun +#left join addProcessedExp using (add_id) +TITLE dvodb Status +FILE addRunSummary.php +MENU ipp.dvodb.dat + + +#WHERE (minidvodbProcessed.fault > 0 or mergedvodbProcessed.fault > 0) +WHERE add_id > 500000 + +# field width format name show link to extras +FIELD count(addRun.add_id), 5, %d, count(add_id) +FIELD addRun.dvodb, 10, %s, dvodb +FIELD addRun.minidvodb_group, 10, %s, minidvodb_group +FIELD addRun.stage, 5, %s , stage +FIELD addRun.state, 5, %s, state +#FIELD addProcessedExp.fault, 5, %d, fault + +MODE summary +#GROUP BY minidvodbRun.minidvodb_id +#GROUP rawExp.exp_id +GROUP dvodb +GROUP stage +GROUP state +#GROUP fault +ORDER add_id Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbFaulted.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbFaulted.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbFaulted.d (revision 33949) @@ -0,0 +1,25 @@ +TABLE minidvodbRun LEFT JOIN minidvodbProcessed using(minidvodb_id) LEFT JOIN mergedvodbRun using(minidvodb_id) LEFT JOIN mergedvodbProcessed using(merge_id) +TITLE dvodb Status: either mini or merge is faulted +FILE dvodbFaulted.php +MENU ipp.dvodb.dat + + +WHERE (minidvodbProcessed.fault > 0 or mergedvodbProcessed.fault > 0) + +#UNRESTRICTED WHERE minidvodbRun.creation_date >= DATE_ADD(DATE (NOW()), INTERVAL -30 DAY) + +# field width format name show link to extras +FIELD minidvodbRun.minidvodb_id, 5, %d, minidvodb_id +FIELD minidvodbRun.minidvodb_name, 5, %s, minidvodb_name +FIELD minidvodbRun.state, 5, %s, minidvodb_state +FIELD minidvodbProcessed.fault, 5, %d, minidvodb fault +FIELD mergedvodbRun.merge_id, 5, %d, merge_id +FIELD mergedvodbRun.mergedvodb, 5, %s, mergedvodb +FIELD mergedvodbRun.state, 5, %s, mergedvodb_state +FIELD mergedvodbProcessed.fault, 5, %d, mergedvodb fault + + + +#MODE summary +#GROUP BY minidvodbRun.minidvodb_id +#GROUP rawExp.exp_id Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbStatus.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbStatus.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/dvodbStatus.d (revision 33949) @@ -0,0 +1,25 @@ +TABLE minidvodbRun LEFT JOIN minidvodbProcessed using(minidvodb_id) LEFT JOIN mergedvodbRun using(minidvodb_id) LEFT JOIN mergedvodbProcessed using(merge_id) +TITLE dvodb Status +FILE dvodbStatus.php +MENU ipp.dvodb.dat + + +#WHERE (minidvodbProcessed.fault > 0 or mergedvodbProcessed.fault > 0) + +#UNRESTRICTED WHERE minidvodbRun.creation_date >= DATE_ADD(DATE (NOW()), INTERVAL -30 DAY) + +# field width format name show link to extras +FIELD minidvodbRun.minidvodb_id, 5, %d, minidvodb_id +FIELD minidvodbRun.minidvodb_name, 5, %s, minidvodb_name +FIELD minidvodbRun.state, 5, %s, minidvodb_state +FIELD minidvodbProcessed.fault, 5, %d, minidvodb fault +FIELD mergedvodbRun.merge_id, 5, %d, merge_id +FIELD mergedvodbRun.mergedvodb, 5, %s, mergedvodb +FIELD mergedvodbRun.state, 5, %s, mergedvodb_state +FIELD mergedvodbProcessed.fault, 5, %d, mergedvodb fault + + + +#MODE summary +#GROUP BY minidvodbRun.minidvodb_id +#GROUP rawExp.exp_id Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/exposureStatus.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/exposureStatus.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/exposureStatus.d (revision 33949) @@ -2,5 +2,5 @@ TITLE Exposures Status FILE exposureStatus.php -MENU ipp.imfiles.dat +MENU ipp.czar.dat WHERE object != 'ENGINEERING' Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/gpc1MysqlProcessList.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/gpc1MysqlProcessList.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/gpc1MysqlProcessList.d (revision 33949) @@ -2,5 +2,5 @@ TITLE GPC 1 MySql Process List FILE gpc1MysqlProcessList.php -MENU ipp.imfiles.dat +MENU ipp.czar.dat FIELD ID, 5, %d, Process ID Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/maskStats.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/maskStats.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/maskStats.d (revision 33949) @@ -2,5 +2,5 @@ TITLE Mask Stats FILE maskStats.php -MENU ipp.imfiles.dat +MENU ipp.czar.dat WHERE rawExp.exp_id = chipRun.exp_id Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbCopy.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbCopy.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbCopy.d (revision 33949) @@ -0,0 +1,19 @@ +TABLE mergedvodbCopy +TITLE mergedvodb Copies +FILE mergedvodbCopy.php +MENU ipp.dvodb.dat + +# the following WHERE clauses are added to all queries joined by AND + +#WHERE mergedvodbcopy_id >= 0 + +# field size format name show link to extras +FIELD mergedvodbCopy.mergedvodbcopy_id, 5, %d, merge Copy ID +FIELD mergedvodbCopy.merge_id, 5, %d, merge ID +FIELD mergedvodbCopy.mergedvodb_rsync_path, 25, %s, rsync path +FIELD mergedvodbCopy.destination_host, 5, %s, destinatin host +FIELD mergedvodbCopy.fault, 5, %s, Fault +FIELD mergedvodbCopy.state, 5, %s, State +FIELD mergedvodbCopy.epoch, 5, %T, Epoch +FIELD mergedvodbCopy.dtime, 5, %d, dtime + Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbProcessed.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbProcessed.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbProcessed.d (revision 33949) @@ -0,0 +1,21 @@ +TABLE mergedvodbRun, mergedvodbProcessed +TITLE Processed mergedvodbs +FILE mergedvodbProcessed.php +MENU ipp.dvodb.dat + +# the following WHERE clauses are added to all queries joined by AND + +WHERE mergedvodbRun.merge_id = mergedvodbProcessed.merge_id + +# field size format name show link to extras +FIELD mergedvodbRun.merge_id, 5, %d, merge ID +FIELD mergedvodbRun.minidvodb_id, 5, %d, minidvodb_id +FIELD mergedvodbRun.mergedvodb, 5, %s, mergedvodb +FIELD mergedvodbProcessed.merge_order, 5, %d, merge order +FIELD mergedvodbProcessed.dtime_verify, 5, %d, verify time +FIELD mergedvodbProcessed.dtime_merge, 5, %d, merge time +FIELD mergedvodbProcessed.dtime_script, 5, %d, script time +FIELD state, 5, %s, State +FIELD mergedvodbProcessed.fault, 5, %d, Fault +FIELD epoch, 5, %T, epoch + Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbRun.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbRun.d (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/mergedvodbRun.d (revision 33949) @@ -0,0 +1,21 @@ +TABLE mergedvodbRun, minidvodbRun +TITLE mergedvodb Runs +FILE mergedvodbRun.php +MENU ipp.dvodb.dat + +# the following WHERE clauses are added to all queries joined by AND + +WHERE merge_id >= 0 +WHERE mergedvodbRun.minidvodb_id = minidvodbRun.minidvodb_id + +# field size format name show link to extras +FIELD mergedvodbRun.merge_id, 5, %d, merge ID +FIELD mergedvodbRun.minidvodb_id, 5, %d, minidvodb ID +FIELD minidvodbRun.minidvodb_name, 10, %s, minidvodb_name +FIELD mergedvodbRun.mergedvodb, 5, %s, mergedvodb +###FIELD mergedvodbRun.mergedvodb_group, 5, %s, mergedvodb_group +FIELD mergedvodbRun.mergedvodb_path, 25, %s, mergedvodb_path +#FIELD mergedvodbRun.mergedvodb_path, 25, %s, mergedvodb_path +FIELD mergedvodbRun.state, 5, %s, State +FIELD mergedvodbRun.creation_date, 5, %T, epoch + Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbProcessed.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbProcessed.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbProcessed.d (revision 33949) @@ -9,13 +9,10 @@ # field size format name show link to extras -FIELD distinct(minidvodbRun.minidvodb_id), 5, %d, Minidvodb ID - +FIELD minidvodbRun.minidvodb_id, 5, %d, Minidvodb ID FIELD minidvodbRun.minidvodb_name, 5, %s, minidvodb_name FIELD minidvodbRun.minidvodb_group, 5, %s, minidvodb_group -FIELD minidvodbProcessed.merge_order, 5, %d, Merge Order FIELD minidvodbProcessed.dtime_resort, 5, %d, resort time FIELD minidvodbProcessed.dtime_relphot, 5, %d, relphot time -FIELD minidvodbProcessed.dtime_merge, 5, %d, dvomerge time -FIELD minidvodbProcessed.mergedvodb_path, 25, %s, mergedvodb_path +FIELD minidvodbProcessed.dtime_script, 5, %d, script time FIELD state, 5, %s, State FIELD minidvodbProcessed.fault, 5, %d, Fault Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbRun.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbRun.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/minidvodbRun.d (revision 33949) @@ -9,9 +9,9 @@ # field size format name show link to extras -FIELD distinct(minidvodbRun.minidvodb_id), 5, %d, Minidvodb ID +FIELD minidvodbRun.minidvodb_id, 5, %d, Minidvodb ID FIELD minidvodbRun.minidvodb_name, 5, %s, minidvodb_name FIELD minidvodbRun.minidvodb_group, 5, %s, minidvodb_group FIELD minidvodbRun.minidvodb_path, 25, %s, minidvodb_path -FIELD minidvodbRun.mergedvodb_path, 25, %s, mergedvodb_path +#FIELD minidvodbRun.mergedvodb_path, 25, %s, mergedvodb_path FIELD state, 5, %s, State FIELD creation_date, 5, %T, epoch Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/mopsStatus.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/mopsStatus.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/mopsStatus.d (revision 33949) @@ -2,5 +2,5 @@ TITLE MOPS Status FILE mopsStatus.php -MENU ipp.imfiles.dat +MENU ipp.czar.dat #WHERE rawExp.obs_mode!='ENGINEERING' Index: /branches/eam_branches/ipp-20120405/ippMonitor/def/warpProcessedSkyfiles_Images.d =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 33949) @@ -1,5 +1,5 @@ TABLE rawExp JOIN chipRun USING(exp_id) JOIN camRun USING(chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) JOIN warpSkyfile USING(warp_id) TITLE Warp Processed Skyfiles -FILE warpProcessedSkyfiles.php +FILE warpProcessedSkyfiles_Images.php MENU ipp.science.dat Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_ipptopsps.php =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_ipptopsps.php (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_ipptopsps.php (revision 33949) @@ -23,5 +23,5 @@ if ($ID['menu']) {$myMenu = $ID['menu'];} -else {$myMenu = "ipp.imfiles.dat";} +else {$myMenu = "ipp.czar.dat";} menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); @@ -78,5 +78,5 @@ $debug = 0; -$table = "\n"; +$table = "
\n"; Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_labels.php (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/czartool_labels.php (revision 33949) @@ -23,10 +23,9 @@ if ($ID['menu']) {$myMenu = $ID['menu'];} -else {$myMenu = "ipp.imfiles.dat";} +else {$myMenu = "ipp.czar.dat";} menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); $stages=array( - "burntool", "chip", "cam", @@ -34,9 +33,9 @@ "warp", "stack", + "staticsky", "diff", - "magic", - "magicDS", "dist", - "pub"); + "pub", + "chipbackground"); $servers=array( @@ -92,5 +91,5 @@ $debug = 0; -$table = "
\n"; +$table = "
\n"; $labels = getLabels($czardb, $selectedMode); @@ -441,8 +440,4 @@ $anyFaults = false; - $link = $defaultlink; - getStateAndFaults($db, $thisLabel, $selectedState, "burntool", $str, $anyFaults); - createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0); - $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel . "&chipRun.state=".$searchState; getStateAndFaults($db, $thisLabel, $selectedState, "chip", $str, $anyFaults); @@ -465,4 +460,8 @@ createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0); + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "staticsky", $str, $anyFaults); + createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0); + $link = "failedDiffSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $thisLabel . "&diffRun.state=".$searchState; getStateAndFaults($db, $thisLabel, $selectedState, "diff", $str, $anyFaults); @@ -470,12 +469,4 @@ $link = $defaultlink; - getStateAndFaults($db, $thisLabel, $selectedState, "magic", $str, $anyFaults); - createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$distributing) ? "gray" : null); - - $link = $defaultlink; - getStateAndFaults($db, $thisLabel, $selectedState, "magicDS", $str, $anyFaults); - createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$distributing) ? "gray" : null); - - $link = $defaultlink; getStateAndFaults($db, $thisLabel, $selectedState, "dist", $str, $anyFaults); createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$distributing) ? "gray" : null); @@ -485,4 +476,8 @@ createFormattedTableCell($anyFaults, $link, $str, $anyFaults, (!$isUpdate && !$publishing) ? "gray" : null); + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "chipbackground", $str, $anyFaults); + createFormattedTableCell($anyFaults, $link, $str, $anyFaults, 0); + echo "\n"; } @@ -492,29 +487,11 @@ createFormattedTableCell(($selectedLabel != "all"), $link, "All labels", 0, null); - createFormattedTableCell(0, null, "Reverting?", 0, "$columnHeaderColor"); - foreach ($stages as &$stage) { - if ($stage == "burntool") continue; $reverting = getRevertStatus($db, $stage); - #$link = - # "czartool_labels.php?pass=".$pass - # ."&proj=".$proj - # ."&mode=" .$selectedMode - # ."&label=".$selectedLabel - # ."&stage=".$selectedStage - # ."&plottype=".$plotType - # ."&revertstage=".$stage - # ."&revertmode="; -# - #if(!$reverting) {$label = "Start";$link = $link . "on";} - #if($reverting) {$label = "Stop";$link = $link . "off";} - if(!$reverting) {$label = "no";} - if($reverting) {$label = "yes";} + if(!$reverting) {$label = "";} + if($reverting) {$label = "reverting";} unset($reverting); - #createFormattedTableCell(1, $link, $label, 0, "$columnHeaderColor"); createFormattedTableCell(0, null, $label, 0, "$columnHeaderColor"); - #echo ""; - } Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.dat =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.dat (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.dat (revision 33949) @@ -0,0 +1,13 @@ +input | ipp.menu.dat + +menutop | menutop | plain |   | + +menulink | menuselect | link | czartool - standard | czartool_labels.php +menulink | menuselect | link | czartool - ipptopsps | czartool_ipptopsps.php +menulink | menuselect | link | mask stats | maskStats.php +menulink | menuselect | link | night summary | nightSummary.php +menutop | menutop | plain |   | +menulink | menuselect | link | Exposures Status | exposureStatus.php +menulink | menuselect | link | MOPS Exposures Status | mopsStatus.php +menulink | menuselect | link | GPC1 MySql ProcessList | gpc1MysqlProcessList.php + Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.php =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.php (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.czar.php (revision 33949) @@ -0,0 +1,17 @@ +\n"; + +menu_end(); + +?> Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.dvodb.dat =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.dvodb.dat (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.dvodb.dat (revision 33949) @@ -9,5 +9,16 @@ menulink | menuselect | link | minidvodbRun | minidvodbRun.php -menulink | menuselect | link | minidvodbProcessed | minidvodbProcessed.php +menulink | menuselect | link | minidvodbProcessed | minidvodbProcessed.php menulink | menuselect | link | minidvodbCopy | minidvodbCopy.php +menutop | menutop | plain |   | + +menulink | menuselect | link | mergedvodbRun | mergedvodbRun.php +menulink | menuselect | link | mergedvodbProcessed | mergedvodbProcessed.php +menulink | menuselect | link | mergedvodbCopy | mergedvodbCopy.php + +menutop | menutop | plain |   | + +menulink | menuselect | link | dvodbStatus | dvodbStatus.php +menulink | menuselect | link | dvodbFaulted | dvodbFaulted.php +menulink | menuselect | link | addRun Summary | addRunSummary.php Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.menu.dat =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.menu.dat (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.menu.dat (revision 33949) @@ -27,3 +27,4 @@ menutop | menutop | link | Calibration | ipp.cal.php menutop | menutop | link | Plots | ipp.plots.php +menutop | menutop | link | Czar | ipp.czar.php menutop | menutop | link | Admin and Debug | ipp.imfiles.php Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.science.dat =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.science.dat (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/ipp.science.dat (revision 33949) @@ -38,4 +38,5 @@ menulink | menuselect | link | Warp Skyfile Inputs | warpStageSkyfileInputs.php menulink | menuselect | link | Warp Processed Skyfiles | warpProcessedSkyfiles.php +#menulink | menuselect | link | Warp Processed Skyfiles w/ Images | warpProcessedSkyfiles_images.php menulink | menuselect | link | Warp Failed Skyfiles | warpFailedSkyfiles.php Index: /branches/eam_branches/ipp-20120405/ippMonitor/raw/nightSummary.php =================================================================== --- /branches/eam_branches/ipp-20120405/ippMonitor/raw/nightSummary.php (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippMonitor/raw/nightSummary.php (revision 33949) @@ -52,5 +52,5 @@ $myMenu = $ID['menu']; } else { - $myMenu = "ipp.imfiles.dat"; + $myMenu = "ipp.czar.dat"; } Index: /branches/eam_branches/ipp-20120405/ippScripts/scripts/destreak_restore_camera.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippScripts/scripts/destreak_restore_camera.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippScripts/scripts/destreak_restore_camera.pl (revision 33949) @@ -422,5 +422,5 @@ # looks like a censored one if (scalar @{$file{bad_instances}}) { - my $bad = ${$file{bad_instances}}->[0]; + my $bad = ${$file{bad_instances}}[0]; $file{censored} = ($bad =~ /SR_/); print "no good instances found setting censored based on $bad\n"; Index: /branches/eam_branches/ipp-20120405/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippScripts/scripts/nightly_science.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippScripts/scripts/nightly_science.pl (revision 33949) @@ -1539,4 +1539,8 @@ $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; # $cmd .= " -pretend "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + if (defined($pretend)) { $cmd .= ' -pretend '; @@ -1670,4 +1674,8 @@ $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $new_data_group "; $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + # $cmd .= " -pretend "; if (defined($pretend)) { @@ -1724,4 +1732,7 @@ $check_cmd .= " -pretend -simple -rerun -set_label $label -filter $filter "; $check_cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + if (defined($reduction)) { + $check_cmd .= " -set_reduction $reduction "; + } my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -1745,4 +1756,7 @@ $cmd .= " -simple -rerun -set_label $label -filter $filter "; $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } if (defined($pretend)) { Index: /branches/eam_branches/ipp-20120405/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippScripts/scripts/staticsky.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippScripts/scripts/staticsky.pl (revision 33949) @@ -144,4 +144,5 @@ my $path_base = $file->{path_base}; print "input: $path_base\n"; + my $stack_id = $file->{stack_id}; my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name @@ -172,4 +173,5 @@ &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); + print $listFile " STACK_ID S64 " . $stack_id . "\n"; print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; print $listFile " RAW:MASK STR " . $maskRaw . "\n"; Index: /branches/eam_branches/ipp-20120405/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/ipp-20120405/ippTasks/pstamp.pro (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTasks/pstamp.pro (revision 33949) @@ -13,4 +13,5 @@ $pstampFin_DB = 0 $pstampRev_DB = 0 +$pstampRevReq_DB = 0 $pstampDep_DB = 0 $pstampRevDep_DB = 0 @@ -117,4 +118,7 @@ macro pstamp.revert.on + task pstamp.request.revert + active true + end task pstamp.job.revert active true @@ -128,4 +132,7 @@ end macro pstamp.revert.off + task pstamp.request.revert + active false + end task pstamp.job.revert active false @@ -662,4 +669,44 @@ end +task pstamp.request.revert + host local + + periods -poll $LOADPOLL + periods -exec 1200 + periods -timeout 20 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.request.revert.log + $run = pstamptool -revertreq + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRevReq_DB + $pstampRevReq_DB ++ + if ($pstampRevReq_DB >= $DB:n) set pstampRevReq_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end task pstamp.job.revert host local Index: /branches/eam_branches/ipp-20120405/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/Makefile.am (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/Makefile.am (revision 33949) @@ -450,4 +450,5 @@ vptool_pendingrun.sql \ vptool_processedcell.sql \ - vptool_revertrun.sql + vptool_revertrun.sql \ + sctool_list.sql Index: /branches/eam_branches/ipp-20120405/ippTools/share/chiptool_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/chiptool_pendingimfile.sql (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/chiptool_pendingimfile.sql (revision 33949) @@ -6,4 +6,6 @@ rawImfile.magicked as raw_magicked, (rawImfile.user_1 is not NULL and rawImfile.user_1 > 0.5) as deburned, + rawImfile.burntool_state as burntool_state, + rawImfile.video_cells as video_cells, rawExp.exp_tag, rawExp.exp_name, Index: /branches/eam_branches/ipp-20120405/ippTools/share/pstamptool_revertreq_deletejobs.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 33949) @@ -3,3 +3,4 @@ FROM pstampJob JOIN pstampRequest USING(req_id) -WHERE pstampRequest.state = 'new' +WHERE pstampRequest.state = 'new' + AND pstampRequest.fault > 0 Index: /branches/eam_branches/ipp-20120405/ippTools/share/sctool_list.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/sctool_list.sql (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippTools/share/sctool_list.sql (revision 33949) @@ -0,0 +1,11 @@ +SELECT + skycell.tess_id, + skycell.skycell_id, + TRUNCATE(skycell.radeg/15., 4) AS rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat, + skycell.width, + skycell.height +FROM skycell Index: /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_inputs.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_inputs.sql (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_inputs.sql (revision 33949) @@ -6,4 +6,5 @@ FROM stackRun JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) WHERE stackRun.state = 'full' AND stackSumSkyfile.fault = 0 Index: /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_select.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_select.sql (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_definebyquery_select.sql (revision 33949) @@ -10,4 +10,5 @@ FROM stackRun JOIN stackSumSkyfile USING(stack_id) + JOIN skycell using(tess_id, skycell_id) WHERE stackRun.state = 'full' AND stackSumSkyfile.fault = 0 @@ -29,4 +30,5 @@ JOIN stackRun USING(stack_id) JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) WHERE 1 -- restrict by selected filters, stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label Index: /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_result.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_result.sql (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/share/staticskytool_result.sql (revision 33949) @@ -6,7 +6,10 @@ stackRun.tess_id, stackRun.skycell_id, - count(stackRun.filter) AS num_filters + count(stackRun.filter) AS num_filters, + skycell.glong, + skycell.glat FROM staticskyRun JOIN staticskyResult USING(sky_id) JOIN staticskyInput USING(sky_id) -JOIN stackRun using(stack_id) +JOIN stackRun USING(stack_id) +LEFT JOIN skycell USING(tess_id, skycell_id) Index: /branches/eam_branches/ipp-20120405/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/addtool.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/addtool.c (revision 33949) @@ -1417,5 +1417,5 @@ psString query = NULL; psStringAppend(&query, bare_query, minidvodb_group); - psStringAppend(&query, where_string); + psStringAppend(&query," %s ", where_string); Index: /branches/eam_branches/ipp-20120405/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/pstamptoolConfig.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/pstamptoolConfig.c (revision 33949) @@ -125,4 +125,6 @@ psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + // this argument is supplied by pantasks so we accept it but don't use it + psMetadataAddU64(revertreqArgs, PS_LIST_TAIL, "-limit", 0, "not used", 0); // -pendingcleanup Index: /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.c (revision 33949) @@ -56,2 +56,27 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-decl", 0, "DEC value for radius search (degrees)", false); } + +void pxskycellAddArguments(psMetadata *md) +{ + psMetadataAddF32(md, PS_LIST_TAIL, "-ra_min", 0, "search by right ascension (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-dec_min", 0, "search by declination (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ra_max", 0, "search by right ascension (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-dec_max", 0, "search by declination (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-glong_min", 0, "search by galactic longitude (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-glat_min", 0, "search by galactic latitude (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-glong_max", 0, "search by galactic longitude (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-glat_max", 0, "search by galactic latitude (degrees)", NAN); +} + +bool pxskycellAddWhere(pxConfig *config, psMetadata *where) +{ + PXOPT_COPY_F32(config->args, where, "-ra_min", "skycell.radeg", ">="); + PXOPT_COPY_F32(config->args, where, "-dec_min", "skycell.decdeg", ">="); + PXOPT_COPY_F32(config->args, where, "-ra_max", "skycell.radeg", "<"); + PXOPT_COPY_F32(config->args, where, "-dec_max", "skycell.decdeg", "<"); + PXOPT_COPY_F32(config->args, where, "-glong_min", "skycell.glong", ">="); + PXOPT_COPY_F32(config->args, where, "-glat_min", "skycell.glat", ">="); + PXOPT_COPY_F32(config->args, where, "-glong_max", "skycell.glong", "<"); + PXOPT_COPY_F32(config->args, where, "-glat_max", "skycell.glat", "<"); + return true; +} Index: /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.h =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.h (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/pxspace.h (revision 33949) @@ -1,4 +1,4 @@ /* - * pxmagic.h + * pxspace.h * * Copyright (C) 2009 IfA @@ -24,3 +24,6 @@ extern void pxspaceAddArguments(psMetadata *md); +extern bool pxskycellAddWhere(pxConfig *config, psMetadata *where); +extern void pxskycellAddArguments(psMetadata *md); + #endif // PXSPACE_H Index: /branches/eam_branches/ipp-20120405/ippTools/src/sctool.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/sctool.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/sctool.c (revision 33949) @@ -31,8 +31,10 @@ #include "pxdata.h" #include "pxchip.h" +#include "pxspace.h" #include "sctool.h" static bool defineskycellsMode(pxConfig *config); +static bool listMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -53,5 +55,6 @@ switch (config->mode) { - MODECASE(SCTOOL_MODE_DEFINESKYCELLS, defineskycellsMode); + MODECASE(SCTOOL_MODE_DEFINESKYCELLS, defineskycellsMode); + MODECASE(SCTOOL_MODE_LIST, listMode); default: psAbort("invalid option (this should not happen)"); @@ -131,2 +134,65 @@ return true; } +static bool listMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "skycell.tess_id", "LIKE"); + pxskycellAddWhere(config, where); + + psString query = pxDataGet("sctool_list.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search paramters are required\n"); + psFree(where); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("sctool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "skycell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + return true; +} Index: /branches/eam_branches/ipp-20120405/ippTools/src/sctool.h =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/sctool.h (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/sctool.h (revision 33949) @@ -26,4 +26,5 @@ SCTOOL_MODE_NONE = 0x0, SCTOOL_MODE_DEFINESKYCELLS, + SCTOOL_MODE_LIST, } sctoolMode; Index: /branches/eam_branches/ipp-20120405/ippTools/src/sctoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/sctoolConfig.c (revision 33949) +++ /branches/eam_branches/ipp-20120405/ippTools/src/sctoolConfig.c (revision 33949) @@ -0,0 +1,86 @@ +/* + * sctoolConfig.c + * + * Copyright (C) 2012 IfA University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "sctool.h" + +pxConfig *sctoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -defineskycells + psMetadata *defineskycellsArgs = psMetadataAlloc(); + psMetadataAddStr(defineskycellsArgs, PS_LIST_TAIL, "-input", 0, "file containing skycell data (required)", NULL); + + // -list + psMetadata *listArgs = psMetadataAlloc(); + psMetadataAddStr(listArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(listArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess_id (LIKE comparision)", NULL); + // add skycell coordinate search arguments + pxskycellAddArguments(listArgs); + psMetadataAddU64(listArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-defineskycells", "insert skycells from mdc file", SCTOOL_MODE_DEFINESKYCELLS, defineskycellsArgs); + PXOPT_ADD_MODE("-list", "list skycell parameters", SCTOOL_MODE_LIST, listArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/ipp-20120405/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/stacktool.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/stacktool.c (revision 33949) @@ -232,4 +232,6 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); + PXOPT_COPY_F32(config->args, where, "-select_elong_max", "(camProcessedExp.fwhm_major / camProcessedExp.fwhm_minor)", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_elong_max", "(camProcessedExp.iq_fwhm_major / camProcessedExp.iq_fwhm_minor)", "<="); PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); @@ -239,4 +241,6 @@ PXOPT_COPY_F32(config->args, where, "-select_zpt_obs_min", "camProcessedExp.zpt_obs", ">="); PXOPT_COPY_F32(config->args, where, "-select_zpt_obs_max", "camProcessedExp.zpt_obs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_bg_max", "camProcessedExp.bg", "<="); + PXOPT_COPY_F32(config->args, where, "-select_bg_stdev_max", "camProcessedExp.bg_stdev", "<="); PXOPT_COPY_F32(config->args, where, "-select_astrom", "sqrt(POWER(camProcessedExp.sigma_ra, 2) + POWER(camProcessedExp.sigma_dec, 2))", "<="); Index: /branches/eam_branches/ipp-20120405/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/stacktoolConfig.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/stacktoolConfig.c (revision 33949) @@ -82,4 +82,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_elong_max", 0, "define max elongation from fwhm ", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_elong_max", 0, "define max elongation from iq_fwhm ", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN); @@ -90,4 +92,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_min", 0, "define min zero point", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_max", 0, "define max zero point", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_bg_max", 0, "define max background", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_bg_stdev_max", 0, "define max background stdev", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_astrom", 0, "define max astrometry rms", NAN); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", PS_META_DUPLICATE_OK, "include this warp ID (multiple OK)", 0); Index: /branches/eam_branches/ipp-20120405/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/staticskytool.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/staticskytool.c (revision 33949) @@ -29,4 +29,5 @@ #include "pxtools.h" +#include "pxspace.h" #include "staticskytool.h" @@ -131,4 +132,6 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min", "skycell.glat", ">="); + PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max", "skycell.glat", "<="); // find the number of requested filters: @@ -390,7 +393,13 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "staticskyRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "staticskyRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } if (!psListLength(where->list)) { psFree(where); @@ -399,5 +408,5 @@ } - psString query = psStringCopy("UPDATE staticskyRun"); + psString query = psStringCopy("UPDATE staticskyRun JOIN staticskyInput USING(sky_id) JOIN stackRun using(stack_id) JOIN skycell USING(tess_id, skycell_id)"); // pxUpdateRun gets parameters from config->args and updates @@ -638,4 +647,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); PXOPT_LOOKUP_S32(num_filters, config->args, "-num_filters", false, false); Index: /branches/eam_branches/ipp-20120405/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippTools/src/staticskytoolConfig.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippTools/src/staticskytoolConfig.c (revision 33949) @@ -55,4 +55,6 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_glat_min", 0, "define min galactic latitude", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_glat_max", 0, "define max galactic latitude", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); @@ -73,4 +75,7 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess_id", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + pxskycellAddArguments(updaterunArgs); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -113,4 +118,5 @@ psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); + pxskycellAddArguments(resultArgs); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: /branches/eam_branches/ipp-20120405/ppImage/src/ppImage.h =================================================================== --- /branches/eam_branches/ipp-20120405/ppImage/src/ppImage.h (revision 33948) +++ /branches/eam_branches/ipp-20120405/ppImage/src/ppImage.h (revision 33949) @@ -57,4 +57,8 @@ bool addNoise; // Add noise to degrade MD image to 3pi + bool hasVideo; // Determine if this OTA has a video cell + bool useVideoDark; // Should we use a video dark if we can? + bool useVideoMask; // Should we use a video mask if we can? + // output files requested bool BaseFITS; Index: /branches/eam_branches/ipp-20120405/ppImage/src/ppImageDetrendReadout.c =================================================================== --- /branches/eam_branches/ipp-20120405/ppImage/src/ppImageDetrendReadout.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ppImage/src/ppImageDetrendReadout.c (revision 33949) @@ -35,5 +35,18 @@ } } - + // Check to see if we're in a chip that contains video + bool hasVideo = false; + { + if (psMetadataLookupStr(NULL,input->parent->parent->hdu->header,"CELLMODE")) { + psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); + char *Vptr = strchr(psMetadataLookupStr(NULL,input->parent->parent->hdu->header,"CELLMODE"),'V'); + if (Vptr) { + hasVideo = true; + psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); + } + } + } + + // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine. if (options->doMaskBuild) { @@ -44,6 +57,12 @@ // apply the externally supplied mask to the input->mask pixels if (options->doMask) { - pmReadout *mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.MASK"); - pmMaskBadPixels(input, mask, options->maskValue); + pmReadout *mask; + if ((options->useVideoMask)&&(hasVideo)) { + mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.VIDEOMASK"); + } + else { + mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.MASK"); + } + pmMaskBadPixels(input, mask, options->maskValue); } if (options->doMaskBurntool) { @@ -80,9 +99,19 @@ psMetadata *recipe = psMetadataLookupPtr (&mdok, config->recipes, RECIPE_NAME); assert(mdok && recipe); - if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + + if ((options->useVideoDark)&&(hasVideo)) { + if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + oldDark = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.VIDEODARK"); + } else { + dark = pmFPAfileThisCell(config->files, detview, "PPIMAGE.VIDEODARK"); + } + } + else { + if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { oldDark = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.DARK"); - } else { + } else { dark = pmFPAfileThisCell(config->files, detview, "PPIMAGE.DARK"); - } + } + } } Index: /branches/eam_branches/ipp-20120405/ppImage/src/ppImageOptions.c =================================================================== --- /branches/eam_branches/ipp-20120405/ppImage/src/ppImageOptions.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ppImage/src/ppImageOptions.c (revision 33949) @@ -45,5 +45,7 @@ options->doMaskStats = false; // Calculate mask fractions options->addNoise = false; //Degrade an MD image to a 3pi image - + options->hasVideo = false; // Determine if this OTA has a video cell + options->useVideoDark = false; // Use video dark if we can? + options->useVideoMask = false; // Use video mask if we can? // output files requested options->BaseFITS = false; // create output image @@ -362,4 +364,12 @@ options->fringeKeep = psMetadataLookupF32(NULL, recipe, "FRINGE.KEEP"); + // Video cell options + if (psMetadataLookup(recipe, "USE.VIDEO.DARK")) { + options->useVideoDark = psMetadataLookupBool(NULL,recipe,"USE.VIDEO.DARK"); + } + if (psMetadataLookup(recipe, "USE.VIDEO.MASK")) { + options->useVideoMask = psMetadataLookupBool(NULL,recipe,"USE.VIDEO.MASK"); + } + // Pattern correction if (psMetadataLookup(format, "PATTERN.ROW.ORDER")) { @@ -418,5 +428,5 @@ options->patternContinuityEdgeWidth = psMetadataLookupS32(NULL, recipe, "PATTERN.CONTINUITY.WIDTH"); } - + // Remnance options Index: /branches/eam_branches/ipp-20120405/ppImage/src/ppImageParseCamera.c =================================================================== --- /branches/eam_branches/ipp-20120405/ppImage/src/ppImageParseCamera.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ppImage/src/ppImageParseCamera.c (revision 33949) @@ -22,4 +22,28 @@ ppImageOptions *options = ppImageOptionsParse(config); + // parse the header to extract out whether this OTA has video cells. + // CZW: 2012-05-01 I'm somewhat ashamed of what needs to be done to get to the header. There has to be a simpler way, right? +/* pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest */ +/* pmChip *chip; // Chip from FPA */ +/* while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { */ +/* if (!chip->process || !chip->file_exists) { */ +/* continue; */ +/* } */ +/* /\* if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { *\/ */ +/* /\* continue; *\/ */ +/* /\* } *\/ */ +/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ +/* if (psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE")) { */ +/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ +/* char *Vptr = strchr(psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE"),'V'); */ +/* if (Vptr) { */ +/* options->hasVideo = true; */ +/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ +/* } */ +/* } */ +/* } */ +/* pmFPAviewReset(view); */ +/* psFree(view); */ + // the following are defined from the argument list, if given, // otherwise they revert to the config information @@ -42,18 +66,38 @@ } if (options->doDark) { - if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.DARK", "DARK", - PM_FPA_FILE_DARK, PM_DETREND_TYPE_DARK)) { + // Always load the regular Dark + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.DARK", "DARK", + PM_FPA_FILE_DARK, PM_DETREND_TYPE_DARK)) { + psError(PS_ERR_IO, false, "Can't find a dark image source"); + psFree(options); + return NULL; + } + // Sometimes load the video dark if we need it. + if (options->useVideoDark) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.VIDEODARK", "DARK", + PM_FPA_FILE_DARK, PM_DETREND_TYPE_VIDEODARK)) { psError(PS_ERR_IO, false, "Can't find a dark image source"); psFree(options); return NULL; } + } } if (options->doMask) { - if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.MASK", "MASK", - PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) { + // Always load the regular mask + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.MASK", "MASK", + PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) { + psError(PS_ERR_IO, false, "Can't find a mask image source"); + psFree(options); + return NULL; + } + + if (options->useVideoMask) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.VIDEOMASK", "MASK", + PM_FPA_FILE_MASK, PM_DETREND_TYPE_VIDEOMASK)) { psError(PS_ERR_IO, false, "Can't find a mask image source"); psFree(options); return NULL; } + } } if (options->doShutter) { Index: /branches/eam_branches/ipp-20120405/ppMerge/src/ppMergeCamera.c =================================================================== --- /branches/eam_branches/ipp-20120405/ppMerge/src/ppMergeCamera.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/ppMerge/src/ppMergeCamera.c (revision 33949) @@ -224,6 +224,6 @@ format = (HDU)->format; \ } else if (format != (HDU)->format) { \ - psError(PS_ERR_UNKNOWN, true, "Camera format %d doesn't match: %p vs %p", \ - i, format, (HDU)->format); \ + psError(PS_ERR_UNKNOWN, true, "Camera format %d doesn't match: %p vs %p on %s %s", \ + i, format, (HDU)->format, (HDU)->extname, psMetadataLookupStr(NULL,(HDU)->header,"FILENAME")); \ psFree(phuView); \ return false; \ @@ -232,4 +232,23 @@ } \ } + // CZW: 2012-04-10 Version of above to allow different camera formats to be processed together. +/* #define CHECK_LEVEL(HDU, CHIP, CELL) { \ */ +/* if (HDU) { \ */ +/* if (!phuView) { \ */ +/* phuView = pmFPAviewAlloc(0); \ */ +/* phuView->chip = CHIP; \ */ +/* phuView->cell = CELL; \ */ +/* } else if ((phuView->chip != (CHIP)) && (phuView->cell != (CELL))) { \ */ +/* psError(PS_ERR_UNKNOWN, true, "Differing PHU for input %d", i); \ */ +/* psFree(phuView); \ */ +/* return false; \ */ +/* } \ */ +/* if (!format) { \ */ +/* format = (HDU)->format; \ */ +/* } else if (format != (HDU)->format) { \ */ +/* } \ */ +/* continue; \ */ +/* } \ */ +/* } */ psMetadata *format = NULL; ///< Camera format Index: /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadCrosstalk.c =================================================================== --- /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadCrosstalk.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadCrosstalk.c (revision 33949) @@ -53,5 +53,5 @@ pmFPAview *viewMask = pmFPAviewAlloc (0); - float zeropt, exptime, MAX_MAG; + float zeropt, exptime, MAX_MAG, SPIKE_MAX_MAG; psLogMsg ("psastro", PS_LOG_INFO, "determine crosstalk positions"); @@ -105,4 +105,8 @@ MAX_MAG += MagOffset; + // Get the spike maximum, as well + SPIKE_MAX_MAG = psMetadataLookupF32(&status, recipe, "REFSTAR_MASK_BLEED_MAG_MAX"); + SPIKE_MAX_MAG += MagOffset; + psTrace("psastro.crosstalk",2,"%f %f %f %f\n",zeropt,exptime,MAX_MAG,MagOffset); @@ -259,5 +263,5 @@ pmAstromObj *crosstalk = pmAstromObjAlloc(); - + crosstalk->Mag = ref->Mag - MagOffset; crosstalk->chip->x = x_t_chip; @@ -274,8 +278,92 @@ } psArrayAdd(crosstalks,100,crosstalk); - + psFree(targetChipName); psFree(targetCellName); psFree(crosstalk); + + // Determine if we need to add a spike mask. +/* psWarning("BLAH: %g %g %g\n",ref->Mag,MagOffset,SPIKE_MAX_MAG); */ + if (ref->Mag < SPIKE_MAX_MAG) { + if ((X == 2)&&(! ((U == 3)||(U == 5)||(U == 6)))) { + psTrace ("psastro.crosstalk",2,"Cell (%d%d) on chip (%d%d) not a known crosstalk source.",U,V,X,Y); + continue; + } + if ((X == 3)&&(! ((U == 3)))) { + psTrace ("psastro.crosstalk",2,"Cell (%d%d) on chip (%d%d) not a known crosstalk source.",U,V,X,Y); + continue; + } + if ((X == 4)&&(! ((U == 3)))) { + psTrace ("psastro.crosstalk",2,"Cell (%d%d) on chip (%d%d) not a known crosstalk source.",U,V,X,Y); + continue; + } + if ((X == 5)&&(! ((U == 3)||(U == 5)||(U == 6)))) { + psTrace ("psastro.crosstalk",2,"Cell (%d%d) on chip (%d%d) not a known crosstalk source.",U,V,X,Y); + continue; + } + + Xt = X; + Yt = Y; + Vt = V; + + for (Ut = 0; Ut < 8; Ut++) { + + psString targetChipName = NULL; + psStringAppend(&targetChipName,"XY%d%d",Xt,Yt); + psString targetCellName = NULL; + psStringAppend(&targetCellName,"xy%d%d",Ut,Vt); + + psTrace ("psastro.crosstalk",2,"CTsource @ OTA%d%dxy%d%d@%f,%f CTtarget OTA%d%dxy%d%d@%f,%f [%s %s]", + X,Y,U,V,x_cell,y_cell,Xt,Yt,Ut,Vt,x_t_cell,y_t_cell, + targetChipName,targetCellName); +/* psWarning ("CT bleed ct @ OTA%d%dxy%d%d@%f,%f CTtarget OTA%d%dxy%d%d@%f,%f [%s %s]", */ +/* X,Y,U,V,x_cell,y_cell,Xt,Yt,Ut,Vt,x_t_cell,y_t_cell, */ +/* targetChipName,targetCellName); */ + + pmCell *CTtargetCell = getCellByName (CTsourceChip,targetCellName); + if (!CTtargetCell) continue; + + pmChipCoordsForCell(&x_t_chip,&y_t_chip,CTtargetCell,x_t_cell,y_t_cell); + + psTrace ("psastro.crosstalk",2,"CTsource @ OTA%d%dxy%d%d@%f,%f CTtarget OTA%d%dxy%d%d@%f,%f ChipLoc: (%f,%f)", + X,Y,U,V,x_cell,y_cell,Xt,Yt,Ut,Vt,x_t_cell,y_t_cell,x_t_chip,y_t_chip); + + // Hunt down the readout for the target, and save the chip position and magnitude of source star. + pmChip *targetChip = getChipByName(fpa,targetChipName); + if (!targetChip) continue; + if (!targetChip->cells) continue; + if (!targetChip->cells->n) continue; + + pmCell *targetCell = targetChip->cells->data[0]; + if (!targetCell) continue; + if (!targetCell->readouts) continue; + if (!targetCell->readouts->n) continue; + pmReadout *targetReadout = targetCell->readouts->data[0]; + if (!targetReadout) continue; + + pmAstromObj *crosstalk = pmAstromObjAlloc(); + + crosstalk->Mag = ref->Mag; + crosstalk->chip->x = x_t_chip; + crosstalk->chip->y = y_t_chip; + + psArray *crosstalks = psMetadataLookupPtr (&status, targetReadout->analysis, "PSASTRO.CROSSTALKS.SPIKES"); + if (crosstalks == NULL) { + crosstalks = psArrayAllocEmpty(100); + if (!psMetadataAdd(targetReadout->analysis,PS_LIST_TAIL,"PSASTRO.CROSSTALKS.SPIKES", PS_DATA_ARRAY, "crosstalk locations", crosstalks)) { + psError(PSASTRO_ERR_CONFIG, false, "failure to add crosstalks to readout"); + goto escape; + } + psFree(crosstalks); + } + psArrayAdd(crosstalks,100,crosstalk); + + psFree(targetChipName); + psFree(targetCellName); + psFree(crosstalk); + } + } // End satspike crosstalks + + } // refstars } // readout Index: /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadGhosts.c =================================================================== --- /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadGhosts.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/psastro/src/psastroLoadGhosts.c (revision 33949) @@ -128,4 +128,5 @@ for (int i = 0; i < refstars->n; i++) { pmAstromObj *ref = refstars->data[i]; + psTrace("psastro.ghost",5,"Begin ghost %d/%ld: MAX_MAG: %g; ref_mag: %g @ (%.10g,%.10g)",i,refstars->n,MAX_MAG,ref->Mag,ref->sky->r * 180 / M_PI,ref->sky->d * 180.0 / M_PI); if (ref->Mag > MAX_MAG) continue; @@ -163,11 +164,46 @@ pmChip *ghostChip = psastroFindChip (&ghost->chip->x, &ghost->chip->y, fpa, -ghost->srcFP->x, -ghost->srcFP->y); // fprintf (stderr, "raw chip position: %f, %f ", ghost->chip->x, ghost->chip->y); - +/* float ref_chip_x = ghost->chip->x; */ +/* float ref_chip_y = ghost->chip->y; */ ghostChip = psastroFindChip (&ghost->chip->x, &ghost->chip->y, fpa, ghost->FP->x, ghost->FP->y); // fprintf (stderr, "-> model chip position: %f, %f\n", ghost->chip->x, ghost->chip->y); + if (ghostChip) { + psTrace("psastro.ghost",5," in ghost: %d/%ld: ref: (%f,%f) or (%s) ghost: (%f,%f) or (%f,%f,%s) %f inner: (%f,%f,%f) outer: (%f,%f,%f)", + i,refstars->n, + ref->FP->x,ref->FP->y, + psMetadataLookupStr(NULL,chip->concepts,"CHIP.NAME"), + ghost->FP->x,ghost->FP->y, + ghost->chip->x,ghost->chip->y,psMetadataLookupStr(NULL,ghostChip->concepts,"CHIP.NAME"), + rSrc, + ghost->inner.major,ghost->inner.minor,ghost->inner.theta, + ghost->outer.major,ghost->outer.minor,ghost->outer.theta); +/* psWarning(" GHOST_DATA: %d/%ld: ref: (%f,%f) or (%f,%f,%s) ghost: (%f,%f) or (%f,%f,%s) %f inner: (%f,%f,%f) outer: (%f,%f,%f)", */ +/* i,refstars->n, */ +/* ref->FP->x,ref->FP->y, */ +/* ref_chip_x,ref_chip_y,psMetadataLookupStr(NULL,chip->concepts,"CHIP.NAME"), */ +/* ghost->FP->x,ghost->FP->y, */ +/* ghost->chip->x,ghost->chip->y,psMetadataLookupStr(NULL,ghostChip->concepts,"CHIP.NAME"), */ +/* rSrc, */ +/* ghost->inner.major,ghost->inner.minor,ghost->inner.theta, */ +/* ghost->outer.major,ghost->outer.minor,ghost->outer.theta); */ + } + else { + psTrace("psastro.ghost",5," in ghost: %d/%ld: ref: (%f,%f) ghost: (%f,%f) or (%f,%f,%s) inner: (%f,%f,%f) outer: (%f,%f,%f)", + i,refstars->n, + ref->FP->x,ref->FP->y, + ghost->FP->x,ghost->FP->y, + ghost->chip->x,ghost->chip->y,"NONE", + ghost->inner.major,ghost->inner.minor,ghost->inner.theta, + ghost->outer.major,ghost->outer.minor,ghost->outer.theta); + } + + + if (!ghostChip) goto skip; if (!ghostChip->cells) goto skip; if (!ghostChip->cells->n) goto skip; + + pmCell *ghostCell = ghostChip->cells->data[0]; if (!ghostCell) goto skip; @@ -190,4 +226,5 @@ skip: + psFree (ghost); } Index: /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUpdates.c =================================================================== --- /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUpdates.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUpdates.c (revision 33949) @@ -406,4 +406,31 @@ // XXX for now, assume cell binning is 1x1 relative to chip psastroMaskCircle (readoutMask->mask, crosstalkMaskValue, ref->chip->x, ref->chip->y, radius, radius); + } + } + // Crosstalk Bleeds + psArray *bleedcrosstalks = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.CROSSTALKS.SPIKES"); + if (bleedcrosstalks) { + for (int i = 0; i < bleedcrosstalks->n; i++) { + pmAstromObj *ref = bleedcrosstalks->data[i]; + float width = REFSTAR_MASK_BLEED_MAG_SLOPE * (REFSTAR_MASK_BLEED_MAG_MAX - ref->Mag); + psTrace("psastro.crosstalk",2,"Masking spike on Chip %s @ (%f,%f) Magnitude: %f Radius %f\n", + psMetadataLookupStr(&status,readout->parent->parent->concepts,"CHIP.NAME"), + ref->chip->x,ref->chip->y,ref->Mag,width); + // XXX for now, assume cell binning is 1x1 relative to chip + pmCell *refCell = pmCellInChip(refChip,ref->chip->x,ref->chip->y); + if (refCell) { + float xCell = 0.0; + float yCell = 0.0; + pmCellCoordsForChip (&xCell, &yCell, refCell, ref->chip->x, ref->chip->y); + int ySize = psMetadataLookupS32(NULL,refCell->concepts,"CELL.YSIZE"); +/* psWarning("Masking CTspike on Chip %s @ (%f,%f) Magnitude: %f (%f %f) Radius %f Z: %d %d %d %d\n", */ +/* psMetadataLookupStr(&status,readout->parent->parent->concepts,"CHIP.NAME"), */ +/* ref->chip->x,ref->chip->y,ref->Mag,REFSTAR_MASK_BLEED_MAG_SLOPE,REFSTAR_MASK_BLEED_MAG_MAX,width, */ +/* (int) (ref->chip->x - 0.5 * width), (int) (ref->chip->y - yCell), */ +/* (int) (ref->chip->x+0.5 * width + 1), (int) (ref->chip->y + (ySize - yCell))); */ + psastroMaskRectangle (readoutMask->mask, crosstalkMaskValue, + (int) (ref->chip->x - 0.5 * width), (int) (ref->chip->y - yCell), + (int) (ref->chip->x+0.5 * width + 1), (int) (ref->chip->y + (ySize - yCell))); + } } } Index: /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUtils.c =================================================================== --- /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUtils.c (revision 33948) +++ /branches/eam_branches/ipp-20120405/psastro/src/psastroMaskUtils.c (revision 33949) @@ -135,5 +135,5 @@ // XXX optimize this float A = 1.0; - float B = shape.sxy*y*PS_SQR(shape.sx); + float B = 2.0 * shape.sxy*y*PS_SQR(shape.sx); float C = PS_SQR(y*shape.sx/shape.sy) - PS_SQR(shape.sx); Index: /branches/eam_branches/ipp-20120405/tools/czarcal =================================================================== --- /branches/eam_branches/ipp-20120405/tools/czarcal (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/czarcal (revision 33949) @@ -24,5 +24,4 @@ mark serge -roy chris gene Index: /branches/eam_branches/ipp-20120405/tools/dlapstacks =================================================================== --- /branches/eam_branches/ipp-20120405/tools/dlapstacks (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/dlapstacks (revision 33949) @@ -215,6 +215,6 @@ } else { # otherwise use a temporary directory + $outdir = $tempdir; print "deasined files will be saved to $outdir\n" if $save_temps; - $outdir = $tempdir; } } Index: /branches/eam_branches/ipp-20120405/tools/heather/dvomergecheck.pl =================================================================== --- /branches/eam_branches/ipp-20120405/tools/heather/dvomergecheck.pl (revision 33949) +++ /branches/eam_branches/ipp-20120405/tools/heather/dvomergecheck.pl (revision 33949) @@ -0,0 +1,86 @@ +#!/usr/bin/env perl + +#This is a dumb script: It takes as arguments the minidvodb directory and a merge +# dvodb directory. It finds the newest date ($date) in the minidvodb directory, and +# for each filename in the minidvodb directory, checks that it exists in the +# merge directory and that it was updated more recently than $date +# +# It's probable that the files listed as old were not merged into the mergedvodb +# and should be merged in. +# +# It's also possible to fool the script by updating something silly in the minidvodb +# directory, giving a fake $date to compare to. + + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +my ($mini, $mergy); +GetOptions( + 'minidvodb_path=s' => \$mini, # Database host name + 'mergedvodb_path=s' => \$mergy, # Database name + ) or die "Unable to parse arguments.\n"; +die "Unknown option: @ARGV\n" if @ARGV; +die "Required options: --minidvodb_path --mergedvodb_path\n" + unless defined $mini + and defined $mergy; + + + +my $dir = `pwd`; + +#$mini = '/data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809.V2.Mini/LAP.ThreePi.20110809.V2.SSky.1583'; +#$mergy = '/data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809.V2'; + + +my $date_split = `ls -o -1 --time-style +%s --color=never $mini | sort -k 5 | tail -n 1`; +my @splits= split (/\s+/, $date_split); +my $unixdate= $splits[4]; +print $date_split; +my $datecom = "date --date=\@"."$unixdate"; +my $date =`$datecom`; +chomp $date; +print "Latest date in minidvodb directory: $date or $unixdate (unix)\n"; + + +# 3 days, + +my @list_of_dirs = ('n0000', 'n0730', 'n1500', 'n2230', 'n3000', 'n3730', 'n4500', 'n5230', 'n6000', 'n6730', 'n7500', 'n8230', 's0000', 's0730', 's1500', 's2230', 's3000', 's3730', 's4500', 's5230', 's6000', 's6730', 's7500','s8230'); + +foreach my $subdir (@list_of_dirs) { + + +chdir $mini.'/'.$subdir; + +print "grabbing $mini/$subdir\n"; +my @list_of_mini_files = `ls -1 --color=never `; + +chdir $mergy.'/'.$subdir; + +print "grabbing $mergy/$subdir\n"; + +my @list_of_bad = (); + +my @list_of_mergy_files = `ls -alrt --time-style +%s > ~/mergy.$subdir.list`; + +foreach my $minifile (@list_of_mini_files) { + chomp $minifile; + my $mergyfile = `grep $minifile ~/mergy.$subdir.list`; + + my @splits = split (/\s+/, $mergyfile); + # print $mergyfile; + if ($#splits > 5) { + + my $num = `date -d "$date" +%s`; + if ($splits[5] < $num ) { + + print "old: $subdir/$minifile\n"; + } + } else { + + print "dne: $subdir/$minifile\n"; + + } +} +} Index: /branches/eam_branches/ipp-20120405/tools/listcmf =================================================================== --- /branches/eam_branches/ipp-20120405/tools/listcmf (revision 33949) +++ /branches/eam_branches/ipp-20120405/tools/listcmf (revision 33949) @@ -0,0 +1,368 @@ +#!/usr/bin/env perl + +# listcmf +# +# A program to read cmf files for skycells and produce text files that may be used +# to read the data into opihi programs (mana, dvo) using the opihi command 'read' +# +# Depends on the HEASOFT program ftlist which must be in the path +# +# one line is produced per source (IPP_IDET) +# +# If "exended source analysis" (xsrc) and "extended source fits" (xfits) extensions exist +# in the cmf file selected parameters from those tables are included +# Sources that do not have extended source or models data have their corresponding columns set to NULL +# so when read in mana they get set to NAN +# +# the option +# --opihi_read_file (mana filename) +# is supplied a file that contains the command to read the vectors written to the specified filename +# +# The filter is appended to the names of the columns (except for IPP_IDET) so that list files +# made from the set of outputs from a multifilter staticskyRun may be read in sequence to yield +# a set of vectors with multi-filter information for each object +# +# For example +# listcmf my.r.band.cmf r.list --opihi_read_file r.mana +# listcmf my.i.band.cmf i.list --opihi-read-file i.mana +# +# To read the data into mana +# +# datafile r.list +# input r.mana +# +# The vectors are populated and the variables $ZPT_r and $EXPTIME_r are set +# +# datafile i.list +# input i.mana +# +# set i_minus_r = $ZPT_i + PSF_INST_MAG_i + 2.5 * log($EXPTIME_i) - ($ZPT_r + PSF_INST_MAG_r + 2.5 * log($EXPTIME_r)) +# +# The model fits columns names are appended with a tag for the model name as well as the filter +# So to compare the de Vaucouleurs model magnitude in two filters +# +# set dev_i_minus_r = $ZPT_i + EXT_INST_MAG_DEV_i + 2.5 * log($EXPTIME_i) - ($ZPT_r + EXT_INST_MAG_DEV_r + 2.5 * log($EXPTIME_r)) +# +# +# set sersic_i_minus_r = $ZPT_i + EXT_INST_MAG_SERSIC_i + 2.5 * log($EXPTIME_i) - ($ZPT_r + EXT_INST_MAG_SERSIC_r + 2.5 * log($EXPTIME_r)) + +use strict; +use warnings; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename; + +# make sure HEASOSFT is intialized +die "HEASOFT not initalized try running heainit\n" if !defined $ENV{HEADAS}; + + +my ( $cmf, $out, $opihi_read_file, $append, $save_temps, $verbose); + +GetOptions( + 'opihi-read-file|r=s' => \$opihi_read_file, + 'append' => \$append, + 'save-temps' => \$save_temps, + 'verbose|v' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV > 2; + +$cmf = $ARGV[0]; +$out = $ARGV[1]; + +# first read the psf extension +my $input = $cmf; + +die "usage: listcmf [ ]\n" unless defined($cmf); + +# output file handle +my $oh; +if (defined $out and $out ne "-") { + my $to_open = $append ? ">>$out" : ">$out"; + open $oh, $to_open or die "failed to open output list file: $out\n"; +} else { + $oh = *STDOUT; +} + + +my ($exp_time, $zero_point, $filter); +my %extensions; +# read the keywords and list of extensions from the header +{ + my $header_data = `ftlist $cmf kh include=EXPTIME,FPA.ZP,FPA.FILTERID`; + + die "failed to extract header information form $cmf\n" if !$header_data; + + my @lines = split "\n", $header_data; + # require at least a header and psf extension and the 3 keywords + die "failed to extract header information from $cmf\n" if scalar @lines < 11; + + foreach my $line (@lines) { + next if !$line; + next if $line =~ /------/; + if ($line =~ /HDU/) { + my (undef, $hdu_num, $hdu_name, $type, $cols, undef, undef, $rows) = split " ", $line; + + if (! ($hdu_name =~ /SkyChip/) ) { + print STDERR "unexpected extension name found: $hdu_name\n"; + die "expected SkyChip.*\n"; + } + my (undef, $ext) = split '\.', $hdu_name; + if ($type eq 'BinTable') { + printf STDERR "HDU %2d %4s %-12s %4d cols %7d rows\n", $hdu_num, $ext, $type, $cols, $rows if $verbose; + # save extension offset (number - 1) in extensions hash + $extensions{$ext} = $hdu_num - 1; + } elsif ($ext eq 'hdr') { + # primary header + printf STDERR "HDU %2d %4s %-12s\n", $hdu_num, $ext, "$type $cols" if $verbose; + } + } elsif ($line =~ /=/) { + # keyword line + my ($key, $value); + if ($line =~ /HIERARCH/) { + (undef, $key, undef, $value) = split " ", $line; + # print STDERR "$key $value\n" if $verbose; + } else { + ($key, undef, $value) = split " ", $line; + # print STDERR "$key $value\n" if $verbose; + } + if ($key eq 'EXPTIME') { + $exp_time = $value; + } elsif ($key eq 'FPA.FILTERID') { + $filter = substr $value, 1, 1; + } elsif ($key eq 'FPA.ZP') { + $zero_point = $value; + } + } + } + die "failed to find filter in header" unless defined $filter; + die "failed to find exposure time in header" unless defined $exp_time; + die "failed to find zero point in header" unless defined $zero_point; + printf STDERR "$cmf: %2d extensions filter: $filter zpt: $zero_point exposure_time: $exp_time\n", scalar (keys %extensions) if $verbose; +} + +my $psf_ext = $extensions{psf}; +die "no psf extension found in $cmf\n" unless $psf_ext; + +print STDERR "psf extension offset is $psf_ext\n" if $verbose; + +# get the psf extensions columns +$input = "$cmf+$psf_ext"; +my @ids; +my @lines; +my @psf_cols; + +read_extension($input, 'psf', \@ids, \@lines, \@psf_cols, undef, "PADDING"); + +printf STDERR "read %d cols %d rows from psf extension\n", scalar @psf_cols, scalar @lines if $verbose; + +my @xsrc_cols; +my $xsrc_ext = $extensions{xsrc}; +if ($xsrc_ext) { + $input = "$cmf+$xsrc_ext"; + read_extension($input, 'xsrc', \@ids, \@lines, \@xsrc_cols, undef, + 'PROF_SB,PROF_FLUX,PROF_FILL'); + + my $total_cols = scalar @psf_cols + scalar @xsrc_cols; + my $n_psf = scalar @psf_cols; + extend_incomplete_lines(\@lines, $total_cols, $n_psf); +} else { + print STDERR "no xsrc extension found in $cmf\n" unless $xsrc_ext; +} + +my @xfit_cols; +my $xfit_ext = $extensions{xfit}; +if ($xfit_ext) { + my $xfit_includes = 'IPP_IDET,X_EXT,Y_EXT,X_EXT_SIG,Y_EXT_SIG,EXT_INST_MAG,EXT_INST_MAG_SIG,NPARAMS,MODEL_TYPE,EXT_WIDTH_MAJ,EXT_WIDTH_MIN,EXT_THETA,EXT_PAR_07'; + + $input = "$cmf+$xfit_ext"; + my @ext_cols; + read_extension($input, 'xfit', \@ids, \@lines, \@ext_cols, $xfit_includes, + undef); + + my @model_types; + for (my $i = 0; $i < scalar @lines; $i++) { + next unless $lines[$i] =~ /PS_MODEL/; + my @words = split " ", $lines[$i]; + for (my $j = 0; $j < scalar @words; $j++) { + next unless $words[$j] =~ /PS_MODEL/; + my (undef, undef, $type) = split /\_/, $words[$j]; + push @model_types, $type; + } + last; + } + die "no model types found\n" if !scalar @model_types; + # create new column names one for each model type + foreach my $model_type (@model_types) { + foreach my $col (@ext_cols) { + push @xfit_cols, "${col}_${model_type}"; + } + } + my $ncols_previous = scalar @psf_cols + scalar @xsrc_cols; + my $total_cols = $ncols_previous + scalar @xfit_cols; + extend_incomplete_lines(\@lines, $total_cols, $ncols_previous); +} else { + print STDERR "no xfit extension found in $cmf\n" unless $xfit_ext; +} + +my $base = basename($cmf) ; +{ + print $oh "# cmf: $base FILTER: $filter ZPT: $zero_point EXPTIME: $exp_time\n"; + print $oh "#\n"; + print $oh "# "; + + print $oh "IPP_IDET "; + for (my $i = 1; $i < scalar @psf_cols; $i++) { + print $oh " $psf_cols[$i]_$filter"; + } + for (my $i = 0; $i < scalar @xsrc_cols; $i++) { + print $oh " $xsrc_cols[$i]_$filter" + } + for (my $i = 0; $i < scalar @xfit_cols; $i++) { + print $oh " $xfit_cols[$i]_$filter"; + } + print $oh "\n"; +} + +foreach my $line (@lines) { + print $oh "$line\n"; +} +close $oh; + +if ($opihi_read_file) { + open OUT, ">$opihi_read_file" or die "failed to open $opihi_read_file\n"; + printf OUT '$EXPTIME_%s = %f' . "\n", $filter, $exp_time; + printf OUT '$ZPT_%s = %f' . "\n", $filter, $zero_point; + print OUT "read IPP_IDET 1"; + my $n = 2; + for (my $i = 1; $i < scalar @psf_cols; $i++) { + print OUT " $psf_cols[$i]_$filter ", $n++; + } + for (my $i = 0; $i < scalar @xsrc_cols; $i++) { + print OUT " $xsrc_cols[$i]_$filter ", $n++; + } + for (my $i = 0; $i < scalar @xfit_cols; $i++) { + print OUT " $xfit_cols[$i]_$filter ", $n++; + } + print OUT "\n"; + close OUT; +} + + +sub read_extension { + my ($input, $ext, $r_ids, $r_lines, $r_cols, $include_list, $exclude_list) = @_; + + die "can't have both include list and exclude list\n" if defined $include_list and defined $exclude_list; + + my $col_list = `ftlist $input c`; + die "failed to read column list\n" unless $col_list; + + my @lines = split "\n", $col_list; + die "failed to read column list\n" unless scalar @lines; + + my $exclude_str = ""; + my %exclude; + if ($exclude_list) { + $exclude_str = '\'[col '; + my @cols_to_skip = split '\,', $exclude_list; + foreach my $col (@cols_to_skip) { + $exclude{$col} = 1; + $exclude_str .= "-$col;"; + } + $exclude_str .= ']\''; + } + my %include; + if ($include_list) { + my @cols_to_keep = split '\,', $include_list; + foreach my $col (@cols_to_keep) { + $include{$col} = 1; + } + } + + # process the column list including and excluding columns based + # on the arguments + foreach my $line (@lines) { + next if !$line; + next if $line =~ /HDU/; + my ($col, $name, $format) = split " ", $line; + next if $col =~ /\D/; # not a number + if (defined($include_list) && ! $include{$name}) { + print STDERR "excluding $name\n" if $verbose; + next; + } elsif ($exclude{$name}) { + print STDERR "excluding $name\n" if $verbose; + next; + } elsif ($ext ne 'psf' and $name eq 'IPP_IDET') { + print STDERR "excluding $name from $ext column list\n" if $verbose; + next; + } else { + # it's a keeper + # XXX TODO: check the format and insure that it is not a vector + # because we currently don't support them + } + push @$r_cols, $name; + } + + my $cmd = "ftlist $input$exclude_str t rownum=no colheader=no"; + $cmd .= " columns=$include_list" if $include_list; + print STDERR "$cmd\n" if $verbose; + + @lines = `$cmd`; + if ($ext eq 'psf') { + # psf extension contains our list of ids + # XXX: we are assuming here that the ids in a psf extension + # are sequential + foreach my $line (@lines) { + # id is first column + chomp $line; + my ($ipp_idet) = split " ", $line; + push @$r_ids, $ipp_idet; + push @$r_lines, $line; + } + } else { + my $num_objects = scalar @$r_lines; + foreach my $line (@lines) { + chomp $line; + my ($id, $rest_of_line) = identify_line($line); + if ($id > $num_objects) { + die "TILT: id $id is greater than num objects\n"; + } + # just append the line to this object's line + $r_lines->[$id] .= "$rest_of_line"; + } + } +} + +# find the id for this line and return it and the rest of the string +# following the id keeping white space +sub identify_line { + my $line = shift; + + my ($id) = split " ", $line; + + my $start_of_id = index $line, $id; + my $rest_of_line = substr $line, ($start_of_id + length($id)); + + return ($id, $rest_of_line); +} +sub extend_incomplete_lines { + my $lines = shift; + my $ntotal = shift; + my $nprevious = shift; + for (my $i = 0; $i < scalar @$lines; $i++) { + my @words = split " ", $lines->[$i]; + my $nwords = scalar @words; + if ($nwords == $ntotal) { + # excellent + } elsif ($nwords == $nprevious) { + # need to add null words + for (my $j = $nwords; $j < $ntotal; $j++) { + $lines->[$i] .= " NULL"; + } + } else { + # we are confused + die "unexpected number of words found $nwords on line $i expected $nprevious or $ntotal\n"; + } + } +} Index: /branches/eam_branches/ipp-20120405/tools/repair_bad_instance =================================================================== --- /branches/eam_branches/ipp-20120405/tools/repair_bad_instance (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/repair_bad_instance (revision 33949) @@ -123,5 +123,10 @@ return 0; } + if (! ($nextend =~ /\d+/)) { + print "failed to find NEXTEND for $file. command returned $nextend\n"; + return 0; + } print "\t$file: $nextend extensions\n"; + $command = 'fhead -x ' . ($nextend - 1) . " $file | grep EXTNAME | awk '{print \$3}'"; Index: /branches/eam_branches/ipp-20120405/tools/runbgchip.pl =================================================================== --- /branches/eam_branches/ipp-20120405/tools/runbgchip.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/runbgchip.pl (revision 33949) @@ -97,5 +97,10 @@ exit 0 if $pretend; -exit system $command; +my $rc = system $command; +my $status = $rc >> 8; +if ($rc) { + print "command returned $rc $status\n"; +} +exit $status; Index: /branches/eam_branches/ipp-20120405/tools/runchipimfile.pl =================================================================== --- /branches/eam_branches/ipp-20120405/tools/runchipimfile.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/runchipimfile.pl (revision 33949) @@ -110,5 +110,13 @@ exit 0 if $pretend; -exit system $command; +my $rc = system $command; +my $status = $rc >> 8; +if ($rc) { + print STDERR "command failed with $rc $status\n"; + $status = 1 unless $status; +} else { + print "Successfully rebuilt chip $chip_id $class_id\n"; +} +exit $status; Index: /branches/eam_branches/ipp-20120405/tools/runwarpskycell.pl =================================================================== --- /branches/eam_branches/ipp-20120405/tools/runwarpskycell.pl (revision 33948) +++ /branches/eam_branches/ipp-20120405/tools/runwarpskycell.pl (revision 33949) @@ -105,5 +105,6 @@ exit 0 if $pretend; -exit system $command; +my $rc = system $command; +exit $rc >> 8;
$label