Index: /trunk/dbconfig/background.md =================================================================== --- /trunk/dbconfig/background.md (revision 29492) +++ /trunk/dbconfig/background.md (revision 29493) @@ -21,4 +21,5 @@ class_id STR 64 path_base STR 255 + data_state STR 64 magicked S64 0 dtime_script F32 0.0 @@ -57,4 +58,5 @@ skycell_id STR 64 path_base STR 255 + data_state STR 64 magicked S64 0 dtime_script F32 0.0 Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 29492) +++ /trunk/dbconfig/changes.txt (revision 29493) @@ -1945,2 +1945,10 @@ ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell); ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell); + +-- changes to support cleanup of the background preserved images +ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; +ALTER TABLE chipBackgroundImfile ADD KEY(data_state); +ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; +ALTER TABLE warpBackgroundSkyfile ADD KEY(data_state); + +ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac; Index: /trunk/ippScripts/scripts/magic_process.pl =================================================================== --- /trunk/ippScripts/scripts/magic_process.pl (revision 29492) +++ /trunk/ippScripts/scripts/magic_process.pl (revision 29493) @@ -17,4 +17,5 @@ use IPC::Cmd 0.36 qw( can_run run ); use File::Temp qw( tempfile ); +use File::Copy; use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); @@ -41,5 +42,5 @@ # Parse the command-line arguments -my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile); +my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot); GetOptions( @@ -49,4 +50,5 @@ 'dbname=s' => \$dbname, # Database name 'baseroot=s' => \$baseroot, # Output root name + 'final-outroot=s' => \$final_outroot, # location for final outputs 'save-temps' => \$save_temps, # Save temporary files? 'verbose' => \$verbose, # Print stuff? @@ -86,4 +88,16 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# list of VerifyStreaks input and output files to copy to nebulous +my @verify_outputs = qw( +clusterPos.txt +duplicate.png +mask.png +original.png +original.fits +residual.png +residual.fits +clusters.list +); ### Get a list of inputs @@ -312,25 +326,4 @@ -### Input result into database -{ - my $command = "$magictool -addresult"; - $command .= " -magic_id $magic_id"; - $command .= " -node $node"; - $command .= " -path_base $outroot"; - $command .= " -dbname $dbname" if defined $dbname; - - # Add the processed file to the database - unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code); - } - } else { - print "Skipping command: $command\n"; - } -} - if ($node eq "root") { my $streaks_file = "$outroot.streaks"; @@ -349,5 +342,4 @@ } - &run_verifystreaks($baseroot); my $exp_id; # Exposure identifier @@ -366,4 +358,6 @@ } + &run_verifystreaks($baseroot, $exp_id); + { my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file @@ -383,9 +377,43 @@ } + my $output_streaks = $final_outroot . ".streaks"; + if ($output_streaks and ($output_streaks ne $streaks_file)) { + copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1); + foreach my $f (@verify_outputs) { + my $src = "$baseroot.verify/${exp_id}_$f"; + my $dest = "$final_outroot.${f}"; + copy_to_nebulous($ipprc, $src, $dest, 1); + } + } else { + $output_streaks = $streaks_file; + } + +### Input result into database +# XXXX: if this succeeds but addmask fails the magicRun is in a goofy state XXXX +{ + my $command = "$magictool -addresult"; + $command .= " -magic_id $magic_id"; + $command .= " -node $node"; + $command .= " -path_base $outroot"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + { my $command = "$magictool -addmask"; $command .= " -magic_id $magic_id"; - $command .= " -uri $streaks_file"; + $command .= " -uri $final_outroot"; $command .= " -streaks $num_streaks"; $command .= " -dbname $dbname" if defined $dbname; @@ -411,4 +439,5 @@ my $baseroot = shift; + my $exp_id = shift; unless ($VerifyStreaks) { @@ -431,6 +460,7 @@ my @files = <$baseroot.*.clusters>; - unless (open ($FILE, ">$outdir/clusters.list")) { - print "failed to create cluster file $outdir/clusters.list\n"; + my $clusters_list = "$outdir/${exp_id}_clusters.list"; + unless (open ($FILE, ">$clusters_list")) { + print "failed to create cluster file $clusters_list\n"; return 1; } @@ -441,9 +471,9 @@ close ($FILE); if ($status) { - print "failed to create cluster file $outdir/clusters.list\n"; + print "failed to create cluster file $clusters_list\n"; return 1; } - my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap"; + my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -513,4 +543,40 @@ } +# Copy a file to nebulous and optionally replicate it +# We should consider making this an ipprc function. For now try it here so we can print +# the right error messages +sub copy_to_nebulous { + my $ipprc = shift; + my $src = shift; + my $dest = shift; + my $replicate = shift; + + print "copying $src to $dest\n"; + + $ipprc->file_exists($src) or + &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + + # copy the file to it's final destination - which is presumably in nebulous + # we delete it so that all instances are deleted in case it has been replicated + if ($ipprc->file_exists($dest)) { + $ipprc->file_delete($dest) or + &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + } + my $dest_resolved = $ipprc->file_resolve($dest, 'create'); + &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved; + + copy ($src, $dest_resolved) or + &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + + if ($replicate and (file_scheme($dest) eq 'neb')) { + my $neb = $ipprc->nebulous(); + $neb->setxattr($dest, 'user.copies', 2, 'create') or + &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + + $neb->replicate($dest) or + &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + } +} + sub my_die { Index: /trunk/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 29492) +++ /trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 29493) @@ -3,4 +3,6 @@ magicRun.workdir, rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, rawExp.camera, -- convert magic_id into a boolean value (1 or 0) Index: /trunk/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /trunk/ippTools/share/magictool_toprocess_tree.sql (revision 29492) +++ /trunk/ippTools/share/magictool_toprocess_tree.sql (revision 29493) @@ -3,4 +3,6 @@ magicRun.workdir, rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, rawExp.camera, -- convert magic_id into a boolean value (1 or 0) @@ -16,4 +18,5 @@ WHERE magicRun.state = 'new' + AND (Label.active OR Label.active IS NULL) -- WHERE hook %s ORDER BY Index: /trunk/ippTools/share/magictool_totree.sql =================================================================== --- /trunk/ippTools/share/magictool_totree.sql (revision 29492) +++ /trunk/ippTools/share/magictool_totree.sql (revision 29493) @@ -18,2 +18,3 @@ AND magicTree.node IS NULL AND magicRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /trunk/ippTools/src/bgtool.c =================================================================== --- /trunk/ippTools/src/bgtool.c (revision 29492) +++ /trunk/ippTools/src/bgtool.c (revision 29493) @@ -658,5 +658,5 @@ } - if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script, + if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script, hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { @@ -1497,5 +1497,5 @@ } - if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script, + if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script, hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { Index: /trunk/ippTools/src/magictool.c =================================================================== --- /trunk/ippTools/src/magictool.c (revision 29492) +++ /trunk/ippTools/src/magictool.c (revision 29493) @@ -50,5 +50,5 @@ static bool exposureMode(pxConfig *config); -static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString); static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename); @@ -131,5 +131,5 @@ psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else - PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "=="); @@ -394,9 +394,20 @@ // required PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + + psString setString = NULL; + if (fault || clearfault) { + psStringAppend(&setString, ", fault = %d", fault); + } + if (note) { + psStringAppend(&setString, ", note = '%s'", note); + } if (state) { // set detRun.state to state - return setmagicRunState(config, magic_id, state); + return setmagicRunState(config, magic_id, state, setString); } @@ -612,5 +623,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0"); + psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0"); if (psListLength(where->list)) { @@ -1364,5 +1375,5 @@ } -static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -1380,7 +1391,13 @@ return false; } - - char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) { + psString query = NULL; + psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state); + if (setString) { + psStringAppend(&query, setString); + } + psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);; + +// char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_id); Index: /trunk/ippTools/src/magictoolConfig.c =================================================================== --- /trunk/ippTools/src/magictoolConfig.c (revision 29492) +++ /trunk/ippTools/src/magictoolConfig.c (revision 29493) @@ -79,5 +79,8 @@ psMetadata *updaterunArgs = psMetadataAlloc(); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", NULL); // -addinputskyfile