Index: /trunk/dbconfig/background.md =================================================================== --- /trunk/dbconfig/background.md (revision 35153) +++ /trunk/dbconfig/background.md (revision 35154) @@ -5,4 +5,5 @@ chip_bg_id S64 0 chip_id S64 0 + cam_id S64 0 state STR 64 workdir STR 255 @@ -48,5 +49,4 @@ dist_group STR 64 reduction STR 64 - alt_cam_label STR 64 note STR 255 registered TAI NULL Index: /trunk/ippScripts/scripts/background_chip.pl =================================================================== --- /trunk/ippScripts/scripts/background_chip.pl (revision 35153) +++ /trunk/ippScripts/scripts/background_chip.pl (revision 35154) @@ -43,5 +43,5 @@ # Parse the command-line arguments my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, - $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked ); + $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $cam_path_base, $magicked ); GetOptions( 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier @@ -53,4 +53,5 @@ 'threads=s' => \$threads, # Number of threads to use 'chip_path_base=s' => \$chip_path_base, # optional chip_path_base + 'cam_path_base=s' => \$cam_path_base, # optional camera stage path_base 'magicked=s' => \$magicked, # magicked status of input 'verbose' => \$verbose, # Print to stdout @@ -94,4 +95,6 @@ my $dump_config; if ($chip_path_base and $no_update) { + # XXX: this path through the code has not been excercized in awhile and probably does not work + # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config # or do stats. Get path to input from command line. @@ -119,9 +122,15 @@ my $input = $$inputs[0]; # Input of interest $in_path = $input->{path_base}; + $cam_path_base = $input->{cam_path_base}; $magicked = $input->{magicked}; } my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id); -my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +my $in_mask; +if ($cam_path_base) { + $in_mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); +} else { + $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +} my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id); my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); @@ -177,4 +186,6 @@ my $apply_auxiliary_mask = 1; +# XXX Temporary lookup of auxiliary mask file from static location. +# At some point I'll convert this to register the masks in the database and use detselect if ($apply_auxiliary_mask) { my @auxmask_start_date = qw( @@ -238,5 +249,4 @@ print "Auxililary mask file is $auxmask\n"; } - @@ -272,5 +282,6 @@ &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); - my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP"; + my $resolved_stats = $ipprc->file_resolve($out_stats); + my $command = "$ppStatsFromMetadata $resolved_stats - BACKGROUND_CHIP"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /trunk/ippScripts/scripts/background_warp.pl =================================================================== --- /trunk/ippScripts/scripts/background_warp.pl (revision 35153) +++ /trunk/ippScripts/scripts/background_warp.pl (revision 35154) @@ -39,10 +39,9 @@ my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); -my ($astrometry, $imageName, $maskName, $weightName, $magicked, $exp_tag, $alternate_astrometry_label); +my ($astrometry, $imageName, $maskName, $weightName, $magicked); GetOptions( 'warp_bg_id|i=s' => \$warp_bg_id, # Warp identifier 'skycell_id|s=s' => \$skycell_id, # Skycell identifier 'tess_dir|s=s' => \$tess_dir, # Tesselation identifier - 'exp_tag|=s' => \$exp_tag, # exposure tag 'camera|c=s' => \$camera, # Camera name 'dbname|d=s' => \$dbname, # Database name @@ -52,5 +51,4 @@ 'mask-list=s' => \$maskName, # list of input mask files 'astrometry=s' => \$astrometry, - 'alt_cam_label=s' => \$alternate_astrometry_label, # camRun.label for alternate astrometry 'magicked=s' => \$magicked, 'threads=s' => \$threads, # Number of threads to use for pswarp @@ -86,9 +84,4 @@ } -if ($alternate_astrometry_label) { - my_die( "--exp_tag is required when using alternate astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR ) - unless defined $exp_tag; -} - my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -128,20 +121,4 @@ if (!defined $imageName) { # go find our inputs - my $astrom_path_base; - if (defined $alternate_astrometry_label) { - my $command = "$camtool -processedexp -exp_tag $exp_tag -label $alternate_astrometry_label"; - $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code); - } - my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR); - my $entry = $entries->[0]; - $astrom_path_base = $entry->{path_base}; - } - my ($imageFile, $maskFile, $weightFile); ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); @@ -176,13 +153,8 @@ $magicked = $file->{magicked}; - if (!$astrom_path_base) { - my $cam_path = $file->{cam_path_base}; - my $astrom = $ipprc->filename($astromSource, $cam_path); - &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; - $astrometry = $astrom; - } elsif (!$astrometry) { - # using alternate astrometry - $astrometry = $ipprc->filename($astromSource, $astrom_path_base); - } + my $cam_path = $file->{cam_path_base}; + my $astrom = $ipprc->filename($astromSource, $cam_path); + &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; + $astrometry = $astrom; } close $imageFile; Index: /trunk/ippTools/share/bgtool_chipinputs.sql =================================================================== --- /trunk/ippTools/share/bgtool_chipinputs.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_chipinputs.sql (revision 35154) @@ -1,7 +1,9 @@ SELECT - chipProcessedImfile.* + chipProcessedImfile.*, + camProcessedExp.path_base AS cam_path_base FROM chipBackgroundRun JOIN chipRun USING(chip_id) JOIN chipProcessedImfile USING(chip_id) +LEFT JOIN camProcessedExp ON chipBackgroundRun.cam_id = camProcessedExp.cam_id WHERE chipRun.state = 'full' AND chipProcessedImfile.fault = 0 Index: /trunk/ippTools/share/bgtool_definechip.sql =================================================================== --- /trunk/ippTools/share/bgtool_definechip.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_definechip.sql (revision 35154) @@ -1,6 +1,12 @@ SELECT - chipRun.* + chipRun.*, + camRun.cam_id, + chip_bg_id, + CURRENT_TIMESTAMP AS registered FROM chipRun JOIN rawExp USING(exp_id) -LEFT JOIN chipBackgroundRun USING(chip_id) -WHERE chipRun.state = 'full' +JOIN chipRun as altChipRun USING(exp_id) +JOIN camRun ON altChipRun.chip_id = camRun.chip_id +JOIN camProcessedExp USING(cam_id) +LEFT JOIN chipBackgroundRun ON chipBackgroundRun.chip_id = chipRun.chip_id -- labelHook %s +WHERE chipRun.state = 'full' AND camRun.state ='full' AND camProcessedExp.quality = 0 Index: /trunk/ippTools/share/bgtool_definewarp.sql =================================================================== --- /trunk/ippTools/share/bgtool_definewarp.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_definewarp.sql (revision 35154) @@ -1,12 +1,19 @@ SELECT - warpRun.*, - chipBackgroundRun.chip_bg_id + warpRun.warp_id, + chipBackgroundRun.* + -- the following items are selected for aid in debugging + , + warpsChipRun.chip_id as warpschip_id, + chipRun.chip_id AS thischip_id, + warpBackgroundRun.warp_bg_id FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN chipBackgroundRun USING(chip_id) +JOIN chipRun as warpsChipRun ON camRun.chip_id = warpsChipRun.chip_id JOIN rawExp USING(exp_id) -LEFT JOIN warpBackgroundRun USING(chip_bg_id) +JOIN chipRun USING(exp_id) +JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id + AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0) +LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s WHERE chipBackgroundRun.state = 'full' - AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed + AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated Index: /trunk/ippTools/share/bgtool_tochip.sql =================================================================== --- /trunk/ippTools/share/bgtool_tochip.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_tochip.sql (revision 35154) @@ -16,3 +16,3 @@ AND (Label.active OR Label.active IS NULL) -- WHERE hook %s -ORDER BY priority DESC, chip_bg_id +ORDER BY priority DESC, chip_bg_id, class_id Index: /trunk/ippTools/share/bgtool_warp.sql =================================================================== --- /trunk/ippTools/share/bgtool_warp.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_warp.sql (revision 35154) @@ -4,5 +4,4 @@ warpBackgroundRun.workdir, warpBackgroundRun.label, - warpBackgroundRun.alt_cam_label, rawExp.exp_id, rawExp.exp_name, Index: /trunk/ippTools/share/bgtool_warpinputs.sql =================================================================== --- /trunk/ippTools/share/bgtool_warpinputs.sql (revision 35153) +++ /trunk/ippTools/share/bgtool_warpinputs.sql (revision 35154) @@ -11,5 +11,5 @@ JOIN chipBackgroundImfile USING(chip_bg_id, class_id) JOIN fakeRun USING(fake_id) -JOIN camProcessedExp USING(cam_id) +JOIN camProcessedExp ON camProcessedExp.cam_id = fakeRun.cam_id WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed AND warpSkyfile.fault = 0 Index: /trunk/ippTools/src/bgtool.c =================================================================== --- /trunk/ippTools/src/bgtool.c (revision 35153) +++ /trunk/ippTools/src/bgtool.c (revision 35154) @@ -285,4 +285,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); @@ -324,5 +325,6 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); - pxAddLabelSearchArgs(config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-cam_label", "camRun.label", "=="); if (!psListLength(where->list)) { @@ -343,4 +345,10 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_STR(cam_label, config->args, "-cam_label", false, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + if (!cam_id && !cam_label) { + psError(PXTOOLS_ERR_CONFIG, true, "either cam_id or cam_label is required"); + return false; + } // Get chip runs to promote to chipBackgroundRun @@ -364,24 +372,11 @@ } + psString labelHook = psStringCopy(""); if (!rerun) { - psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL"); if (label) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.label = '%s'" - " OR chipBackgroundRun.label IS NULL)", - label); - } - if (data_group) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.data_group = '%s'" - " OR chipBackgroundRun.data_group IS NULL)", - data_group); - } - if (dist_group) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.dist_group = '%s'" - " OR chipBackgroundRun.dist_group IS NULL)", - dist_group); - } + // check for run with the newly specified label + psStringAppend(&labelHook, "\nAND (chipBackgroundRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND chip_bg_id IS NULL"); } @@ -391,7 +386,8 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { psError(psErrorCodeLast(), false, "database error"); psFree(query); + psFree(labelHook); if (!psDBRollback(config->dbh)) { psError(psErrorCodeLast(), false, "database error"); @@ -400,4 +396,5 @@ } psFree(query); + psFree(labelHook); psArray *output = p_psDBFetchResult(config->dbh); // Matching rows @@ -420,5 +417,5 @@ if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -435,5 +432,5 @@ psMetadata *md = output->data[i]; - chipRunRow *row = chipRunObjectFromMetadata(md); + chipBackgroundRunRow *row = chipBackgroundRunObjectFromMetadata(md); if (!row) { psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); @@ -445,5 +442,5 @@ } - if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new", + if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, row->cam_id, "new", workdir ? workdir : row->workdir, label ? label : row->label, @@ -560,5 +557,5 @@ if (psArrayLength(output)) { - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -621,5 +618,5 @@ return true; } - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundImfile", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1295,5 +1292,6 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); - pxAddLabelSearchArgs(config, where, "-chip_label", "chipBackgroundRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_bg_label", "chipBackgroundRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_label", "chipRun.label", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -1310,5 +1308,4 @@ PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(alt_cam_label, config->args, "-set_alt_cam_label", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -1336,24 +1333,11 @@ } + psString labelHook = psStringCopy(""); if (!rerun) { - psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL"); if (label) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.label = '%s'" - " OR warpBackgroundRun.label IS NULL)", - label); - } - if (data_group) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.data_group = '%s'" - " OR warpBackgroundRun.data_group IS NULL)", - data_group); - } - if (dist_group) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.dist_group = '%s'" - " OR warpBackgroundRun.dist_group IS NULL)", - dist_group); - } + // check for run with the newly specified label + psStringAppend(&labelHook, "\nAND (warpBackgroundRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND warp_bg_id IS NULL"); } @@ -1363,6 +1347,7 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(psErrorCodeLast(), false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(labelHook); psFree(query); if (!psDBRollback(config->dbh)) { @@ -1371,4 +1356,5 @@ return false; } + psFree(labelHook); psFree(query); @@ -1392,5 +1378,5 @@ if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1409,7 +1395,8 @@ psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id"); - warpRunRow *row = warpRunObjectFromMetadata(md); + + warpBackgroundRunRow *row = warpBackgroundRunObjectFromMetadata(md); if (!row) { - psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psError(psErrorCodeLast(), false, "failed to convert metadata into warpRun"); psFree(output); if (!psDBRollback(config->dbh)) { @@ -1425,5 +1412,4 @@ dist_group ? dist_group : row->dist_group, reduction ? reduction : row->reduction, - alt_cam_label, note ? note : row->note, NULL, 0)) { Index: /trunk/ippTools/src/bgtoolConfig.c =================================================================== --- /trunk/ippTools/src/bgtoolConfig.c (revision 35153) +++ /trunk/ippTools/src/bgtoolConfig.c (revision 35154) @@ -50,4 +50,5 @@ psMetadata *definechipArgs = psMetadataAlloc(); psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-cam_id", 0, "choose cam_id", 0); psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); @@ -89,5 +90,6 @@ psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", 0, "search on chipRun label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-cam_label", 0, "search on camRun label", NULL); psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false); @@ -279,5 +281,6 @@ psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL); - psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_bg_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false); @@ -285,5 +288,4 @@ psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); - psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_alt_cam_label", 0, "define alternate astrometry sources", NULL); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL);