Index: /tags/ipp-20121026/dbconfig/changes.txt =================================================================== --- /tags/ipp-20121026/dbconfig/changes.txt (revision 34793) +++ /tags/ipp-20121026/dbconfig/changes.txt (revision 34794) @@ -2262,2 +2262,5 @@ UPDATE dbversion set schema_version = '1.1.72', updated= CURRENT_TIMESTAMP(); + +ALTER TABLE skycalResult ADD column n_forced INT after n_extended; +UPDATE dbversion set schema_version = '1.1.73', updated= CURRENT_TIMESTAMP(); Index: /tags/ipp-20121026/dbconfig/staticsky.md =================================================================== --- /tags/ipp-20121026/dbconfig/staticsky.md (revision 34793) +++ /tags/ipp-20121026/dbconfig/staticsky.md (revision 34794) @@ -56,4 +56,5 @@ n_detections S32 0 n_extended S32 0 + n_forced S32 0 zpt_obs F32 0.0 zpt_stdev F32 0.0 Index: /tags/ipp-20121026/ippScripts/scripts/skycalibration.pl =================================================================== --- /tags/ipp-20121026/ippScripts/scripts/skycalibration.pl (revision 34793) +++ /tags/ipp-20121026/ippScripts/scripts/skycalibration.pl (revision 34794) @@ -134,60 +134,63 @@ # First check the expected file name where the stack_id is the FILE_ID my $file; -if (!$singlefilter) { +# if (!$singlefilter) { +if (1) { + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id); if (! $ipprc->file_resolve($file)) { - # XXX: Beginning of section that can be removed eventually - - # no file with the expected name found - # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID - # but instead used FILE_ID = [0 .. num_filters-1] print "\nfailed to resolve $file\n"; - if (!$filter) { - &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); - } - print "Trying old style FILE_ID\n"; - my $max_filters = 5; - for (my $i=0; $i < $max_filters; $i++) { - my $file_id = sprintf "%03d", $i; - $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); - my $resolved = $ipprc->file_resolve($file); - if (!$resolved) { - # we fail here assumming that if file_id N doesn't exist, neither to N+1 - print "\nfailed to resolve $file\n\n"; - &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + # file with proper file rule not found. Try older systems + if ($singlefilter) { + # input is from a single filter static sky run which used psphot instead of psphotStack + # The file rule is different for psphot + $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($file); + } else { + # XXX: Beginning of section that can be removed eventually + # no file with the expected name found + # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID + # but instead used FILE_ID = [0 .. num_filters-1] + if (!$filter) { + &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); } - # Check the filter id for this file - my $command = "$fhead $resolved | grep FILTERID"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + print "Trying old style FILE_ID\n"; + my $max_filters = 5; + for (my $i=0; $i < $max_filters; $i++) { + my $file_id = sprintf "%03d", $i; + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); + my $resolved = $ipprc->file_resolve($file); + if (!$resolved) { + # we fail here assumming that if file_id N doesn't exist, neither to N+1 + print "\nfailed to resolve $file\n\n"; + &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Check the filter id for this file + my $command = "$fhead $resolved | grep FILTERID"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) + my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); + my $this_filter = substr $filt, 1, 7; + + # if it matches we're done + last if $this_filter eq $filter; + + # nope loop around to try the next one + print "Input file for $filter is not $file ($this_filter)\n"; + $file = undef; } - # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) - my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); - my $this_filter = substr $filt, 1, 7; - - # if it matches we're done - last if $this_filter eq $filter; - - # nope loop around to try the next one - print "Input file for $filter is not $file ($this_filter)\n"; - $file = undef; - } + } + # XXX: End of section that can be removed eventually + } + if ($file) { print "\nInput file for $stack_id filter: $filter: $file\n"; - - # XXX: End of section that can be removed eventually - } -} else { - # input is from a single filter static sky run use a different file rule - # XXX: can't we just make staticsky.pl use the same file rule? - $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); - &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) - unless $ipprc->file_exists($file); -} - -if (!$file) { - &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR); + } else { + &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR); + } } @@ -197,9 +200,11 @@ } -# read the input header to find the number of detections and the number of detections -# with extended model +# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET +# read the input header to find the number of detections +my $compatability_flags = ""; my $n_detections = 0; -my $n_extended = 0; { + # NSTARS is buggy. It only counts sources with a model, and counts them twice. + # We want the actual number of sources (real + matched). Use the length of the psf table my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -211,11 +216,6 @@ unless defined $n_detections; - $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - (undef, $n_extended) = split " ", join "", @$stdout_buf; - chomp $n_extended; - &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR) - unless defined $n_extended; + $compatability_flags .= " -n_detections $n_detections"; + } @@ -257,4 +257,7 @@ } + # if ppStats didn't get n_detections use the compatability version extracted above + $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/); + my $quality; ($quality) = $cmdflags =~ /-quality (\d+)/; @@ -276,6 +279,4 @@ { my $command = "$staticskytool -skycal_id $skycal_id"; - $command .= " -n_detections $n_detections"; - $command .= " -n_extended $n_extended"; $command .= " -addskycalresult -path_base $outroot"; $command .= " $cmdflags"; Index: /tags/ipp-20121026/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /tags/ipp-20121026/ippTools/share/pxadmin_create_tables.sql (revision 34793) +++ /tags/ipp-20121026/ippTools/share/pxadmin_create_tables.sql (revision 34794) @@ -2119,4 +2119,5 @@ n_detections INT, n_extended INT, + n_forced INT, zpt_obs FLOAT, zpt_stdev FLOAT, Index: /tags/ipp-20121026/ippTools/src/staticskytool.c =================================================================== --- /tags/ipp-20121026/ippTools/src/staticskytool.c (revision 34793) +++ /tags/ipp-20121026/ippTools/src/staticskytool.c (revision 34794) @@ -132,14 +132,4 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); -#ifdef notdef - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min", "skycell.glat", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max", "skycell.glat", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min", "skycell.radeg/15", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max", "skycell.radeg/15", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min", "skycell.radeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max", "skycell.radeg", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min", "skycell.decdeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max", "skycell.decdeg", "<="); -#endif if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -686,5 +676,5 @@ psStringAppend(&query, "\nGROUP BY sky_id"); if (num_filters) { - psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters); } @@ -1049,4 +1039,8 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, whereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -1128,4 +1122,5 @@ psString sky_label = psMetadataLookupStr(&status, row, "label"); psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString sky_dist_group = psMetadataLookupStr(&status, row, "dist_group"); // create a staticskyRun @@ -1138,5 +1133,5 @@ label ? label : sky_label, data_group ? data_group : sky_data_group, - dist_group, + dist_group ? dist_group : sky_dist_group, reduction, registered, @@ -1159,5 +1154,7 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -1276,6 +1273,7 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); - PXOPT_LOOKUP_S32(n_detections, config->args, "-n_detections", false, false); - PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_S32(n_detections, config->args,"-n_detections", false, false); + PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_S32(n_forced, config->args, "-n_forced", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); @@ -1318,4 +1316,5 @@ n_detections, n_extended, + n_forced, zpt_obs, zpt_stdev, Index: /tags/ipp-20121026/ippTools/src/staticskytoolConfig.c =================================================================== --- /tags/ipp-20121026/ippTools/src/staticskytoolConfig.c (revision 34793) +++ /tags/ipp-20121026/ippTools/src/staticskytoolConfig.c (revision 34794) @@ -126,5 +126,5 @@ psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); - psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0); pxskycellAddArguments(resultArgs); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -166,4 +166,5 @@ psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + pxskycellAddArguments(defineskycalrunArgs); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -181,6 +182,9 @@ psMetadata *updateskycalrunArgs = psMetadataAlloc(); psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky ID", 0); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxskycellAddArguments(updateskycalrunArgs); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -215,4 +219,5 @@ psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_detections", 0, "define number of detections", 0); psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended detections", 0); + psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_forced", 0, "define number of forced detections", 0); psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); Index: /tags/ipp-20121026/ippconfig/recipes/ppStats.config =================================================================== --- /tags/ipp-20121026/ippconfig/recipes/ppStats.config (revision 34793) +++ /tags/ipp-20121026/ippconfig/recipes/ppStats.config (revision 34794) @@ -127,4 +127,8 @@ HEADER STR IMAGE_V HEADER STR STREAK_V + + HEADER STR NDET # sources->n + HEADER STR NDET_EXT + HEADER STR NDET_FRC ANALYSIS MULTI # metadata blocks to search in chip/cell/readout->analysis Index: /tags/ipp-20121026/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /tags/ipp-20121026/ippconfig/recipes/ppStatsFromMetadata.config (revision 34793) +++ /tags/ipp-20121026/ippconfig/recipes/ppStatsFromMetadata.config (revision 34794) @@ -544,4 +544,7 @@ ENTRY VAL DT_ASTR F32 CONSTANT -dtime_astrom ENTRY VAL NASTRO S32 CONSTANT -n_astrom + ENTRY VAL NDET S32 CONSTANT -n_detections + ENTRY VAL NDET_EXT S32 CONSTANT -n_extended + ENTRY VAL NDET_FRC S32 CONSTANT -n_forced ENTRY VAL ZPT_OBS F32 CONSTANT -zpt_obs