Index: /branches/eam_branches/ipp-20110213/DataStore/scripts/dsgetfileset =================================================================== --- /branches/eam_branches/ipp-20110213/DataStore/scripts/dsgetfileset (revision 30674) +++ /branches/eam_branches/ipp-20110213/DataStore/scripts/dsgetfileset (revision 30675) @@ -113,4 +113,12 @@ exit $status; } + } elsif (($file_type =~ /gzip/) and ($outfile =~ /tgz/) ){ + $command = "tar -x --directory $outdir -f $outfile"; + $rc = system $command; + if ($rc) { + my $status = $rc >> 8; + print STDERR "failed to extract $outfile: rc: $rc status: $status\n"; + exit $status; + } } } @@ -158,5 +166,5 @@ =item * --unpack -Uncompress fits files if compressed. +Uncompress fits files if compressed or extract files from gzipped tar (tgz) files. =item * --first-file Index: /branches/eam_branches/ipp-20110213/ippTools/share/disttool_pending_raw.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/disttool_pending_raw.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/disttool_pending_raw.sql (revision 30675) @@ -12,5 +12,6 @@ TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, CAST(NULL AS CHAR(255)) AS alt_path_base, - chipProcessedImfile.path_base as chip_path_base, + -- pass camera stage path base since we want the camera mask file. The script knows what to do + camProcessedExp.path_base as chip_path_base, CAST(NULL AS CHAR(255)) AS state, CAST(NULL AS CHAR(255)) AS data_state, @@ -26,4 +27,6 @@ JOIN chipProcessedImfile USING(chip_id, class_id) +JOIN camRun USING(chip_id) +JOIN camProcessedExp using(cam_id) LEFT JOIN distComponent ON distRun.dist_id = distComponent.dist_id @@ -36,4 +39,7 @@ AND distComponent.dist_id IS NULL AND (rawExp.magicked OR distRun.no_magic) + -- need to have magicked the chip image which makes the camera mask + AND chipProcessedImfile.magicked != 0 + AND camRun.magicked > 0 AND (Label.active OR Label.active IS NULL) UNION @@ -52,5 +58,6 @@ TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, magicDSFile.backup_path_base AS alt_path_base, - chipProcessedImfile.path_base AS chip_path_base, + -- pass camera stage path base since we want the camera mask file. The script knows what to do + camProcessedExp.path_base AS chip_path_base, CAST('full' AS CHAR(255)) AS state, CAST('full' AS CHAR(255)) AS data_state, @@ -77,6 +84,6 @@ AND distRun.clean = 0 AND distComponent.dist_id IS NULL - AND chipProcessedImfile.data_state = 'full' - AND chipProcessedImfile.magicked > 0 + AND chipProcessedImfile.magicked != 0 + AND camRun.magicked > 0 AND (Label.active OR Label.active IS NULL) UNION Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_camera.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_camera.sql (revision 30675) @@ -11,4 +11,5 @@ camProcessedExp.path_base, camProcessedExp.path_base AS cam_path_base, + recovery_path_base, 0 AS bothways, 0 AS bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_chip.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_chip.sql (revision 30675) @@ -11,4 +11,5 @@ chipProcessedImfile.path_base, camProcessedExp.path_base AS cam_path_base, + recovery_path_base, 0 AS bothways, 0 AS bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql (revision 30675) @@ -13,4 +13,5 @@ magicDSFile.recovery_path_base, "NULL" AS cam_path_base, + recovery_path_base, CAST(diffRun.bothways AS SIGNED) AS bothways, 0 AS bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_raw.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_raw.sql (revision 30675) @@ -11,4 +11,5 @@ rawImfile.uri AS path_base, CAST(NULL AS CHAR(255)) AS cam_path_base, + recovery_path_base, 0 AS bothways, bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_warp.sql (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_warp.sql (revision 30675) @@ -11,4 +11,5 @@ warpSkyfile.path_base, "NULL" AS cam_path_base, + recovery_path_base, 0 AS bothways, 0 AS bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 30675) @@ -709,6 +709,9 @@ +// update the magicked column for the underlying component +// Note: Must be called inside a transaction + static bool -setMagicked(pxConfig *config, psS64 magic_ds_id, psString component) +setMagicked(pxConfig *config, psS64 magic_ds_id, psString component, bool clearMagicked) { // first query the magicDSRun to find the stage and the stage_id @@ -741,47 +744,64 @@ psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psFree(output); + + psS64 newMagickedValue; + if (clearMagicked) { + newMagickedValue = 0; + } else { + newMagickedValue = magic_id; + } ippStage stageNum = ippStringToStage(stage); // chose the appropriate query based on the stage + char *clearRunQuery = NULL; switch (stageNum) { case IPP_STAGE_RAW: query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; + clearRunQuery = "UPDATE rawExp SET magicked = 0 where exp_id = %" PRId64; break; case IPP_STAGE_CHIP: query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; + clearRunQuery = "UPDATE chipRun set magicked = 0 where chip_id = %" PRId64; break; case IPP_STAGE_CHIP_BG: query = "UPDATE chipBackgroundImfile SET magicked = %" PRId64 " where chip_bg_id = %" PRId64 " AND class_id = '%s'"; + clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64; break; case IPP_STAGE_CAMERA: - // no there is no magicked column in camProcessedExp so we have nothing to do - psFree(output); - return true; + psFree(query); + query = NULL; + clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64; + break; case IPP_STAGE_WARP: query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; + clearRunQuery = "UPDATE warpRun SET magicked = 0 where warp_id = %" PRId64; break; case IPP_STAGE_WARP_BG: query = "UPDATE warpBackgroundSkyfile SET magicked = %" PRId64 " where warp_bg_id = %" PRId64 " AND skycell_id = '%s'"; + clearRunQuery = "UPDATE warpBackgroundRun SET magicked = 0 where warp_bg_id = %" PRId64; break; case IPP_STAGE_DIFF: query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; + clearRunQuery = "UPDATE diffRun SET magicked = 0 where diff_id = %" PRId64; break; default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); - psFree(output); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(output); - - psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); - return false; + return false; + } + + if (query) { + if (!p_psDBRunQueryF(config->dbh, query, newMagickedValue, stage_id, component)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + if (clearMagicked && clearRunQuery) { + if (!p_psDBRunQueryF(config->dbh, clearRunQuery, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } } @@ -896,5 +916,5 @@ if (setmagicked) { // set the image file's magicked flag - if (!setMagicked(config, magic_ds_id, component)) { + if (!setMagicked(config, magic_ds_id, component, false)) { psError(PS_ERR_UNKNOWN, false, "setMagicked failed"); if (!psDBRollback(config->dbh)) { @@ -1111,4 +1131,6 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -1116,6 +1138,7 @@ psString queryFile = NULL; bool stateIsUpdate = false; + bool toRestored = !strcmp(state, "goto_restored"); if (state) { - if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) { + if (! strcmp(state, "new") || toRestored) { queryFile = "magicdstool_revertdestreakedfile.sql"; } else if (!strcmp(state, "update")) { @@ -1153,4 +1176,8 @@ } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); @@ -1159,4 +1186,18 @@ } psFree(query); + if (toRestored) { + // clear the underlying component's magicked value + if (!setMagicked(config, magic_ds_id, component, true)) { + psError(PS_ERR_UNKNOWN, false, "setMagicked failed"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } @@ -1661,8 +1702,7 @@ } if (!strcmp(data_state, "full")) { - // if -tofullfile optoinally set the magicked value for the component - PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); - // set the image file's magicked flag - if (!setMagicked(config, magic_ds_id, component)) { + // set the component's magicked flag + + if (!setMagicked(config, magic_ds_id, component, false)) { psError(PS_ERR_UNKNOWN, false, "setMagicked failed"); if (!psDBRollback(config->dbh)) { @@ -1716,14 +1756,4 @@ return change_file_data_state(config, "full"); } -/* -static bool topurgedimfileMode(pxConfig *config) -{ - return change_imfile_data_state(config, "purged", "goto_purged"); -} -static bool toscrubbedfileMode(pxConfig *config) -{ - return change_file_data_state(config, "scrubbed", "goto_scrubbed"); -} -*/ // a very specfic function to queue a cleaned magicDSFile to be updated Index: /branches/eam_branches/ipp-20110213/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/regtool.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/src/regtool.c (revision 30675) @@ -259,4 +259,13 @@ continue; } + char *is_ccim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'c'); + if (is_ccim) { // Is a camera commanded exposure + // fprintf(stderr,"IN camera commanded!\n"); + if ((strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DOMEFLAT") == 0)|| + (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DARK") == 0) || + (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"BIAS") == 0)) { + continue; + } + } if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| @@ -367,4 +376,14 @@ } + char *is_ccim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'c'); + if (is_ccim) { // Is a camera commanded exposure + // fprintf(stderr,"IN camera commanded!\n"); + if ((strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DOMEFLAT") == 0)|| + (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DARK") == 0) || + (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"BIAS") == 0)) { + continue; + } + } + bool status = false; char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id"); @@ -389,4 +408,5 @@ } + if (0 && !strcmp(this_class_id, "ota44")) { printf("STAT 1: %s (%d %d) %d %d %d\n", @@ -857,5 +877,6 @@ PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); - + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) { psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); @@ -905,5 +926,10 @@ psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); } - + if (hostname) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.hostname = '%s'", hostname); + } psString query = pxDataGet("regtool_updateprocessedimfile.sql"); if (!query) { Index: /branches/eam_branches/ipp-20110213/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/regtoolConfig.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ippTools/src/regtoolConfig.c (revision 30675) @@ -189,4 +189,5 @@ ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", INT16_MAX); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); + ADD_OPT(Str, updateprocessedimfileArgs, "-hostname", "set host name", NULL); ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes", "set bytes", INT32_MAX); ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); Index: /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMonet.c =================================================================== --- /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMonet.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMonet.c (revision 30675) @@ -146,5 +146,5 @@ psU32 flags = psMetadataLookupU32(NULL, row, "FLAGS"); - float width = 0.5 * (major + minor); + float width = (float) (0.5 * (major + minor)); gzprintf(gz, "%s , %.2f , %.2f , %.8lf , %.8lf , %.3f , %.3f , %.2f , %#08x\n", Index: /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsDetections.c =================================================================== --- /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsDetections.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsDetections.c (revision 30675) @@ -183,6 +183,6 @@ det->apMagRadius = psVectorRealloc(det->apMagRadius, num); det->apMagRaw = psVectorRealloc(det->apMagRadius, num); - det->apFlux = psVectorRealloc(det->apMagRadius, num); - det->apFluxSig = psVectorRealloc(det->apMagRadius, num); + det->apFlux = psVectorRealloc(det->apFlux, num); + det->apFluxSig = psVectorRealloc(det->apFluxSig, num); det->peakFluxAsMag = psVectorRealloc(det->peakFluxAsMag, num); det->calPsfMag = psVectorRealloc(det->calPsfMag, num); Index: /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsMerge.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsMerge.c (revision 30675) @@ -22,7 +22,7 @@ ) { - float dx = detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0; - float dy = detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0; - return PS_SQR(dx) + PS_SQR(dy); + float dx = (float) (detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0); + float dy = (float) (detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0); + return PS_SQR(dx) + PS_SQR(dy); } @@ -165,10 +165,10 @@ if (num == 0) { //All detections were NULL?! - psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n); + psTrace("ppMops.merge", 3, "All %ld detections were NULL\n", detections->n); return NULL; } psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num); - merged->seeing /= num; + merged->seeing /= (float) num; return merged; Index: /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsRead.c =================================================================== --- /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsRead.c (revision 30674) +++ /branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsRead.c (revision 30675) @@ -45,5 +45,5 @@ if (args->version == 0) { psTrace("ppMops.read", 1, "Changing args->version to %d\n", skyChipPsfVersion); - args->version = skyChipPsfVersion; + args->version = (unsigned short) skyChipPsfVersion; } if (skyChipPsfVersion == 0) { @@ -87,6 +87,6 @@ det->mjd = psMetadataLookupF64(NULL, header, "MJD-OBS") + det->exptime / 2.0 / 3600 / 24; - det->seeing = 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") + - psMetadataLookupF32(NULL, header, "FWHM_MIN")); + det->seeing = (float) 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") + + psMetadataLookupF32(NULL, header, "FWHM_MIN")); int naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns @@ -145,16 +145,16 @@ //Values are set only if the version is 2 if (skyChipPsfVersion == 2) { - det->psfInstFlux->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PSF_INST_FLUX"); - det->psfInstFluxSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PSF_INST_FLUX_SIG"); - det->apMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG"); - det->apMagRaw->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG_RAW"); - det->apMagRadius->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG_RADIUS"); - det->apFlux->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_FLUX"); - det->apFluxSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_FLUX_SIG"); - det->peakFluxAsMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PEAK_FLUX_AS_MAG"); - det->calPsfMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "CAL_PSF_MAG"); - det->calPsfMagSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "CAL_PSF_MAG_SIG"); - det->sky->data.F32[numGood] = psMetadataLookupS32(NULL, row, "SKY"); - det->skySig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "SKY_SIGMA"); + det->psfInstFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX"); + det->psfInstFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX_SIG"); + det->apMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG"); + det->apMagRaw->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RAW"); + det->apMagRadius->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RADIUS"); + det->apFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX"); + det->apFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX_SIG"); + det->peakFluxAsMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PEAK_FLUX_AS_MAG"); + det->calPsfMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG"); + det->calPsfMagSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG_SIG"); + det->sky->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY"); + det->skySig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY_SIGMA"); det->qualityPerfect->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_QF_PERFECT"); det->momentsR1->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_R1"); @@ -236,5 +236,5 @@ numGood++; } - det->seeing *= plateScale / numGood; + det->seeing *= ((float) plateScale) / ((float) numGood); det->x->n = numGood; Index: /branches/eam_branches/ipp-20110213/tools/examine_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/ipp-20110213/tools/examine_burntool_pcontrol.pl (revision 30674) +++ /branches/eam_branches/ipp-20110213/tools/examine_burntool_pcontrol.pl (revision 30675) @@ -145,5 +145,5 @@ # Get the data. The limit is there to keep the database happy. I don't think you can observe that many # images on a single night. - $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 600000"; + $sth = "SELECT exp_id,rawImfile.exp_name,rawImfile.obs_mode,rawImfile.dateobs,class_id,burntool_state,rawImfile.comment,rawExp.state FROM rawImfile JOIN rawExp USING(exp_id) WHERE rawImfile.dateobs >= '${dmin}' AND rawImfile.dateobs <= '${dmax}' order by rawImfile.dateobs limit 600000"; $data_ref = $db->selectall_arrayref( $sth ); @@ -168,5 +168,5 @@ } print "# 0 1 2 3 4 5 6 7 \n"; - print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n"; + print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 rawExp.state comment\n"; # If the database returns no entries (because we have a mistake in pcontrol.pro perhaps), return a null entry. @@ -180,5 +180,5 @@ foreach $row_ref (@{ $data_ref }) { - ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref }; + ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment,$exp_state) = @{ $row_ref }; # Correct for nulls in the database. @@ -202,6 +202,6 @@ if ($cur_exp_id != -99) { - printf("%6d %11s %11s %19s %2d %64s %s\n", - $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment); + printf("%6d %11s %11s %19s %2d %64s %15s %s\n", + $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_exp_state,$cur_comment); if (($burncount != 60)&&(exists($opt{A}))) { @@ -228,12 +228,13 @@ } ($cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$cur_comment) = ($exp_id,$exp_name,$obs_mode,$dateobs,$comment); + $cur_exp_state = $exp_state; class_to_vector($class_id,$burntool_state); } } # Final entry needs a manual print to clear it out. Let's tack on a footer as well. - printf("%6d %11s %11s %19s %2d %64s %s\n", - $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment); + printf("%6d %11s %11s %19s %2d %64s %15s %s\n", + $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_exp_state,$cur_comment); print "# 0 1 2 3 4 5 6 7 \n"; - print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n"; + print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 rawExp.state comment\n"; print"\n"; Index: /branches/eam_branches/ipp-20110213/tools/log_parsing/nebulous_activity.py =================================================================== --- /branches/eam_branches/ipp-20110213/tools/log_parsing/nebulous_activity.py (revision 30675) +++ /branches/eam_branches/ipp-20110213/tools/log_parsing/nebulous_activity.py (revision 30675) @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +# +# Plot the apache error activity versus the nebulous activity +# + +import sys +import os +import gzip +import parse_apache_log + +def getFilenamesByPrefix(directoryName, prefix): + """ + Get a list of file names in directory starting by prefix + """ + filenames = [] + for entry in os.listdir(directoryName): + fullpath = directoryName + os.sep + entry + if entry.startswith(prefix): + if not os.path.isdir(fullpath): + filenames.append(fullpath) + return filenames + +def processAccessFile(filename): + """ + Process a Nebulous apache access_log file (that can be gzipped or not) + """ + isGZipFile = False + try: + inFile = gzip.open(filename) + inFile.read(1) + inFile.close() + isGZipFile = True + except IOError: + pass + if isGZipFile: + inFile = gzip.open(filename) + else: + inFile = open(filename) + sys.stderr.write('File %s is %sa gzip file\n' % (filename, + '' if isGZipFile else 'not ')) + counts = parse_apache_log.process(inFile) + #parse_apache_log.show_results(counts, sys.stdout) + return counts + +def processErrorFile(filename): + """ + Process a Nebulous apache error_log file (that can be gzipped or not) + """ + isGZipFile = False + try: + inFile = gzip.open(filename) + inFile.read(1) + inFile.close() + isGZipFile = True + except IOError: + pass + if isGZipFile: + inFile = gzip.open(filename) + else: + inFile = open(filename) + sys.stderr.write('File %s is %sa gzip file\n' % (filename, + '' if isGZipFile else 'not ')) + counts = parse_apache_log.processErrorLog(inFile) + #parse_apache_log.show_errors(counts, sys.stdout) + return counts + +if __name__ == '__main__': + """ + Usage: python sys.argv[0] + + Application log is written to /dev/stderr + Results are written to /dev/stdout + """ + # All logs are apache2 logs and can be found in one directory + # which happens to be the first script argument + logDirectoryName = sys.argv[1] + + sys.stderr.write('### Building access log filenames list\n') + accessFilenames = getFilenamesByPrefix(logDirectoryName, + 'access_log') + sys.stderr.write(' %d access_log files\n' % (len(accessFilenames))) + for accessFilename in accessFilenames: + sys.stderr.write('\tFilename: %s\n' % accessFilename) + sys.stderr.write('### Building error log filenames list\n') + errorFilenames = getFilenamesByPrefix(logDirectoryName, + 'error_log') + sys.stderr.write(' %d error_log files\n' % (len(errorFilenames))) + for errorFilename in errorFilenames: + sys.stderr.write('\tFilename: %s\n' % errorFilename) + + accessCounts = dict() + for accessFilename in accessFilenames: + accessCounts.update(processAccessFile(accessFilename)) + errorCounts = dict() + for errorFilename in errorFilenames: + errorCounts.update(processErrorFile(errorFilename)) + + # Now show results + keys = set() + keys.update(accessCounts.keys()) + keys.update(errorCounts.keys()) + for key in keys: + try: + nebulousAccessCount = 0 + for index in range(5): + nebulousAccessCount += accessCounts[key][index] + createCount = accessCounts[key][0] + findInsCount = accessCounts[key][1] + deleteCount = accessCounts[key][2] + failStatCount = accessCounts[key][3] + statCount = accessCounts[key][4] + otherCount = accessCounts[key][5] + unknownCount = accessCounts[key][6] + except KeyError: + nebulousAccessCount = -1 + createCount = -1 + findInsCount = -1 + deleteCount = -1 + failStatCount = -1 + statCount = -1 + otherCount = -1 + unknownCount = -1 + try: + errorCount = errorCounts[key] + except KeyError: + errorCount = -1 + sys.stdout.write('%s %d %d %d %d %d %d %d %d %d\n' % (key, nebulousAccessCount, errorCount, + createCount, findInsCount, deleteCount, + failStatCount, statCount, otherCount, + unknownCount)) Index: /branches/eam_branches/ipp-20110213/tools/log_parsing/parse_apache_log.py =================================================================== --- /branches/eam_branches/ipp-20110213/tools/log_parsing/parse_apache_log.py (revision 30674) +++ /branches/eam_branches/ipp-20110213/tools/log_parsing/parse_apache_log.py (revision 30675) @@ -4,10 +4,16 @@ import re import datetime +import fileinput nebulousPattern = re.compile('POST /nebulous HTTP/1.1') +errorPattern = re.compile('exit signal Segmentation fault') + # Resolution: 10 minutes of 60 seconds of 1000000 microseconds datetime.resolution = 10*60*1000000 def message(size): + """ + Guess the message type according to the message size + """ if size == 488: # delete == 488 @@ -34,4 +40,7 @@ def show_results(counts, output=sys.stderr): + """ + Display results + """ output.write('# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb\n') for count in sorted(counts.iterkeys()): @@ -45,12 +54,18 @@ counts[count][6])) -if __name__ == '__main__': +def process(linesInFile = None): + """ + Process Nebulous entries in apache2 log file + """ counts = dict() - linesCount = 1 - for line in sys.stdin: - if linesCount % 10000 == 0: - sys.stderr.write('Analyzed lines: %d\n' % linesCount) - if linesCount % 100000 == 0: + if linesInFile == None: + input = fileinput.input() + else: + input = linesInFile + for line in input: + if linesCount % 100000 == 0: + sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount) + if linesCount % 500000 == 0: show_results(counts) linesCount += 1 @@ -85,4 +100,85 @@ counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0] counts[dt.isoformat()][6] += 1 + return counts - show_results(counts, sys.stdout) +def show_errors(counts, output=sys.stderr): + """ + Display error results + """ + output.write('# Epoch | Errors\n') + for count in sorted(counts.iterkeys()): + output.write('%19s | %6d\n' % (count, counts[count])) + +def processErrorLog(linesInFile = None): + """ + Process errors in apache2 error log file + """ + counts = dict() + linesCount = 1 + if linesInFile == None: + input = fileinput.input() + else: + input = linesInFile + for line in input: + if linesCount % 100000 == 0: + sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount) + if linesCount % 500000 == 0: + show_errors(counts) + linesCount += 1 + if errorPattern.search(line): + elements = line.split(']') + date = elements[0] + date = date[1:] + # print '[%s]' % (date) + dt = datetime.datetime.strptime(date, + '%a %b %d %H:%M:%S %Y') + dt = dt.replace(minute = int(dt.minute/10)*10, + second = 0) + # print dt.isoformat() + try: + counts[dt.isoformat()] += 1 + except KeyError: + counts[dt.isoformat()] = 1 + return counts + +def usage(): + sys.stderr.write(main.__doc__) + +def main(arguments): + """ +Usage: parse_apache_log [-a ] [-e ] +""" + mainName = arguments.pop() + if len(arguments) == 0: + sys.stderr.write('Reading from /dev/stdin\n') + counts = process() + show_results(counts, sys.stdout) + elif len(arguments) % 2 != 0: + sys.stderr.write('Arguments length: %d\n' % (len(arguments))) + usage() + sys.exit(1) + else: + accessFilenames = [] + errorFilenames = [] + while len(arguments) != 0: + flag = arguments.pop() + filename = arguments.pop() + if flag == '-a': + accessFilenames.append(filename) + elif flag == '-e': + errorFilenames.append(filename) + else: + sys.stderr.write('Unknown flag: [%s]' % (flag)) + usage() + sys.exit(1) + for accessFilename in accessFilenames: + sys.stderr.write('Processing: [%s]' % (accessFilename)) + inFile = open(accessFilename) + counts = process(inFile) + show_results(counts, sys.stdout) + for errorFilename in errorFilenames: + counts = processErrorLog(errorFilename) + show_errors(counts, sys.stdout) + +if __name__ == '__main__': + main(sys.argv) Index: /branches/eam_branches/ipp-20110213/tools/regpeek.pl =================================================================== --- /branches/eam_branches/ipp-20110213/tools/regpeek.pl (revision 30675) +++ /branches/eam_branches/ipp-20110213/tools/regpeek.pl (revision 30675) @@ -0,0 +1,73 @@ +#! /usr/bin/perl -w + +use DBI; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +my $dbname = 'gpc1'; + +GetOptions( + 'logs|L' => \$logs, + 'dbname=s' => \$dbname, + ); + +my $date = shift(@ARGV); +unless(defined($date)) { + my @time = localtime; + $date = sprintf("%4d-%02d-%02d",$time[5] + 1900,$time[4] + 1,$time[3]); +} +print "Date: $date\n"; +chomp(my $bad_exp = `regtool -checkstatus -date $date -class_id ota33 -dbname gpc1 -simple | grep 'stop run' | head -1`); +my @row = split /\s+/, $bad_exp; +if ($#row == 18) { + print " exp_name uri registered Nimfile summit_fault download_state newExp_state rawExp_state imfile_state exp_id exp_type dateobs obs_mode object burntool_state class_id summit_class_id is_downloaded is_registered\n"; +} +elsif ($#row == -1) { + print "Everything looks good.\n"; + exit(0); +} +else { + print " exp_name registered Nimfile summit_fault download_state newExp_state exp_id exp_type obs_mode burntool_state summit_class_id is_downloaded is_registered\n"; +} +print "CheckStatus: $bad_exp\n"; +$bad_exp = (split /\s+/, $bad_exp)[0]; +print "Exposure: $bad_exp is the current pending exposure.\n"; + + +my $db = init_gpc_db(); + +my $query = "select exp_name,class_id,burntool_state,data_state,uri,exp_id FROM rawImfile where exp_name = '$bad_exp'"; + +my $data = $db->selectall_arrayref( $query ); +print "#exp_name class_id bt_state data_state uri\n"; +foreach my $row (@{ $data }) { + my ($exp_name,$class_id,$bt_state,$data_state,$uri,$exp_id) = @{ $row }; + print "$exp_name $class_id $bt_state $data_state $uri\n"; + if ($logs) { + $reg_log = $uri; + $reg_log =~ s/$exp_name/${exp_name}.${exp_id}/g; + $reg_log =~ s/ota/reg.ota/; + $reg_log =~ s/fits/log/; + $burn_log = $uri; + $burn_log =~ s/fits/burn.log/; + + print " REG_LOG: $reg_log\n"; + print " BURN_LOG: $burn_log\n"; + } +} + + + + +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, +{ RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; +return($db); +} +