Index: /branches/ipp-148_http_503_filesset_revert_fix/Makefile.am =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/Makefile.am (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/Makefile.am (revision 41217) @@ -0,0 +1,542 @@ +pxadmin_create_mirror_tables.sql: pxadmin_create_tables.sql + -$(RM) pxadmin_create_mirror_tables.sql + ./parse_pxadmin_create_tables.pl pxadmin_create_tables.sql > pxadmin_create_mirror_tables.sql + +dist_pkgdata_DATA = \ + addtool_checkminidvodbaddrun.sql \ + addtool_donecleanup.sql \ + addtool_find_cam_id_dvo.sql \ + addtool_find_cam_id.sql \ + addtool_find_diff_id.sql \ + addtool_find_diff_id_dvo.sql \ + addtool_find_ff_id.sql \ + addtool_find_ff_id_dvo.sql \ + addtool_find_ffsummary_id.sql \ + addtool_find_ffsummary_id_dvo.sql \ + addtool_find_skycal_id_dvo.sql \ + addtool_find_skycal_id.sql \ + addtool_find_sky_id_multi_dvo.sql \ + addtool_find_sky_id_multi.sql \ + addtool_find_stack_id_dvo.sql \ + addtool_find_stack_id.sql \ + addtool_find_pendingexp.sql \ + addtool_find_pendingexp_cam.sql \ + addtool_find_pendingexp_diff.sql \ + addtool_find_pendingexp_ff.sql \ + addtool_find_pendingexp_ffsummary.sql \ + addtool_find_pendingexp_skycal.sql \ + addtool_find_pendingexp_stack.sql \ + addtool_find_pendingexp_staticsky_multi.sql \ + addtool_find_pendingmergeprocess.sql \ + addtool_find_processedexp_cam.sql \ + addtool_find_processedexp_diff.sql \ + addtool_find_processedexp_ff.sql \ + addtool_find_processedexp_ffsummary.sql \ + addtool_find_processedexp_skycal.sql \ + addtool_find_processedexp_stack.sql \ + addtool_find_processedexp_staticsky_multi.sql \ + addtool_find_minidvodbprocessed.sql \ + addtool_find_minidvodbrun.sql \ + addtool_pendingcleanupexp.sql \ + addtool_pendingcleanuprun.sql \ + addtool_queue_cam_id.sql \ + addtool_queue_diff_id.sql \ + addtool_queue_ff_id.sql \ + addtool_queue_ffsummary.sql \ + addtool_queue_stack_id.sql \ + addtool_queue_sky_id_multi.sql \ + addtool_queue_skycal_id.sql \ + addtool_queue_minidvodbrun.sql \ + addtool_revertminidvodbprocessed.sql \ + addtool_revertprocessedexp_cam.sql \ + addtool_revertprocessedexp_diff.sql \ + addtool_revertprocessedexp_ff.sql \ + addtool_revertprocessedexp_ffsummary.sql \ + addtool_revertprocessedexp_skycal.sql \ + addtool_revertprocessedexp_stack.sql \ + addtool_revertprocessedexp_staticsky.sql \ + addtool_revertprocessedexp_staticsky_multi.sql \ + bgtool_advancechip.sql \ + bgtool_advancewarp.sql \ + bgtool_change_chiprun_state.sql \ + bgtool_change_imfile_data_state.sql \ + bgtool_change_warprun_state.sql \ + bgtool_change_skyfile_data_state.sql \ + bgtool_chipinputs.sql \ + bgtool_chip.sql \ + bgtool_cleanedchip.sql \ + bgtool_cleanedwarp.sql \ + bgtool_definechip.sql \ + bgtool_definewarp.sql \ + bgtool_listchip.sql \ + bgtool_listwarp.sql \ + bgtool_pendingcleanupchiprun.sql \ + bgtool_pendingcleanupchipimfile.sql \ + bgtool_pendingcleanupwarprun.sql \ + bgtool_pendingcleanupwarpskyfile.sql \ + bgtool_revertchip.sql \ + bgtool_revertwarp.sql \ + bgtool_tochip.sql \ + bgtool_tocleanchip.sql \ + bgtool_tocleanwarp.sql \ + bgtool_towarp.sql \ + bgtool_warpinputs.sql \ + bgtool_warp.sql \ + camtool_addprocessedexp.sql \ + camtool_donecleanup.sql \ + camtool_find_chip_id.sql \ + camtool_pendingexp.sql \ + camtool_find_pendingimfile.sql \ + camtool_find_processedexp.sql \ + camtool_pendingcleanupexp.sql \ + camtool_pendingcleanuprun.sql \ + camtool_queue_chip_id.sql \ + camtool_revertprocessedexp.sql \ + camtool_revertupdatedexp.sql \ + camtool_export_run.sql \ + camtool_export_processed_exp.sql \ + chiptool_change_exp_state.sql \ + chiptool_change_imfile_data_state.sql \ + chiptool_completely_processed_exp.sql \ + chiptool_coalesce_run.sql \ + chiptool_definecopy.sql \ + chiptool_donecleanup.sql \ + chiptool_dropprocessedimfile_update.sql \ + chiptool_dropprocessedimfile_delete.sql \ + chiptool_find_rawexp.sql \ + chiptool_listrun.sql \ + chiptool_pendingcleanupimfile.sql \ + chiptool_pendingcleanupimfile_all.sql \ + chiptool_pendingcleanuprun.sql \ + chiptool_pendingimfile.sql \ + chiptool_processedimfile.sql \ + chiptool_revertcleanup.sql \ + chiptool_revertprocessedimfile.sql \ + chiptool_revertupdatedimfile.sql \ + chiptool_run.sql \ + chiptool_runstate.sql \ + chiptool_setimfiletoupdate.sql \ + chiptool_export_run.sql \ + chiptool_export_imfile.sql \ + chiptool_export_processed_imfile.sql \ + chiptool_unmasked.sql \ + detselect_search.sql \ + detselect_select.sql \ + detselect_show.sql \ + dettool_addprocessedexp.sql \ + dettool_childlessrun.sql \ + dettool_definebydetrun.sql \ + dettool_detrunsummary.sql \ + dettool_find_completed_runs.sql \ + dettool_input.sql \ + dettool_normalizedexp.sql \ + dettool_normalizedimfile.sql \ + dettool_normalizedstat.sql \ + dettool_pending.sql \ + dettool_pendingcleanup_detrunsummary.sql \ + dettool_pendingcleanup_normalizedexp.sql \ + dettool_pendingcleanup_normalizedimfile.sql \ + dettool_pendingcleanup_normalizedstat.sql \ + dettool_pendingcleanup_processedexp.sql \ + dettool_pendingcleanup_processedimfile.sql \ + dettool_pendingcleanup_residexp.sql \ + dettool_pendingcleanup_residimfile.sql \ + dettool_pendingcleanup_stacked.sql \ + dettool_processedimfile.sql \ + dettool_raw.sql \ + dettool_residexp.sql \ + dettool_residimfile.sql \ + dettool_revertdetrunsummary.sql \ + dettool_revertnormalizedexp.sql \ + dettool_revertnormalizedimfile.sql \ + dettool_revertnormalizedstat.sql \ + dettool_revertprocessedexp.sql \ + dettool_revertprocessedimfile.sql \ + dettool_revertresidexp.sql \ + dettool_revertresidimfile.sql \ + dettool_revertstacked.sql \ + dettool_runs.sql \ + dettool_stacked.sql \ + dettool_start_new_iteration.sql \ + dettool_stop_completed_correct_runs.sql \ + dettool_tocorrectexp.sql \ + dettool_tocorrectimfile.sql \ + dettool_todetrunsummary.sql \ + dettool_tonormalize.sql \ + dettool_tonormalizedexp.sql \ + dettool_tonormalizedstat.sql \ + dettool_toprocessedexp.sql \ + dettool_toprocessedimfile.sql \ + dettool_toresidexp.sql \ + dettool_toresidimfile.sql \ + dettool_tostacked.sql \ + difftool_change_skyfile_data_state.sql \ + difftool_change_run_state.sql \ + difftool_completed_runs.sql \ + difftool_coalesce_run.sql \ + difftool_definewarpstack.sql \ + difftool_definewarpstack_old.sql \ + difftool_definewarpstack_part1.sql \ + difftool_definewarpstack_part2.sql \ + difftool_definewarpstack_temp_create.sql \ + difftool_definewarpwarp_temp_create.sql \ + difftool_definewarpwarp_temp_insert.sql \ + difftool_definewarpwarp_select.sql \ + difftool_definewarpwarp_insert.sql \ + difftool_definestackstack_part0.sql \ + difftool_definestackstack_part1.sql \ + difftool_donecleanup.sql \ + difftool_export_input_skyfile.sql \ + difftool_export_run.sql \ + difftool_export_skyfile.sql \ + difftool_inputskyfile.sql \ + difftool_listrun.sql \ + difftool_listssrun.sql \ + difftool_pendingcleanuprun.sql \ + difftool_pendingcleanupskyfile.sql \ + difftool_pendingcleanupskyfile_all.sql \ + difftool_revertdiffskyfile_delete.sql \ + difftool_revertdiffskyfile_updated.sql \ + difftool_revertcleanup.sql \ + difftool_setskyfiletoupdate.sql \ + difftool_skyfile.sql \ + difftool_todiffskyfile.sql \ + difftool_tosummary.sql \ + difftool_addsummary.sql \ + disttool_definebyquery_camera.sql \ + disttool_definebyquery_chip.sql \ + disttool_definebyquery_chip_bg.sql \ + disttool_definebyquery_diff.sql \ + disttool_definebyquery_fake.sql \ + disttool_definebyquery_ff.sql \ + disttool_definebyquery_raw.sql \ + disttool_definebyquery_raw_no_magic.sql \ + disttool_definebyquery_sky.sql \ + disttool_definebyquery_sky_singlefilter.sql \ + disttool_definebyquery_skycal.sql \ + disttool_definebyquery_stack.sql \ + disttool_definebyquery_warp.sql \ + disttool_definebyquery_warp_bg.sql \ + disttool_definebyquery_SSdiff.sql \ + disttool_defineinterest.sql \ + disttool_listfilesets.sql \ + disttool_listinterests.sql \ + disttool_pending_camera.sql \ + disttool_pending_chip.sql \ + disttool_pending_chip_bg.sql \ + disttool_pending_diff.sql \ + disttool_pending_fake.sql \ + disttool_pending_ff.sql \ + disttool_pending_raw.sql \ + disttool_pending_sky.sql \ + disttool_pending_skycal.sql \ + disttool_pending_stack.sql \ + disttool_pending_warp.sql \ + disttool_pending_warp_bg.sql \ + disttool_pending_SSdiff.sql \ + disttool_pendingfileset.sql \ + disttool_pendingdest.sql \ + disttool_pendingcleanup.sql \ + disttool_processedcomponent.sql \ + disttool_queuercrun.sql \ + disttool_rerun_select.sql \ + disttool_revertrcrun.sql \ + disttool_revertrun.sql \ + disttool_revertcomponent.sql \ + disttool_revertfileset.sql \ + disttool_toadvance.sql \ + disttool_updateinterest.sql \ + disttool_updatercrun.sql \ + dqstatstool_createbundle.sql \ + dqstatstool_definebyquery.sql \ + dqstatstool_get_contents.sql \ + dqstatstool_get_run.sql \ + faketool_change_exp_state.sql \ + faketool_change_imfile_data_state.sql \ + faketool_completely_processed_exp.sql \ + faketool_donecleanup.sql \ + faketool_export_processed_imfile.sql \ + faketool_export_run.sql \ + faketool_find_camrun.sql \ + faketool_find_pendingexp.sql \ + faketool_pendingcleanupimfile.sql \ + faketool_pendingcleanuprun.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ + faketool_revertprocessedimfile.sql \ + faketool_unmasked.sql \ + flatcorr_chiprundone.sql \ + flatcorr_camerarundone.sql \ + flatcorr_pendingprocess.sql \ + flatcorr_inputimfile.sql \ + flatcorr_dropchip.sql \ + flatcorr_dropcamera.sql \ + magictool_deletemask.sql \ + magictool_restore_camera.sql \ + magictool_restore_chip.sql \ + magictool_restore_diff.sql \ + magictool_restore_raw.sql \ + magictool_restore_warp.sql \ + magictool_create_tmp_warpcomplete.sql \ + magictool_definebyquery_insert.sql \ + magictool_definebyquery_select.sql \ + magictool_definebyquery_select_multidiff.sql \ + magictool_inputs.sql \ + magictool_inputskyfile.sql \ + magictool_mask.sql \ + magictool_tomask.sql \ + magictool_toprocess_inputs.sql \ + magictool_toprocess_runs.sql \ + magictool_toprocess_tree.sql \ + magictool_toskyfilemask.sql \ + magictool_totree.sql \ + magictool_diffskyfile.sql \ + magictool_warpskyfile.sql \ + magictool_chipprocessedimfile.sql \ + magictool_rawimfile.sql \ + magictool_revertnode.sql \ + magictool_exposure.sql \ + magictool_setfull.sql \ + magictool_setgotocleaned.sql \ + magictool_tocleanup.sql \ + magicdstool_clearstatefaults.sql \ + magicdstool_change_file_data_state.sql \ + magicdstool_change_run_state.sql \ + magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ + magicdstool_definebyquery_raw.sql \ + magicdstool_definebyquery_chip.sql \ + magicdstool_definebyquery_chip_bg.sql \ + magicdstool_definebyquery_camera.sql \ + magicdstool_definebyquery_warp.sql \ + magicdstool_definebyquery_warp_bg.sql \ + magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_camera.sql \ + magicdstool_definecopy_chip.sql \ + magicdstool_definecopy_warp.sql \ + magicdstool_destreakedfile.sql \ + magicdstool_getrunids.sql \ + magicdstool_getskycells.sql \ + magicdstool_listrun.sql \ + magicdstool_revertdestreakedfile.sql \ + magicdstool_revertupdated.sql \ + magicdstool_setfiletoupdate.sql \ + magicdstool_tocleanup.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ + magicdstool_toremove.sql \ + magicdstool_torevert_raw.sql \ + magicdstool_torevert_chip.sql \ + magicdstool_torevert_camera.sql \ + magicdstool_torevert_warp.sql \ + magicdstool_torevert_diff.sql \ + minidvodbtool_find_minidvodb_id_dvo.sql \ + minidvodbtool_find_minidvodbcopy.sql \ + minidvodbtool_queue_minidvodb_id.sql \ + minidvodbtool_revertminidvodbcopy.sql \ + mergetool_find_minidvodb_id.sql \ + mergetool_find_mergedvodbcopy.sql \ + mergetool_find_merge_id_dvo.sql \ + mergetool_find_pendingmergeprocess.sql \ + mergetool_find_pending_merge.sql \ + mergetool_find_processed.sql \ + mergetool_queue_mergecopy_id.sql \ + mergetool_queue_merge_id.sql \ + mergetool_revertmergedvodbcopy.sql \ + mergetool_revertmergedvodbprocessed.sql \ + pstamptool_completedreq.sql \ + pstamptool_datastore.sql \ + pstamptool_getdependent.sql \ + pstamptool_listjob.sql \ + pstamptool_listfile.sql \ + pstamptool_listuser.sql \ + pstamptool_pendingcleanup.sql \ + pstamptool_pendingdependent.sql \ + pstamptool_pendingjob.sql \ + pstamptool_pendingreq.sql \ + pstamptool_project.sql \ + pstamptool_revertdependent.sql \ + pstamptool_revertjob.sql \ + pstamptool_revertreq.sql \ + pstamptool_revertreq_deletejobs.sql \ + pstamptool_stopdependentjob.sql \ + pstamptool_updatejob.sql \ + pstamptool_addaccesslevel.sql \ + pxadmin_create_tables.sql \ + pxadmin_create_mirror_tables.sql \ + pxadmin_drop_tables.sql \ + pxadmin_update_version.sql \ + pubtool_definerun.sql \ + pubtool_pending.sql \ + pubtool_revert.sql \ + pztool_find_completed_exp.sql \ + pztool_pendingimfile.sql \ + pztool_revert_downloadimfile_faults.sql \ + pztool_revert_fileset_faults.sql \ + pztool_revertcopied.sql \ + receivetool_list.sql \ + receivetool_addfileset.sql \ + receivetool_pendingfileset.sql \ + receivetool_pendingfile.sql \ + receivetool_revert.sql \ + receivetool_toadvance.sql \ + regtool_checkburntoolimfile.sql \ + regtool_create_dup_table.sql \ + regtool_export_exp.sql \ + regtool_export_imfile.sql \ + regtool_pendingexp.sql \ + regtool_pendingimfile.sql \ + regtool_populate_dup_table.sql \ + regtool_processedexp.sql \ + regtool_processedimfile.sql \ + regtool_revertprocessedexp.sql \ + regtool_revertprocessedimfile.sql \ + regtool_updateprocessedimfile.sql \ + regtool_pendingcompressimfile.sql \ + regtool_finishcompressexp.sql \ + regtool_updatebyquery.sql \ + regtool_updatebyqueryimfile.sql \ + regtool_pendingburntoolimfile.sql \ + regtool_checkstatus.sql \ + stacktool_associationdefine_select.sql \ + stacktool_definebyquery_insert.sql \ + stacktool_definebyquery_insert_random_part1.sql \ + stacktool_definebyquery_insert_random_part2.sql \ + stacktool_definebyquery_select.sql \ + stacktool_definebyquery_test.sql \ + stacktool_donecleanup.sql \ + stacktool_export_input_skyfile.sql \ + stacktool_export_run.sql \ + stacktool_export_sum_skyfile.sql \ + stacktool_find_complete_warps.sql \ + stacktool_inputskyfile.sql \ + stacktool_pendingcleanuprun.sql \ + stacktool_pendingcleanupskyfile.sql \ + stacktool_revertsumskyfile_delete.sql \ + stacktool_sumskyfile.sql \ + stacktool_sassskyfile.sql \ + stacktool_tosum.sql \ + stacktool_tobkg.sql \ + stacktool_tosummary.sql \ + stacktool_addsummary.sql \ + stacktool_revertsummary.sql \ + stacktool_summary.sql \ + staticskytool_definebyquery_select.sql \ + staticskytool_definebyquery_select_by_dg.sql \ + staticskytool_definebyquery_inputs.sql \ + staticskytool_defineskycalrun.sql \ + staticskytool_pendingskycalrun.sql \ + staticskytool_revertskycal.sql \ + staticskytool_skycalresult.sql \ + staticskytool_export_run.sql \ + staticskytool_export_input.sql \ + staticskytool_export_result.sql \ + staticskytool_export_skycalrun.sql \ + staticskytool_export_skycalresult.sql \ + staticskytool_inputs.sql \ + staticskytool_todo.sql \ + staticskytool_result.sql \ + staticskytool_revert.sql \ + staticskytool_revert_update.sql \ + warptool_change_skyfile_data_state.sql \ + warptool_change_run_state.sql \ + warptool_definebyquery.sql \ + warptool_donecleanup.sql \ + warptool_exp.sql \ + warptool_export_imfile.sql \ + warptool_export_run.sql \ + warptool_export_skyfile.sql \ + warptool_export_skycell_map.sql \ + warptool_finished_run_select.sql \ + warptool_finish_run.sql \ + warptool_coalesce_run.sql \ + warptool_imfile.sql \ + warptool_listrun.sql \ + warptool_pendingcleanuprun.sql \ + warptool_pendingcleanupskyfile.sql \ + warptool_pendingcleanupskyfile_all.sql \ + warptool_revertoverlap.sql \ + warptool_revertwarped_delete.sql \ + warptool_revertwarped_updated.sql \ + warptool_revertcleanup.sql \ + warptool_runstate.sql \ + warptool_scmap.sql \ + warptool_setskyfiletoupdate.sql \ + warptool_tooverlap.sql \ + warptool_towarped.sql \ + warptool_towarped_labels.sql \ + warptool_updateskyfile.sql \ + warptool_warped.sql \ + warptool_tosummary.sql \ + warptool_addsummary.sql \ + diffphottool_definerun.sql \ + diffphottool_input.sql \ + diffphottool_pending.sql \ + diffphottool_advance.sql \ + diffphottool_revert.sql \ + diffphottool_data.sql \ + laptool_definerun.sql \ + laptool_definerunbyrelease.sql \ + laptool_exposures.sql \ + laptool_inactiveexp.sql \ + laptool_listsequence.sql \ + laptool_pendingchipexp.sql \ + laptool_pendingexp.sql \ + laptool_pendingrun.sql \ + laptool_WSdiff_check.sql \ + laptool_stacks.sql \ + laptool_pendinggroup.sql \ + laptool_revertgroup.sql \ + laptool_filtersforgroup.sql \ + laptool_revertsass.sql \ + vptool_find_rawexp.sql \ + vptool_pendingimfile.sql \ + vptool_pendingrun.sql \ + vptool_processedcell.sql \ + vptool_revertrun.sql \ + sctool_list.sql \ + releasetool_listsurvey.sql \ + releasetool_tocalibexp.sql \ + releasetool_listrelease.sql \ + releasetool_listrelexp.sql \ + releasetool_definerelexp.sql \ + releasetool_definerelstack.sql \ + releasetool_definerelstack_with_skycal.sql \ + releasetool_deleterelexp.sql \ + releasetool_deleterelstack.sql \ + releasetool_listrelstack.sql \ + releasetool_setrelstackcalibratedfromskycal.sql \ + releasetool_summary.sql \ + releasetool_definerelgroup_select_lap.sql \ + releasetool_definerelgroup_select_data_group.sql \ + releasetool_definerelgroup_select_exp_data_group.sql \ + releasetool_definerelgroup_select_exp_lap.sql \ + releasetool_stacksummary.sql \ + releasetool_pendingrelgroup.sql \ + fftool_definebyquery.sql \ + fftool_definebyquery_select_warps.sql \ + fftool_result.sql \ + fftool_revert.sql \ + fftool_revertsummary.sql \ + fftool_summary.sql \ + fftool_todo.sql \ + fftool_toadvance.sql \ + fftool_export_run.sql \ + fftool_export_input.sql \ + fftool_export_result.sql \ + fftool_export_summary.sql \ + remotetool_definebyquery_chip.sql \ + remotetool_definebyquery_cam.sql \ + remotetool_definebyquery_warp.sql \ + remotetool_definebyquery_stack.sql \ + remotetool_definebyquery_staticsky.sql \ + remotetool_definebyquery_diff.sql \ + remotetool_definebyquery_ff.sql \ + remotetool_dropcomponent.sql \ + remotetool_listcomponent.sql \ + remotetool_doneprep.sql \ + remotetool_updatepoll.sql Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_checkminidvodbaddrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_checkminidvodbaddrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_checkminidvodbaddrun.sql (revision 41217) @@ -0,0 +1,20 @@ +-- this selects the minidvodbs that have complete addRuns +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg +AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count +FROM minidvodbRun +JOIN + (SELECT minidvodb_name, count(state) as cnt2, cnt, minidbg, minidbn + FROM addRun LEFT JOIN + -- this select grabs (and counts) all the addRuns with any state + -- and groups them by the name + (SELECT minidvodb_group as minidbg, minidvodb_name as minidbn, count(state) + AS cnt + FROM addRun where minidvodb_group = '%s' + GROUP BY minidvodb_name ) + AS foo1 ON minidvodb_name = minidbn + -- the second select grabs and counts all the addRuns with state + -- of 'full' + WHERE addRun.state = 'full' and minidvodb_group = '%s' + GROUP BY minidvodb_name ) +AS foo2 +USING(minidvodb_name) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + addRun.* +FROM addRun +WHERE + addRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + camRun.* +FROM camRun +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN (SELECT exp_id AS added_exp_id, + addRun.dvodb AS previous_dvodb + FROM addRun + JOIN camRun on cam_id=stage_id + JOIN chipRun USING(chip_id) + ) as foo + ON exp_id = added_exp_id + AND stage = 'cam' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + camRun.state = 'full' + AND camProcessedExp.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_cam_id_dvo.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT camRun.* FROM camRun +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE camRun.state = 'full' and camProcessedExp.quality = 0 + AND exp_id NOT IN (SELECT exp_id + FROM addRun + JOIN camRun on cam_id=stage_id + JOIN chipRun USING(chip_id) + WHERE stage = 'cam' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id) + +LEFT JOIN (SELECT diff_id AS added_diff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN diffRun on (diff_id = stage_id and addRun.stage = 'diff') + ) as foo + ON diff_id = added_diff_id + -- AND stage = 'diff' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + diffRun.state = 'full' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND added_diff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_diff_id_dvo.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id,tess_id) +JOIN skycell using (skycell_id, tess_id) + JOIN warpRun on (warp1 = warp_id + AND diffInputSkyfile.skycell_id = skycell_id) + JOIN fakeRun using (fake_id) + JOIN camRun using (cam_id) + JOIN chipRun using (chip_id) + JOIN rawExp using (exp_id) +WHERE diffRun.state = 'full' and diffSkyfile.quality = 0 + AND diffInputSkyfile.diff_skyfile_id NOT IN (SELECT diffInputSkyfile.diff_skyfile_id + FROM + diffInputSkyfile + JOIN diffSkyfile USING (diff_id, skycell_id) + JOIN diffRun using(diff_id, tess_id) + JOIN skycell using (skycell_id, tess_id) + left join addRun ON (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = "diff") + WHERE addRun.stage = 'diff' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT + fullForceRun.*, fullForceResult.warp_id +FROM + fullForceRun +JOIN fullForceResult using (ff_id) + +LEFT JOIN (SELECT ff_id AS added_ff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN fullForceRun on (ff_id = stage_id and addRun.stage = 'fullforce') + ) as foo + ON ff_id = added_ff_id + --AND stage = 'skycal' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + fullForceRun.state = 'full' + AND fullForceResult.quality = 0 + AND fullForceResult.fault = 0 + AND added_ff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ff_id_dvo.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT fullForceRun.*, fullForceResult.warp_id FROM fullForceRun JOIN fullForceResult using (ff_id) +JOIN skycalRun using (skycal_id) join stackRun using (stack_id) +JOIN skycell using (skycell_id, tess_id) + +WHERE fullForceRun.state = 'full' and fullForceResult.quality = 0 + AND fullForceRun.ff_id NOT IN (SELECT fullForceRun.ff_id + FROM addRun + JOIN fullForceRun on (fullForceRun.ff_id = addRun.stage_id and addRun.stage = 'fullforce') JOIN fullForceResult on (fullForceRun.ff_id = fullForceResult.ff_id) + WHERE addRun.stage = 'fullforce' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id.sql (revision 41217) @@ -0,0 +1,18 @@ +SELECT fullForceRun.* FROM fullForceRun +JOIN fullForceSummary USING (ff_id) +LEFT JOIN ( + SELECT ff_id AS added_ff_id, + addRun.dvodb AS previous_dvodb + FROM addRun + JOIN fullForceRun ON (ff_id = stage_id and addRun.stage = 'fullforce_summary') + ) as foo +ON ff_id = added_ff_id + -- AND stage = 'skycal' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + fullForceRun.state = 'full' + AND fullForceSummary.quality = 0 + AND fullForceSummary.fault = 0 + AND added_ff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_ffsummary_id_dvo.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT fullForceRun.* FROM fullForceRun +JOIN fullForceSummary USING (ff_id) +JOIN skycalRun USING (skycal_id) +JOIN stackRun USING (stack_id) +JOIN skycell USING (skycell_id, tess_id) + +WHERE fullForceRun.state = 'full' +AND fullForceSummary.quality = 0 +AND fullForceSummary.fault = 0 +AND fullForceRun.ff_id NOT IN ( + SELECT fullForceRun.ff_id FROM addRun + JOIN fullForceRun ON (fullForceRun.ff_id = addRun.stage_id and addRun.stage = 'fullforce_summary') + JOIN fullForceSummary ON (fullForceRun.ff_id = fullForceSummary.ff_id) + WHERE addRun.stage = 'fullforce_summary' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbprocessed.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbprocessed.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbprocessed.sql (revision 41217) @@ -0,0 +1,8 @@ +SELECT + minidvodbProcessed.*, + minidvodbRun.minidvodb_name, + minidvodbRun.minidvodb_group +FROM minidvodbProcessed +JOIN minidvodbRun + USING(minidvodb_id) + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + minidvodbRun.* +FROM minidvodbRun + + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun2.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun2.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_minidvodbrun2.sql (revision 41217) @@ -0,0 +1,23 @@ +-- this selects the minidvodbs that have complete addRuns +SELECT minidvodb_name, minidvodb_path, state, minidbg +AS minidvodb_group +FROM minidvodbRun +JOIN + (SELECT minidvodb_name, count(state) as cnt2, cnt, minidbg, minidbn + FROM addRun LEFT JOIN + -- this select grabs (and counts) all the addRuns with any state + -- and groups them by the name + (SELECT minidvodb_group as minidbg, minidvodb_name as minidbn, count(state) + AS cnt + FROM addRun + GROUP BY minidvodb_name ) + AS foo1 ON minidvodb_name = minidbn + -- the second select grabs and counts all the addRuns with state + -- of 'full' + WHERE addRun.state = 'full' + GROUP BY minidvodb_name ) +AS foo2 +USING(minidvodb_name) +WHERE cnt2 = cnt + -- the 3rd select only grabs the columns where the counts are the same + -- (same number of 'full' and any state) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT + addRun.* +FROM addRun +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE +((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_cam.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_cam.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_cam.sql (revision 41217) @@ -0,0 +1,34 @@ +SELECT + addRun.*, + camProcessedExp.path_base as stageroot, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + minidvodbRun.minidvodb_host as dvodbhost +FROM addRun +JOIN minidvodbRun + on addRun.minidvodb_group = minidvodbRun.minidvodb_group +JOIN camRun + on cam_id = stage_id +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + camRun.state = 'full' + AND addRun.stage = 'cam' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + AND minidvodbRun.state = 'active' + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_diff.sql (revision 41217) @@ -0,0 +1,21 @@ + SELECT + addRun.*, + diffSkyfile.path_base as stageroot, + 'GPC1' as camera, + 'PS1' as telescope +FROM addRun +JOIN diffInputSkyfile + ON (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = "diff") +JOIN diffSkyfile + USING (diff_id, skycell_id) +JOIN diffRun + using(diff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + diffRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ff.sql (revision 41217) @@ -0,0 +1,19 @@ + SELECT + addRun.*, + fullForceResult.path_base as stageroot, + 'GPC1' as camera, + 'PS1' as telescope +FROM addRun +JOIN fullForceResult + ON (ff_id = stage_id and warp_id = stage_extra1 and stage = "fullforce") +JOIN fullForceRun + using(ff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + fullForceRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ffsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ffsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_ffsummary.sql (revision 41217) @@ -0,0 +1,19 @@ + SELECT + addRun.*, + fullForceSummary.path_base as stageroot, + 'GPC1' as camera, + 'PS1' as telescope +FROM addRun +JOIN fullForceSummary + ON (ff_id = stage_id and stage = "fullforce_summary") +JOIN fullForceRun + using(ff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + fullForceRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_skycal.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT + addRun.*, + skycalResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN skycalRun + ON skycal_id = stage_id +JOIN skycalResult + USING (skycal_id) +JOIN stackRun + using(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + skycalRun.state = 'full' + AND stage = 'skycal' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_stack.sql (revision 41217) @@ -0,0 +1,28 @@ +SELECT + addRun.*, + stackSumSkyfile.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN stackRun + ON stack_id = stage_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + stackRun.state = 'full' + AND stage = 'stack' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky.sql (revision 41217) @@ -0,0 +1,32 @@ +SELECT + addRun.*, + staticskyResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN staticskyRun + ON sky_id = stage_id +JOIN staticskyResult + USING (sky_id) +JOIN staticskyInput + USING (sky_id) +JOIN stackRun + USING (stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + staticskyRun.state = 'full' + AND stage = 'staticsky' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky_multi.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky_multi.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingexp_staticsky_multi.sql (revision 41217) @@ -0,0 +1,32 @@ +SELECT + addRun.*, + staticskyResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN staticskyRun + ON sky_id = stage_id +JOIN staticskyResult + USING (sky_id) +JOIN staticskyInput + USING (sky_id) +JOIN stackRun + USING (stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + staticskyRun.state = 'full' + AND stage = 'staticsky' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingmergeprocess.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingmergeprocess.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_pendingmergeprocess.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT minidvodbRun.* +FROM minidvodbRun +LEFT JOIN minidvodbProcessed +USING(minidvodb_id) +WHERE (minidvodbRun.state = 'to_be_merged' AND minidvodbProcessed.minidvodb_id IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_cam.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_cam.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_cam.sql (revision 41217) @@ -0,0 +1,18 @@ +SELECT + addProcessedExp.*, + addRun.label, + addRun.workdir, + camRun.cam_id, + camRun.label as cam_label, + camRun.data_group as cam_data_group, + rawExp.exp_id, + exp_name +FROM addProcessedExp +JOIN addRun + USING(add_id) +JOIN camRun + on cam_id = stage_id +JOIN chipRun + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_diff.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ff.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ffsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ffsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_ffsummary.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_skycal.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_stack.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky_multi.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky_multi.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_processedexp_staticsky_multi.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + staticskyRun.* +FROM + staticskyRun +join staticskyResult using (sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT sky_id AS added_sky_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN staticskyRun on sky_id = stage_id + ) as foo + ON sky_id = added_sky_id + AND stage = 'staticsky' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + AND added_sky_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_dvo.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT staticskyRun.* FROM staticskyRun +JOIN staticskyResult USING(sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 + AND stack_id NOT IN (SELECT stack_id + FROM addRun + JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id + JOIN staticskyRun USING(sky_id) + JOIN staticskyInput USING(sky_id) + WHERE addRun.stage = 'staticsky' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + distinct(sky_id) as stage_id, + staticskyRun.*, + staticskyResult.num_inputs, stack_id +FROM + staticskyRun +join staticskyResult using (sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT stack_id AS added_stack_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN staticskyRun on sky_id = stage_id + ) as foo + ON stack_id = added_stack_id + AND stage = 'staticsky' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + AND added_stack_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_sky_id_multi_dvo.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT distinct(sky_id) as stage_id, staticskyRun.*, staticskyResult.num_inputs, stack_id FROM staticskyResult +JOIN staticskyRun USING(sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 + AND stack_id NOT IN (SELECT stack_id + FROM addRun + JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id + JOIN staticskyRun USING(sky_id) + JOIN staticskyInput USING(sky_id) + WHERE addRun.stage = 'staticsky' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + skycalRun.* +FROM + skycalRun +JOIN skycalResult using (skycal_id) +JOIN staticskyInput using (sky_id, stack_id) +JOIN staticskyRun using (sky_id) + +LEFT JOIN (SELECT skycal_id AS added_skycal_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN skycalRun on skycal_id = stage_id + ) as foo + ON skycal_id = added_skycal_id + AND stage = 'skycal' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + skycalRun.state = 'full' + AND skycalResult.quality = 0 + AND added_skycal_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_skycal_id_dvo.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT skycalRun.* FROM skycalRun +JOIN skycalResult USING (skycal_id) +JOIN staticskyInput USING (sky_id,stack_id) +JOIN stackRun USING(stack_id) + +WHERE skycalRun.state = 'full' and skycalResult.quality = 0 + AND stack_id NOT IN (SELECT stack_id + FROM addRun + JOIN skycalResult on skycalResult.skycal_id = addRun.stage_id + JOIN skycalRun USING(skycal_id) + WHERE addRun.stage = 'skycal' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + stackRun.* +FROM stackSumSkyfile +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT stack_id AS added_stack_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN stackRun on stack_id = stage_id + ) as foo + ON stack_id = added_stack_id + AND stage = 'stack' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + stackRun.state = 'full' + AND stackSumSkyfile.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_find_stack_id_dvo.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT stackRun.* FROM stackSumSkyfile +JOIN stackRun USING(stack_id) +WHERE stackRun.state = 'full' and stackSumSkyfile.quality = 0 + AND stack_id NOT IN (SELECT stack_id + FROM addRun + JOIN stackInputSkyfile on stackInputSkyfile.stack_id = addRun.stage_id + JOIN stackRun USING(stack_id) + WHERE addRun.stage = 'stack' AND %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanupexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanupexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanupexp.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + addProcessedExp.*, + addRun.state, + addRun.workdir, + addRun.label, + addRun.dvodb, +FROM addRun +JOIN addProcessedExp + USING(add_id) +WHERE + (addRun.state = 'goto_cleaned' OR addRun.state = 'goto_purged' OR addRun.state = 'goto_scrubbed') + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT + addRun.add_id, + rawExp.camera, + addRun.state +FROM addRun +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (addRun.state = 'goto_cleaned' OR addRun.state = 'goto_purged' OR addRun.state = 'goto_scrubbed') Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_cam_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_cam_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_cam_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'cam', -- stage + cam_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_diff_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_diff_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_diff_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'diff', -- stage + diff_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM diffRun + WHERE + diffRun.state = 'full' + AND diffRun.diff_id = %lld + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ff_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ff_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ff_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'fullforce', -- stage + ff_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM fullForceRun + WHERE + fullForceRun.state = 'full' + AND fullForceRun.ff_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ffsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ffsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_ffsummary.sql (revision 41217) @@ -0,0 +1,24 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'fullforce_summary', -- stage + ff_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM fullForceRun + WHERE + fullForceRun.state = 'full' + AND fullForceRun.ff_id = %lld + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_minidvodbrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_minidvodbrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_minidvodbrun.sql (revision 41217) @@ -0,0 +1,11 @@ +-- create rcRun's for all destinations that have filesets that they are interested in available +INSERT INTO minidvodbRun +SELECT -- rows in this select must match rcRun + minidvodb_name, + minidvodb_path, + 'new' + +FROM addRun +LEFT JOIN minidvodbRun USING(minidvodb_name) +-- WHERE minidvodbRun.minidvodb_name IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'staticsky', -- stage + sky_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM staticskyRun + WHERE + staticskyRun.state = 'full' + AND staticskyRun.sky_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id_multi.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id_multi.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_sky_id_multi.sql (revision 41217) @@ -0,0 +1,22 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'staticsky', -- stage + sky_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM staticskyRun + WHERE + staticskyRun.state = 'full' + AND staticskyRun.sky_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_skycal_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_skycal_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_skycal_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'skycal', -- stage + skycal_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM skycalRun + WHERE + skycalRun.state = 'full' + AND skycalRun.skycal_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_stack_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_stack_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_queue_stack_id.sql (revision 41217) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'stack', -- stage + stack_id, -- stage_id + %d, --stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s', -- minidvodb_name + '%s' -- minidvodb_host + FROM stackRun + WHERE + stackRun.state = 'full' + AND stackRun.stack_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertminidvodbprocessed.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertminidvodbprocessed.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertminidvodbprocessed.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM minidvodbProcessed +USING minidvodbProcessed, minidvodbRun, addRun +WHERE + minidvodbProcessed.minidvodb_id = minidvodbRun.minidvodb_id + AND addRun.minidvodb_name = minidvodbRun.minidvodb_name + AND minidvodbProcessed.fault != 0 + AND minidvodbRun.state = 'merged' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp.sql (revision 41217) @@ -0,0 +1,9 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_cam.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_cam.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_cam.sql (revision 41217) @@ -0,0 +1,10 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.stage_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'cam' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_diff.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'diff' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ff.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'fullforce' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ffsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ffsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_ffsummary.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'fullforce_summary' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_skycal.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'skycal' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_stack.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'stack' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'staticsky' Index: /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky_multi.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky_multi.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/addtool_revertprocessedexp_staticsky_multi.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'staticsky' Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancechip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancechip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancechip.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT DISTINCT + chipBackgroundRun.chip_bg_id, + chipBackgroundImfile.magicked +FROM chipBackgroundRun +JOIN camRun using(cam_id) +JOIN chipProcessedImfile ON chipBackgroundRun.chip_id = chipProcessedImfile.chip_id +JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id +JOIN chipProcessedImfile AS bgsub_chipProcessedImfile + ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id + AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id) +LEFT JOIN chipBackgroundImfile + ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id + AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id) +WHERE chipBackgroundRun.state = 'new' + AND chipProcessedImfile.quality = 0 + AND chipProcessedImfile.fault = 0 + AND bgsub_chipProcessedImfile.quality = 0 +-- WHERE hook %s +GROUP BY chipBackgroundRun.chip_bg_id +HAVING + COUNT(chipBackgroundImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(IF(chipBackgroundImfile.fault > 0, 1, 0)) = 0 + AND SUM(IF(chipBackgroundImfile.quality > 0, 1, 0)) != COUNT(*) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancewarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancewarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_advancewarp.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT DISTINCT + warpBackgroundRun.warp_bg_id, + warpBackgroundSkyfile.magicked +FROM warpBackgroundRun +JOIN warpSkyfile USING(warp_id) +LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id) +WHERE warpBackgroundRun.state = 'new' + AND warpSkyfile.quality = 0 + AND warpSkyfile.fault = 0 +-- WHERE hook %s +GROUP BY warp_bg_id +HAVING + COUNT(warpBackgroundSkyfile.skycell_id) = COUNT(warpSkyfile.skycell_id) + AND SUM(IF(warpBackgroundSkyfile.fault > 0, 1, 0)) = 0 + AND SUM(IF(warpBackgroundSkyfile.quality > 0, 1, 0)) != COUNT(*) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_chiprun_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_chiprun_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_chiprun_state.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE chipBackgroundRun +SET chipBackgroundRun.state = '%s', + chipBackgroundRun.magicked = IF(chipBackgroundRun.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld +AND (SELECT COUNT(class_id) + FROM chipBackgroundImfile + WHERE + chipBackgroundImfile.chip_bg_id = chipBackgroundRun.chip_bg_id + AND data_state != '%s' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_imfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_imfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_imfile_data_state.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE chipBackgroundImfile + JOIN chipBackgroundRun USING(chip_bg_id) +SET data_state = '%s', + chipBackgroundImfile.magicked = IF(chipBackgroundImfile.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld AND class_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_skyfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_skyfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_skyfile_data_state.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE warpBackgroundSkyfile + JOIN warpBackgroundRun USING(warp_bg_id) +SET data_state = '%s', + warpBackgroundSkyfile.magicked = IF(warpBackgroundSkyfile.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld AND skycell_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_warprun_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_warprun_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_change_warprun_state.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE warpBackgroundRun +SET state = '%s', + warpBackgroundRun.magicked = IF(warpBackgroundRun.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld +AND (SELECT COUNT(skycell_id) + FROM warpBackgroundSkyfile + WHERE + warpBackgroundSkyfile.warp_bg_id = warpBackgroundRun.warp_bg_id + AND data_state != '%s' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chip.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + chipBackgroundImfile.*, + chipBackgroundRun.state, + chipBackgroundRun.workdir, + chipBackgroundRun.label, + chipBackgroundRun.data_group, + chipBackgroundRun.chip_id, + chipBackgroundRun.cam_id, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM chipBackgroundRun +JOIN chipBackgroundImfile USING(chip_bg_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chipinputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chipinputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_chipinputs.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + chipProcessedImfile.*, + camProcessedExp.path_base AS cam_path_base +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN chipProcessedImfile USING(chip_id) +LEFT JOIN camProcessedExp ON chipBackgroundRun.cam_id = camProcessedExp.cam_id +WHERE chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedchip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedchip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedchip.sql (revision 41217) @@ -0,0 +1,3 @@ +UPDATE chipBackgroundRun +SET state = '%s' +WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedwarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedwarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_cleanedwarp.sql (revision 41217) @@ -0,0 +1,3 @@ +UPDATE warpBackgroundRun +SET state = '%s' +WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definechip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definechip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definechip.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + chipRun.*, + camRun.cam_id, + chip_bg_id, + CURRENT_TIMESTAMP AS registered +FROM chipRun +JOIN rawExp USING(exp_id) +JOIN chipRun as altChipRun USING(exp_id) +JOIN camRun ON altChipRun.chip_id = camRun.chip_id +JOIN camProcessedExp USING(cam_id) +LEFT JOIN chipBackgroundRun ON chipBackgroundRun.chip_id = chipRun.chip_id -- labelHook %s +WHERE chipRun.state = 'full' AND camRun.state ='full' AND camProcessedExp.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definewarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definewarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_definewarp.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + warpRun.warp_id, + chipBackgroundRun.* + -- the following items are selected for aid in debugging + , + warpsChipRun.chip_id as warpschip_id, + chipRun.chip_id AS thischip_id, + warpBackgroundRun.warp_bg_id +FROM warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun as warpsChipRun ON camRun.chip_id = warpsChipRun.chip_id +JOIN rawExp USING(exp_id) +JOIN chipRun USING(exp_id) +JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id + AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0) +LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.chip_bg_id -- label hook %s +WHERE chipBackgroundRun.state = 'full' + AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listchip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listchip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listchip.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT chip_bg_id, + class_id, + chipProcessedImfile.path_base +FROM chipBackgroundRun + JOIN chipBackgroundImfile USING(chip_bg_id) + JOIN chipProcessedImfile USING(chip_id, class_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listwarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listwarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_listwarp.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT warp_bg_id, + skycell_id, + warpSkyfile.path_base +FROM warpBackgroundRun + JOIN warpBackgroundSkyfile USING(warp_bg_id) + JOIN warpSkyfile USING(warp_id, skycell_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchipimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchipimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchipimfile.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + chipBackgroundImfile.*, + chipBackgroundRun.state, + chipBackgroundRun.workdir, + chipBackgroundRun.label +FROM chipBackgroundRun +JOIN chipBackgroundImfile + USING(chip_bg_id) +WHERE + ((chipBackgroundRun.state = 'goto_cleaned' AND (chipBackgroundImfile.data_state = 'full' + OR chipBackgroundImfile.data_state = 'update')) +OR + (chipBackgroundRun.state = 'goto_scrubbed' AND chipBackgroundImfile.data_state != 'scrubbed') +OR + (chipBackgroundRun.state = 'goto_purged' AND chipBackgroundImfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchiprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchiprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupchiprun.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + chipBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM chipBackgroundRun + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (chipBackgroundRun.state = 'goto_cleaned' + OR chipBackgroundRun.state = 'goto_purged' + OR chipBackgroundRun.state = 'goto_scrubbed' + ) + Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarprun.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT warpBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM warpBackgroundRun + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (warpBackgroundRun.state = 'goto_cleaned' + OR warpBackgroundRun.state = 'goto_purged' + OR warpBackgroundRun.state = 'goto_scrubbed') + Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarpskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarpskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_pendingcleanupwarpskyfile.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + warpBackgroundSkyfile.*, + warpBackgroundRun.state, + warpBackgroundRun.workdir, + warpBackgroundRun.label +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile + USING(warp_bg_id) +WHERE + ((warpBackgroundRun.state = 'goto_cleaned' AND (warpBackgroundSkyfile.data_state = 'full' + OR warpBackgroundSkyfile.data_state = 'update')) +OR + (warpBackgroundRun.state = 'goto_scrubbed' AND warpBackgroundSkyfile.data_state != 'scrubbed') +OR + (warpBackgroundRun.state = 'goto_purged' AND warpBackgroundSkyfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertchip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertchip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertchip.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE chipBackgroundImfile +FROM chipBackgroundRun +JOIN chipBackgroundImfile USING(chip_bg_id) +WHERE chipBackgroundImfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertwarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertwarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_revertwarp.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE warpBackgroundSkyfile +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile USING(warp_bg_id) +WHERE warpBackgroundSkyfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tochip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tochip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tochip.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT + chipBackgroundRun.*, + chipProcessedImfile.class_id, + rawExp.exp_tag, + rawExp.camera +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN chipProcessedImfile USING(chip_id) +JOIN camRun using(cam_id) +JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id +JOIN chipProcessedImfile AS bgsub_chipProcessedImfile + ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id + AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id) +LEFT JOIN chipBackgroundImfile + ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id + AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id) +LEFT JOIN Label ON chipBackgroundRun.label = Label.label +WHERE chipBackgroundImfile.chip_bg_id IS NULL + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND bgsub_chipProcessedImfile.quality = 0 + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s +ORDER BY priority DESC, chip_bg_id, class_id Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanchip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanchip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanchip.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + chipBackgroundRun.chip_bg_id, + chipBackgroundRun.state, + rawExp.camera +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE chipBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanwarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanwarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_tocleanwarp.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + warpBackgroundRun.warp_bg_id, + warpBackgroundRun.state, + rawExp.camera +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE warpBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_towarp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_towarp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_towarp.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT DISTINCT + warpBackgroundRun.*, + warpRun.tess_id, + skycell_id, + rawExp.exp_tag, + rawExp.camera +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN chipBackgroundRun USING(chip_bg_id) +JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id) +LEFT JOIN Label ON Label.label = warpBackgroundRun.label +WHERE warpBackgroundSkyfile.warp_bg_id IS NULL + AND chipBackgroundRun.state = 'full' + AND chipBackgroundImfile.fault = 0 + AND chipBackgroundImfile.quality = 0 + AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s +ORDER BY priority DESC, warp_bg_id, skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warp.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + warpBackgroundSkyfile.*, + warpBackgroundRun.state, + warpBackgroundRun.warp_id, + camRun.cam_id, + warpBackgroundRun.chip_bg_id, + warpBackgroundRun.workdir, + warpBackgroundRun.label, + warpRun.label as warp_label, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile USING(warp_bg_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warpinputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warpinputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/bgtool_warpinputs.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + chipBackgroundImfile.path_base AS chip_path_base, + chipBackgroundImfile.class_id, + chipBackgroundImfile.magicked, + camProcessedExp.path_base AS cam_path_base +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN chipBackgroundRun USING(chip_bg_id) +JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +JOIN fakeRun USING(fake_id) +JOIN camProcessedExp ON camProcessedExp.cam_id = fakeRun.cam_id +WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND chipBackgroundRun.state = 'full' + AND chipBackgroundImfile.fault = 0 + AND chipBackgroundImfile.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_addprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_addprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_addprocessedexp.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + camRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + chipRun.magicked AS chip_magicked, + camProcessedExp.path_base +FROM camRun +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN camProcessedExp + USING(cam_id) +LEFT JOIN camMask + ON camRun.label = camMask.label +WHERE + chipRun.state = 'full' + AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) OR camRun.state = 'update') + AND camMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + camRun.* +FROM camRun +WHERE + camRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_processed_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_processed_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_processed_exp.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * from camProcessedExp Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_export_run.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT camRun.* from camRun Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_chip_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_chip_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_chip_id.sql (revision 41217) @@ -0,0 +1,28 @@ +SELECT + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.sun_angle +FROM chipRun +JOIN rawExp + using(exp_id) +WHERE + chipRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_pendingimfile.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT + camRun.cam_id, + chipProcessedImfile.*, + (IF(rawExp.exp_time IS NOT NULL AND deteff_magref IS NOT NULL, + (2.5 * LOG10(rawExp.exp_time) + chipProcessedImfile.deteff_magref), + NULL)) + AS deteff_inst, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM camRun +JOIN chipRun + USING (chip_id) +JOIN chipProcessedImfile + USING (chip_id, exp_id) +JOIN rawExp + USING(exp_id) +JOIN rawImfile + USING(exp_id, class_id) +LEFT JOIN camProcessedExp + USING(cam_id) +LEFT JOIN camMask + ON camRun.label = camMask.label +WHERE + camMask.label IS NULL + AND rawImfile.ignored = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_processedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_processedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_find_processedexp.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT + camProcessedExp.*, + HEX(IFNULL(camProcessedExp.astrom_chips, 0)) AS good_astrom_chips, + camRun.workdir, + camRun.label, + camRun.data_group, + camRun.state, + camRun.magicked, + chipRun.chip_id, + chipRun.state as chip_state, + chipRun.magicked as chip_magicked, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.exp_time, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.filter, + rawExp.filelevel, + rawExp.magicked as raw_magicked, + rawExp.comment +FROM camRun +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanupexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanupexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanupexp.sql (revision 41217) @@ -0,0 +1,16 @@ +SELECT + camProcessedExp.*, + camRun.state, + camRun.workdir, + camRun.label, + camRun.reduction, + camRun.expgroup, + camRun.dvodb, + camRun.tess_id, + camRun.end_stage +FROM camRun +JOIN camProcessedExp + USING(cam_id) +WHERE + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') + Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + camRun.workdir, + camRun.state +FROM camRun +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_pendingexp.sql (revision 41217) @@ -0,0 +1,28 @@ +SELECT + camRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + chipRun.magicked AS chip_magicked, + camProcessedExp.path_base, + IFNULL(Label.priority, 10000) AS priority +FROM camRun +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN camProcessedExp + USING(cam_id) +LEFT JOIN camMask + ON camRun.label = camMask.label +LEFT JOIN Label + ON camRun.label = Label.label +WHERE + chipRun.state = 'full' + AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) OR + (camRun.state = 'update' and camProcessedExp.fault = 0 and camProcessedExp.quality = 0)) + AND camMask.label IS NULL + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_queue_chip_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_queue_chip_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_queue_chip_id.sql (revision 41217) @@ -0,0 +1,34 @@ +-- camtool only operates on exposures so we can safely queue more then one at a +-- time without worrying about losing the track of the generated cam_id +INSERT INTO camRun + SElECT + 0, -- cam_id + chip_id, -- chip_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- data_group + '%s', -- dist_group + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s', -- end_stage + %lld, -- magicked + '%s', -- software ver + '%s', -- maskfrac_ref_npix + '%s', -- _ref_static + '%s', -- _ref_dynamic + '%s', -- _ref_magic + '%s', -- _ref_advisory + '%s', -- maskfrac_max_npix + '%s', -- _max_static + '%s', -- _max_dynamic + '%s', -- _max_magic + '%s', -- _max_advisory + '%s' -- note + FROM chipRun + WHERE + chipRun.state = 'full' + AND chipRun.chip_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertprocessedexp.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM camProcessedExp +USING camProcessedExp, camRun, chipRun, rawExp +WHERE + camRun.cam_id = camProcessedExp.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND camProcessedExp.fault != 0 + AND camRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertupdatedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertupdatedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/camtool_revertupdatedexp.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE camProcessedExp + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +SET camProcessedExp.fault = 0 +WHERE + camRun.state = 'update' + AND camProcessedExp.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_exp_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_exp_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_exp_state.sql (revision 41217) @@ -0,0 +1,18 @@ +-- change state of chipRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the constituant imfiles are in the right state +-- arguments are new state (cleaned or purged) chip_id and new state again for +-- the chipProcessedImfile sub query +UPDATE chipRun +JOIN rawExp using(exp_id) + SET chipRun.state = '%s', chipRun.update_mode = 0 + -- set magicked hook %s + WHERE + chipRun.chip_id = %lld + AND (SELECT + COUNT(chip_id) + FROM chipProcessedImfile + WHERE + chipProcessedImfile.chip_id = chipRun.chip_id + AND chipProcessedImfile.quality = 0 + AND data_state != '%s' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_imfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_imfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_change_imfile_data_state.sql (revision 41217) @@ -0,0 +1,12 @@ +-- handle changes in data_state. Used for the modes tocleanedimfile and topurgedimfile +-- args are new data_state, a possibly empty string for updating the magicked state, +-- chip_id and class_id +UPDATE chipProcessedImfile +JOIN chipRun USING(chip_id, exp_id) +JOIN rawImfile USING(exp_id, class_id) + SET + chipProcessedImfile.data_state = '%s' + -- set magicked hook %s +WHERE + chip_id = %lld + AND class_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_coalesce_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_coalesce_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_coalesce_run.sql (revision 41217) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from chipProcessedImfile where data_state = 'full' AND fault = 0 AND chip_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_completely_processed_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_completely_processed_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_completely_processed_exp.sql (revision 41217) @@ -0,0 +1,48 @@ +-- the output of this query must match the format of chipRun row +SELECT DISTINCT + chip_id, + exp_id, + state, + workdir, + workdir_state, + label, + data_group, + dist_group, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + imfile_magicked as magicked, + 0 as update_mode, + note +FROM + (SELECT + chipRun.*, + rawImfile.class_id as rawimfile_class_id, + chipProcessedImfile.class_id, + -- XXX using chipProcessedImfile assumes that all imfile's have + -- the same magicked value if that isn't right then more than one + -- row will be returned. In practice this is the case but bugs could + -- cause it to not be true. + -- We could use rawExp.magicked but that would make it possible for + -- the chipRun to have a different magicked value than the imfiles + chipProcessedImfile.magicked AS imfile_magicked + FROM chipRun + JOIN rawImfile + ON chipRun.exp_id = rawImfile.exp_id + AND rawImfile.ignored = 0 + LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'new' + GROUP BY + chipRun.chip_id, + chipRun.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + AND SUM(chipProcessedImfile.quality > 0) != COUNT(*) + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_definecopy.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_definecopy.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_definecopy.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + rawExp.*, + newExp.label, + chipOld.workdir AS old_workdir, + chipOld.data_group AS old_data_group, + chipOld.state AS old_state +FROM chipRun AS chipOld +JOIN rawExp USING(exp_id) +JOIN newExp using (exp_id) +LEFT JOIN chipRun AS chipNew + ON chipNew.exp_id = chipOld.exp_id + AND chipNew.label = '%s' +WHERE chipNew.chip_id IS NULL + AND rawExp.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + chipRun.* +FROM chipRun +WHERE + chipRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_delete.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_delete.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_delete.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM magicDSFile + USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile +WHERE chipRun.chip_id = chipProcessedImfile.chip_id + AND magicDSRun.stage ='chip' + AND magicDSRun.stage_id = chipRun.chip_id + AND magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_update.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_update.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_dropprocessedimfile_update.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE chipProcessedImfile LEFT JOIN camRun USING(chip_id) + SET chipProcessedImfile.quality = %d, + chipProcessedImfile.data_state = 'full', + chipProcessedImfile.magicked = 0, + chipProcessedImfile.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + chipImfile.* +FROM chipImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_processed_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_processed_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_processed_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + chipProcessedImfile.* +FROM chipProcessedImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_export_run.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + chipRun.* +FROM chipRun Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_rawexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_rawexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_rawexp.sql (revision 41217) @@ -0,0 +1,10 @@ +-- this query is used to find potental rawExps to be queued for chiptool +-- processeing +SELECT DISTINCT + rawExp.*, + newExp.label +FROM rawExp +JOIN newExp using (exp_id) +LEFT JOIN chipRun ON chipRun.exp_id = rawExp.exp_id -- JOIN hook %s +WHERE + rawExp.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_unprocessed_imfile.pl =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_unprocessed_imfile.pl (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_find_unprocessed_imfile.pl (revision 41217) @@ -0,0 +1,8 @@ +SELECT DISTINCT + chipInputImfile.* +FROM chipInputImfile +LEFT JOIN chipProcessedImfile + USING(chip_id, class_id) +WHERE + chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.class_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_listrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_listrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_listrun.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT DISTINCT + chipRun.chip_id, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.data_group, + chipRun.dist_group, + chipRun.note, + chipRun.magicked, + chipRun.reduction, + camRun.cam_id, + camRun.state AS camRun_state, + camProcessedExp.path_base AS cam_path_base, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.magicked AS raw_magicked, + magicDSRun.state AS dsRun_state, + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id +FROM chipRun +JOIN rawExp + USING(exp_id) +LEFT JOIN camRun USING(chip_id) +LEFT JOIN camProcessedExp USING(cam_id) +LEFT JOIN magicDSRun + ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop' Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + chipProcessedImfile.*, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.reduction, + chipRun.expgroup, + chipRun.dvodb, + chipRun.tess_id, + chipRun.end_stage +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +WHERE + ((chipRun.state = 'goto_cleaned' + AND (chipProcessedImfile.data_state = 'full' + OR chipProcessedImfile.data_state = 'update' + OR chipProcessedImfile.data_state = 'error_cleaned') + AND chipProcessedImfile.quality = 0) +OR + (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed') +OR + (chipRun.state = 'goto_purged' AND chipProcessedImfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile_all.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile_all.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanupimfile_all.sql (revision 41217) @@ -0,0 +1,16 @@ +-- list of chipProcessedImfiles for runs in a goto_cleaned like state without regard to +-- data_state or quality. Used for confirming that a run is really in the cleaned state +SELECT + chipProcessedImfile.*, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.reduction, + chipRun.expgroup, + chipRun.dvodb, + chipRun.tess_id, + chipRun.end_stage +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +WHERE ( chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged' ) Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT DISTINCT + chipRun.chip_id, + rawExp.camera, + rawExp.exp_tag, + chipRun.state, + chipRun.workdir, + chipRun.label, + IFNULL(Label.priority, 10000) AS priority +FROM chipRun +JOIN rawExp +USING (exp_id) +LEFT JOIN Label ON chipRun.label = Label.label +WHERE + (chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_pendingimfile.sql (revision 41217) @@ -0,0 +1,42 @@ +SELECT + chipRun.*, + chipImfile.chip_imfile_id, + rawImfile.class_id, + rawImfile.uri, + rawImfile.magicked as raw_magicked, + (rawImfile.user_1 is not NULL and rawImfile.user_1 > 0.5) as deburned, + rawImfile.burntool_state as burntool_state, + rawImfile.video_cells as video_cells, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + IFNULL(Label.priority, 10000) AS priority, + chipProcessedImfile.path_base +FROM chipRun +JOIN rawExp + USING(exp_id) +JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + AND rawImfile.ignored = 0 +JOIN chipImfile + USING(chip_id, class_id) +LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id +LEFT JOIN chipMask + ON chipRun.label = chipMask.label +LEFT JOIN Label ON chipRun.label = Label.label +WHERE + ((chipRun.state = 'new' + AND chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.exp_id IS NULL + AND chipProcessedImfile.class_id IS NULL + AND chipMask.label IS NULL) + OR + (chipRun.state = 'update' + AND chipProcessedImfile.data_state = 'update' + AND chipProcessedImfile.fault = 0)) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_processedimfile.sql (revision 41217) @@ -0,0 +1,47 @@ +SELECT DISTINCT + chipProcessedImfile.chip_id, + chipImfile.chip_imfile_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.path_base, + chipProcessedImfile.magicked, + chipProcessedImfile.data_state, + chipProcessedImfile.fault, + chipProcessedImfile.quality, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.data_group, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawImfile.magicked AS raw_magicked, + rawImfile.burntool_state, + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id, + IFNULL(magicDSRun.state,0) AS dsRun_state, + IFNULL(magicDSFile.fault,0) AS dsFile_fault, + IFNULL(magicDSFile.data_state,0) AS dsFile_data_state +FROM chipRun +JOIN chipImfile + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id, class_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id +JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id +LEFT JOIN magicDSRun + ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop' +LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id AND chipProcessedImfile.class_id = magicDSFile.component Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertcleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertcleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertcleanup.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE chipProcessedImfile + JOIN chipRun using(chip_id,exp_id) +SET chipProcessedImfile.data_state = 'full', + chipRun.state = '%s' +WHERE + chipRun.state = '%s' + AND chipProcessedImfile.data_state = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertprocessedimfile.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM chipProcessedImfile +USING chipProcessedImfile, chipRun, rawExp +WHERE + chipRun.chip_id = chipProcessedImfile.chip_id + AND rawExp.exp_id = chipProcessedImfile.exp_id + AND chipRun.state = 'new' + AND chipProcessedImfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertupdatedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertupdatedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_revertupdatedimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE chipProcessedImfile + JOIN chipRun using(chip_id,exp_id) + JOIN rawExp using(exp_id) +SET chipProcessedImfile.fault = 0 +WHERE + chipRun.state = 'update' + AND chipProcessedImfile.data_state = 'update' + AND chipProcessedImfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_run.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + chipRun.* +FROM chipRun +JOIN rawExp + USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_runstate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_runstate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_runstate.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + chip_id, + chipRun.state, + exp_id, + magic_ds_id, + magicDSRun.state as magic_ds_state +FROM chipRun + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun on stage = 'chip' and stage_id = chip_id and re_place + Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_setimfiletoupdate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_setimfiletoupdate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_setimfiletoupdate.sql (revision 41217) @@ -0,0 +1,18 @@ +UPDATE chipRun + JOIN chipProcessedImfile USING(chip_id) + -- LEFT JOIN magicDSRun ON (stage_id = chip_id AND stage = 'chip') + -- LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + -- AND component = class_id) +SET chipRun.state = 'update', + chipProcessedImfile.data_state = 'update', + chipProcessedImfile.fault = 0 + -- set hook %s +WHERE chip_id = %lld + AND (chipRun.state = 'cleaned' OR chipRun.state = 'update') + AND (chipProcessedImfile.data_state = 'cleaned') + AND (chipProcessedImfile.quality = 0) + -- don't queue update if the associated magicDSFile exists and isn't cleaned + -- AND (chipRun.magicked = 0 + -- OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + -- AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) + -- ) Index: /branches/ipp-148_http_503_filesset_revert_fix/chiptool_unmasked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/chiptool_unmasked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/chiptool_unmasked.sql (revision 41217) @@ -0,0 +1,43 @@ +-- SELECT +-- * +-- FROM +-- (SELECT +-- chipMask.label as label, +-- count(chipRun.chip_id) as n_chipruns +-- FROM +-- chipMask +-- LEFT JOIN +-- chipRun +-- USING +-- (label) +-- WHERE chipRun.chip_id IS NOT NULL +-- GROUP BY label) as chipMask + +-- SELECT +-- * +-- FROM +-- SELECT +-- chipRun.*, +-- rawExp.*, +-- chipRun.chip_id as n_chipruns +-- FROM +-- chipRun +-- JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +-- LEFT JOIN chipMask +-- USING (label) +-- WHERE chipMask.label IS NULL +-- AND filter = 'r' +-- AND chipRun.exp_id = 61 +-- GROUP BY chipRun.label) as chipUnmask + +SELECT + * +FROM + (SELECT + chipRun.label, + count(chipRun.chip_id) as n_chipruns + FROM chipRun + LEFT JOIN chipMask + USING (label) + WHERE chipMask.label IS NULL + GROUP BY chipRun.label) as chipUnmask Index: /branches/ipp-148_http_503_filesset_revert_fix/detselect_search.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/detselect_search.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/detselect_search.sql (revision 41217) @@ -0,0 +1,29 @@ +-- this query needs to use the same fields in both of the tables in +-- the union statement, but we need to report the +-- detRunSummary.iteration in the first case, and this is missing in +-- the second case. + +SELECT DISTINCT + det_id, + good_iteration as iteration, + filelevel +FROM + (SELECT DISTINCT + detRun.*, + detRunSummary.iteration as good_iteration + FROM detRun + JOIN detRunSummary + USING(det_id) + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.*, + detRun.iteration as good_iteration + FROM detRun + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop') Index: /branches/ipp-148_http_503_filesset_revert_fix/detselect_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/detselect_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/detselect_select.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT DISTINCT + * +FROM + (SELECT DISTINCT + detRun.state, + detNormalizedImfile.* + FROM detNormalizedImfile + JOIN detRun + USING(det_id) + JOIN detRunSummary + ON detNormalizedImfile.det_id = detRunSummary.det_id + AND detNormalizedImfile.iteration = detRunSummary.iteration + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.state, + detRegisteredImfile.* + FROM detRegisteredImfile + JOIN detRun + USING(det_id) + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop' OR state = 'register') Index: /branches/ipp-148_http_503_filesset_revert_fix/detselect_show.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/detselect_show.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/detselect_show.sql (revision 41217) @@ -0,0 +1,35 @@ +-- this query needs to use the same fields in both of the tables in +-- the union statement, but we need to report the +-- detRunSummary.iteration in the first case, and this is missing in +-- the second case. + +SELECT DISTINCT + det_id, + good_iteration as iteration, + filelevel, + det_type, + mode, + registered, + filter, + time_begin, + time_end +FROM + (SELECT DISTINCT + detRun.*, + detRunSummary.iteration as good_iteration + FROM detRun + JOIN detRunSummary + USING(det_id) + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.*, + detRun.iteration as good_iteration + FROM detRun + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_addprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_addprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_addprocessedexp.sql (revision 41217) @@ -0,0 +1,34 @@ +SELECT DISTINCT + detProcessedImfile.det_id, + detRun.iteration, + detRun.det_type, + detProcessedImfile.exp_id +FROM detRun +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id +LEFT JOIN detProcessedExp + ON detInputExp.det_id = detProcessedExp.det_id + AND detProcessedImfile.exp_id= detProcessedExp.exp_id +LEFT JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id + AND detProcessedImfile.class_id = rawImfile.class_id +WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedExp.det_id IS NULL + AND detProcessedExp.exp_id IS NULL + AND detInputExp.include = 1 + AND detRun.det_id = %lld + AND detProcessedImfile.exp_id = %lld +GROUP BY + detProcessedImfile.class_id, + rawImfile.class_id, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_childlessrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_childlessrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_childlessrun.sql (revision 41217) @@ -0,0 +1,12 @@ +-- dettool -childlessrun + +SELECT DISTINCT + detRun.* + FROM detRun + LEFT JOIN detRun as foo + ON foo.ref_det_id = detRun.det_id +-- XXX do we need to restrict to foo.ref_iter = detRun.iteration ? + WHERE + detRun.state = 'stop' + AND detRun.mode = 'master' + AND foo.det_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_definebydetrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_definebydetrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_definebydetrun.sql (revision 41217) @@ -0,0 +1,13 @@ +-- dettool -definebydetrun + +INSERT INTO detInputExp + SELECT + %lld, + 0, + detResidExp.exp_id, + detResidExp.accept + FROM detResidExp + JOIN rawExp + USING(exp_id) + WHERE det_id = %lld + AND iteration = %d Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_detrunsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_detrunsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_detrunsummary.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT DISTINCT + detRunSummary.*, + detRun.det_type, + detRun.mode + FROM detRun + JOIN detRunSummary + USING(det_id, iteration) + WHERE + detRun.state = 'run' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedexp.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + detNormalizedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedExp + USING(det_id,iteration) +WHERE + (detNormalizedExp.data_state = 'cleaned' + OR detNormalizedExp.data_state = 'scrubbed' + OR detNormalizedExp.data_state = 'purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedimfile.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + detNormalizedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedImfile + USING(det_id,iteration) +WHERE + (detNormalizedImfile.data_state = 'cleaned' + OR detNormalizedImfile.data_state = 'scrubbed' + OR detNormalizedImfile.data_state = 'purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedstat.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedstat.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_normalizedstat.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + detNormalizedStatImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedStatImfile + USING(det_id,iteration) +WHERE + (detNormalizedStatImfile.data_state = 'cleaned' + OR detNormalizedStatImfile.data_state = 'scrubbed' + OR detNormalizedStatImfile.data_state = 'purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedexp.sql (revision 41217) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedExp + USING(det_id) +WHERE + (detProcessedExp.data_state = 'cleaned' + OR detProcessedExp.data_state = 'scrubbed' + OR detProcessedExp.data_state = 'purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_processedimfile.sql (revision 41217) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedImfile + USING(det_id) +WHERE + (detProcessedImfile.data_state = 'cleaned' + OR detProcessedImfile.data_state = 'scrubbed' + OR detProcessedImfile.data_state = 'purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residexp.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + detResidExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidExp + USING(det_id,iteration) +WHERE + (detResidExp.data_state = 'cleaned' + OR detResidExp.data_state = 'scrubbed' + OR detResidExp.data_state = 'purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_residimfile.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + detResidImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidImfile + USING(det_id,iteration) +WHERE + (detResidImfile.data_state = 'cleaned' + OR detResidImfile.data_state = 'scrubbed' + OR detResidImfile.data_state = 'purged') + + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_stacked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_stacked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_donecleanup_stacked.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + detStackedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detStackedImfile + USING(det_id,iteration) +WHERE + (detStackedImfile.data_state = 'cleaned' + OR detStackedImfile.data_state = 'scrubbed' + OR detStackedImfile.data_state = 'purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_input_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_input_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_input_exp.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detInputExp.* +FROM detInputExp Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_exp.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detNormalizedExp.* +FROM detNormalizedExp Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detNormalizedImfile.* +FROM detNormalizedImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_stat_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_stat_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_normalized_stat_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detNormalizedStatImfile.* +FROM detNormalizedStatImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_exp.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detProcessedExp.* +FROM detProcessedExp Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_processed_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detProcessedImfile.* +FROM detProcessedImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_registered_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_registered_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_registered_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detRegisteredImfile.* +FROM detRegisteredImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_exp.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detResidExp.* +FROM detResidExp Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_resid_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detResidImfile.* +FROM detResidImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detRun.* +FROM detRun Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run_summary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run_summary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_run_summary.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detRunSummary.* +FROM detRunSummary Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_stacked_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_stacked_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_export_stacked_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + detStackedImfile.* +FROM detStackedImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_find_completed_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_find_completed_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_find_completed_runs.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT DISTINCT + det_id, + iteration +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detInputExp.exp_id + FROM detRun + LEFT JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + LEFT JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + LEFT JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + GROUP BY + detRun.det_id, + detRun.iteration + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) AS residdetrun Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_input.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_input.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_input.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT DISTINCT * + FROM detInputExp + JOIN rawExp + USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedexp.sql (revision 41217) @@ -0,0 +1,8 @@ +SELECT + detNormalizedExp.* + FROM detNormalizedExp + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +SELECT + detNormalizedImfile.* + FROM detNormalizedImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedstat.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedstat.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_normalizedstat.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + * +FROM + (SELECT + detNormalizedStatImfile.* + FROM detNormalizedStatImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pending.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pending.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pending.sql (revision 41217) @@ -0,0 +1,10 @@ +-- dettool -pending : not sure this is still used (EAM : 2008.07.10) + +SELECT + rawExp.* + FROM rawExp + LEFT JOIN detInputExp + ON rawExp.exp_id = detInputExp.exp_id + WHERE + detInputExp.exp_id IS NULL + AND rawExp.object != 'object' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_detrunsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_detrunsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_detrunsummary.sql (revision 41217) @@ -0,0 +1,42 @@ +SELECT DISTINCT detRunSummary.* +FROM detRunSummary + JOIN detProcessedImfile USING (det_id) + JOIN detProcessedExp USING (det_id) + JOIN detStackedImfile USING (det_id,iteration) + JOIN detResidExp USING (det_id,iteration) + JOIN detResidImfile USING (det_id,iteration) + JOIN detNormalizedStatImfile USING (det_id,iteration) + JOIN detNormalizedImfile USING (det_id,iteration) + JOIN detNormalizedExp USING (det_id,iteration) +WHERE (!( + ( + detRunSummary.data_state = 'full' OR + detRunSummary.data_state = 'cleaned' + ) OR + ( + !((detProcessedImfile.data_state = 'cleaned' OR + detProcessedImfile.data_state = 'scrubbed' OR + detProcessedImfile.data_state = 'purged') AND + (detProcessedExp.data_state = 'cleaned' OR + detProcessedExp.data_state = 'scrubbed' OR + detProcessedExp.data_state = 'purged') AND + (detStackedImfile.data_state = 'cleaned' OR + detStackedImfile.data_state = 'scrubbed' OR + detStackedImfile.data_state = 'purged') AND + (detResidExp.data_state = 'cleaned' OR + detResidExp.data_state = 'scrubbed' OR + detResidExp.data_state = 'purged') AND + (detResidImfile.data_state = 'cleaned' OR + detResidImfile.data_state = 'scrubbed' OR + detResidImfile.data_state = 'purged') AND + (detNormalizedStatImfile.data_state = 'cleaned' OR + detNormalizedStatImfile.data_state = 'scrubbed' OR + detNormalizedStatImfile.data_state = 'purged') AND + (detNormalizedImfile.data_state = 'cleaned' OR + detNormalizedImfile.data_state = 'scrubbed' OR + detNormalizedImfile.data_state = 'purged') AND + (detNormalizedExp.data_state = 'cleaned' OR + detNormalizedExp.data_state = 'scrubbed' OR + detNormalizedExp.data_state = 'purged') + ) + ))) Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedexp.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + detNormalizedExp.*, + rawExp.camera +FROM detNormalizedExp +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detNormalizedExp.data_state = 'goto_cleaned' + OR detNormalizedExp.data_state = 'goto_scrubbed' + OR detNormalizedExp.data_state = 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedimfile.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + detNormalizedImfile.*, + rawExp.camera +FROM detNormalizedImfile +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detNormalizedImfile.data_state = 'goto_cleaned' + OR detNormalizedImfile.data_state = 'goto_scrubbed' + OR detNormalizedImfile.data_state = 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedstat.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedstat.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_normalizedstat.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + detNormalizedStatImfile.*, + rawExp.camera +FROM detNormalizedStatImfile +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detNormalizedStatImfile.data_state = 'goto_cleaned' + OR detNormalizedStatImfile.data_state = 'goto_scrubbed' + OR detNormalizedStatImfile.data_state = 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedexp.sql (revision 41217) @@ -0,0 +1,13 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT DISTINCT + detProcessedExp.*, + rawExp.camera +FROM detProcessedExp +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detProcessedExp.data_state = 'goto_cleaned' + OR detProcessedExp.data_state = 'goto_scrubbed' + OR detProcessedExp.data_state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_processedimfile.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + detProcessedImfile.*, + rawExp.camera + FROM detProcessedImfile +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detProcessedImfile.data_state = 'goto_cleaned' + OR detProcessedImfile.data_state = 'goto_scrubbed' + OR detProcessedImfile.data_state = 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residexp.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT DISTINCT + detResidExp.*, + rawExp.camera +FROM detResidExp +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detResidExp.data_state = 'goto_cleaned' + OR detResidExp.data_state = 'goto_scrubbed' + OR detResidExp.data_state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_residimfile.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT DISTINCT + detResidImfile.*, + rawExp.camera +FROM detResidImfile +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detResidImfile.data_state = 'goto_cleaned' + OR detResidImfile.data_state = 'goto_scrubbed' + OR detResidImfile.data_state = 'goto_purged') + + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_stacked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_stacked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_pendingcleanup_stacked.sql (revision 41217) @@ -0,0 +1,12 @@ +ySELECT DISTINCT + detStackedImfile.*, + rawExp.camera +FROM detStackedImfile +JOIN detInputExp + USING(det_id,exp_id) +JOIN rawExp + USING(exp_id) +WHERE + (detStackedImfile.data_state = 'goto_cleaned' + OR detStackedImfile.data_state = 'goto_scrubbed' + OR detStackedImfile.data_state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_processedimfile.sql (revision 41217) @@ -0,0 +1,14 @@ +-- is this DISTINCT needed? +SELECT DISTINCT + detRun.det_type, + rawExp.exp_time, + detProcessedImfile.* +FROM detProcessedImfile +JOIN detRun + USING(det_id) +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + AND detProcessedImfile.exp_id = detInputExp.exp_id +JOIN rawExp + ON rawExp.exp_id = detProcessedImfile.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_raw.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + detInputExp.det_id, + detRun.det_type, + rawImfile.* +FROM detRun +JOIN detInputExp + USING(det_id) +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_residexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_residexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_residexp.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + detRun.mode, + detResidExp.*, + detInputExp.include + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN detResidExp + USING(det_id, iteration, exp_id) + WHERE + detRun.state = 'run' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_residimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_residimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_residimfile.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + detRun.det_type, + detRun.mode, + rawExp.exp_time, + detResidImfile.* +FROM detResidImfile +JOIN detRun + USING(det_id, iteration) +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + AND detResidImfile.exp_id = detInputExp.exp_id +JOIN rawExp + ON rawExp.exp_id = detResidImfile.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertdetrunsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertdetrunsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertdetrunsummary.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detRunSummary +USING detRunSummary, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detRunSummary.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedexp.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detNormalizedExp +USING detNormalizedExp, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detNormalizedExp.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detNormalizedImfile +USING detNormalizedImfile, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detNormalizedImfile.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedstat.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedstat.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertnormalizedstat.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detNormalizedStatImfile +USING detNormalizedStatImfile, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detNormalizedStatImfile.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedexp.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detProcessedExp +USING detProcessedExp, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detProcessedExp.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertprocessedimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detProcessedImfile +USING detProcessedImfile, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detProcessedImfile.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidexp.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detResidExp +USING detResidExp, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detResidExp.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertresidimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detResidImfile +USING detResidImfile, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detResidImfile.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertstacked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertstacked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_revertstacked.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM detStackedImfile +USING detStackedImfile, detRun +WHERE + fault != 0 +AND + state = 'run' +AND + detStackedImfile.det_id = detRun.det_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_runs.sql (revision 41217) @@ -0,0 +1,2 @@ + +SELECT * from detRun Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_stacked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_stacked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_stacked.sql (revision 41217) @@ -0,0 +1,12 @@ +-- select detStackedImfile.* +-- by: +-- where det_id, iteration, class_id is not in detNormalizedImfile + +SELECT + detStackedImfile.* + FROM detStackedImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_start_new_iteration.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_start_new_iteration.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_start_new_iteration.sql (revision 41217) @@ -0,0 +1,33 @@ +-- select detRun.iteration +-- select detInputExp.exp_id +-- select detResidExp.accept +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all exp_ids in the current det_id/iteration from detResidExp +-- compare the counts of exp_ids + +SELECT * FROM + (SELECT DISTINCT + detRun.det_id AS det_id, + detRun.iteration, + detInputExp.exp_id, + detResidExp.accept + FROM detRun + JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + GROUP BY + detRun.det_id, + detRun.iteration, + detInputExp.exp_id + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_stop_completed_correct_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_stop_completed_correct_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_stop_completed_correct_runs.sql (revision 41217) @@ -0,0 +1,30 @@ +-- det1 is the 'correction' detRun, det2 is the 'parent' detRun +UPDATE detRun +SET state = 'stop' +WHERE det_id = (SELECT det_id FROM + (SELECT + det1.det_id, + imfile.class_id, + detRegisteredImfile.class_id as reg_class_id + FROM detRun AS det1 + INNER JOIN detRun as det2 + ON det1.ref_det_id = det2.det_id + -- do we need to restrict by iteration? not clear this logic below is right or tested + JOIN detNormalizedImfile as imfile + ON det2.det_id = imfile.det_id + AND det2.iteration = imfile.iteration + LEFT JOIN detRegisteredImfile + ON det1.det_id = detRegisteredImfile.det_id + AND det1.iteration = detRegisteredImfile.iteration + AND imfile.class_id = detRegisteredImfile.class_id + WHERE + det1.state = 'run' + AND det1.mode = 'correction' + AND det2.state = 'stop' + GROUP BY + det1.det_id + HAVING + COUNT(imfile.class_id) = COUNT(reg_class_id) + AND SUM(imfile.fault) = 0 + ) as Foo +) Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectexp.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT * FROM detRun +WHERE + detRun.mode = 'correction' + AND detRun.state = 'run' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tocorrectimfile.sql (revision 41217) @@ -0,0 +1,32 @@ +-- select imfiles from detRun 1 (the reference detRun) that match the desired correction in detRun 2 +SELECT + det1.*, + imfile.class_id, + imfile.uri, + imfile.bg, + imfile.bg_stdev, + imfile.bg_mean_stdev, + imfile.user_1, + imfile.user_2, + imfile.user_3, + imfile.user_4, + imfile.user_5, + imfile.path_base, + imfile.fault +FROM detRun AS det1 +JOIN detRun as det2 + ON det1.ref_det_id = det2.det_id +JOIN detNormalizedImfile as imfile + ON det1.ref_det_id = imfile.det_id + AND det1.ref_iter = imfile.iteration +LEFT JOIN detRegisteredImfile + ON det1.det_id = detRegisteredImfile.det_id + AND det1.iteration = detRegisteredImfile.iteration + AND imfile.class_id = detRegisteredImfile.class_id +WHERE + det1.state = 'run' + AND det1.mode = 'correction' + AND det2.state = 'stop' + AND detRegisteredImfile.det_id IS NULL + AND detRegisteredImfile.iteration IS NULL + AND detRegisteredImfile.class_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_todetrunsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_todetrunsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_todetrunsummary.sql (revision 41217) @@ -0,0 +1,48 @@ +-- select detRun.det_id +-- select detRun.iteration +-- select detRun.det_type +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all exp_ids in the current det_id/iteration from detResidExp +-- compare the counts of exp_ids + +SELECT DISTINCT + det_id, + iteration, + det_type, + mode, + workdir, + camera +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detInputExp.exp_id, + rawExp.camera + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + LEFT JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + LEFT JOIN detRunSummary + ON detRun.det_id = detRunSummary.det_id + AND detRun.iteration = detRunSummary.iteration + WHERE + detRun.state = 'run' + AND detRunSummary.det_id IS NULL + AND detRunSummary.iteration IS NULL + AND detResidExp.fault = 0 + GROUP BY + detRun.det_id, + detRun.iteration + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) AS Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalize.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalize.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalize.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT DISTINCT + detRun.det_type, + detRun.workdir, + rawExp.camera, + detStackedImfile.uri, + detNormalizedStatImfile.* + FROM detRun + JOIN detStackedImfile + USING(det_id, iteration) + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN detNormalizedStatImfile + ON detStackedImfile.det_id = detNormalizedStatImfile.det_id + AND detStackedImfile.iteration = detNormalizedStatImfile.iteration + AND detStackedImfile.class_id = detNormalizedStatImfile.class_id + LEFT JOIN detNormalizedImfile + ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id + AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration + AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id + WHERE + detNormalizedImfile.det_id IS NULL + AND detNormalizedImfile.iteration IS NULL + AND detNormalizedImfile.class_id IS NULL + AND detNormalizedStatImfile.fault = 0 + AND detRun.state = 'run' Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedexp.sql (revision 41217) @@ -0,0 +1,48 @@ +-- this is used by both 'tonormalizedexp' and 'addnormalizedexp' + +SELECT + * +FROM (SELECT + * + FROM ( + SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + rawExp.camera, + rawExp.telescope, + rawExp.exp_type, + detNormalizedImfile.class_id as norm_class_id, + detNormalizedImfile.fault, + detStackedImfile.class_id as stack_class_id + FROM detRun + JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN detStackedImfile + ON detStackedImfile.det_id = detRun.det_id + AND detStackedImfile.iteration = detRun.iteration + LEFT JOIN detNormalizedImfile + ON detStackedImfile.det_id = detNormalizedImfile.det_id + AND detStackedImfile.iteration = detNormalizedImfile.iteration + AND detStackedImfile.class_id = detNormalizedImfile.class_id + LEFT JOIN detNormalizedExp + ON detInputExp.det_id = detNormalizedExp.det_id + AND detInputExp.iteration = detNormalizedExp.iteration + WHERE + detRun.state = 'run' AND + detRun.mode = 'master' AND + detNormalizedExp.det_id IS NULL AND + detNormalizedExp.iteration IS NULL AND + detInputExp.include = 1 + ) as Foo + GROUP BY + det_id, + iteration + HAVING + COUNT(norm_class_id) = COUNT(stack_class_id) + AND SUM(fault) = 0 +) as Bar Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedstat.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedstat.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tonormalizedstat.sql (revision 41217) @@ -0,0 +1,37 @@ +-- a det_run + iteration is ready for normstat when: all detResidImfile entries for that iteration corresponding to all detProcessedImfile entries are available +SELECT + detRun.det_id, + detRun.det_type, + detRun.iteration, + detRun.camera, + detRun.workdir, + detProcessedImfile.class_id, + COUNT(detProcessedImfile.class_id), + COUNT(detResidImfile.class_id) +FROM detRun +JOIN detInputExp + USING (det_id, iteration) +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id + AND detProcessedImfile.exp_id = detInputExp.exp_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detProcessedImfile.det_id + AND detResidImfile.exp_id = detProcessedImfile.exp_id + AND detResidImfile.class_id = detProcessedImfile.class_id + AND detResidImfile.iteration = detRun.iteration +LEFT JOIN detNormalizedStatImfile + ON detNormalizedStatImfile.det_id = detRun.det_id + AND detNormalizedStatImfile.iteration = detRun.iteration + AND detNormalizedStatImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detNormalizedStatImfile.det_id IS NULL + AND detNormalizedStatImfile.iteration IS NULL + AND detNormalizedStatImfile.class_id IS NULL +GROUP BY + detRun.iteration, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedexp.sql (revision 41217) @@ -0,0 +1,61 @@ +-- select detProcessedImfile.det_id +-- select detRun.iteration +-- select detRun.det_type +-- select detProcessedImfile.exp_id +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all rawImfiles in the current exp_ids +-- compare to detProcessedImfiles by det_id/exp_id +-- found how many imfile there are in each class_id +-- and: +-- det_id is not in detProcessedExp; +-- iteration is not in detProcessedExp; + +SELECT DISTINCT + det_id, + iteration, + det_type, + exp_id, + camera, + workdir, + exp_tag +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + rawImfile.class_id as rawimfile_class_id, + rawExp.camera, + rawExp.exp_tag + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detProcessedImfile + ON detRun.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id + LEFT JOIN detProcessedExp + ON detProcessedImfile.det_id = detProcessedExp.det_id + AND detProcessedImfile.exp_id = detProcessedExp.exp_id + WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedExp.det_id IS NULL + AND detProcessedExp.exp_id IS NULL + AND detInputExp.include = 1 + GROUP BY + rawExp.exp_id, + detRun.det_id + HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) + AND SUM(detProcessedImfile.fault) = 0 + ) AS detProcessedExp + Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_toprocessedimfile.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + detRun.det_id, + detRun.det_type, + detRun.workdir, + detRun.reduction, + rawImfile.*, + rawExp.exp_tag +FROM detRun +JOIN detInputExp + USING(det_id, iteration) +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +LEFT JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND rawImfile.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedImfile.det_id IS NULL + AND detProcessedImfile.exp_id IS NULL + AND detProcessedImfile.class_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidexp.sql (revision 41217) @@ -0,0 +1,114 @@ +-- this is used by both 'toresidexp' and 'addresidexp' + +-- which returns a list of exposures for which all component class ids have had +-- residuals created. This list includes the detrend id, iteration, exposure +-- id, detrend type, and whether the exposure was included in the stack for +-- this iteration. + +-- require the corresponding detNormalizedExp to complete before starting + +-- select detRun.det_id +-- select detRun.iteration +-- select detRun.det_type +-- select detInputExp.exp_id +-- select detInputExp.include +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- compare to detInputExp.imfiles to derResidImfile by class_id +-- and: +-- detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + +SELECT + det_id, + iteration, + det_type, + mode, + exp_id, + include, + camera, + workdir, + exp_tag +FROM +( SELECT + detRun.det_id AS det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detInputExp.exp_id, + detInputExp.include, + rawExp.camera, + detRun.workdir, + rawExp.exp_tag + FROM detRun + JOIN detNormalizedExp + USING(det_id, iteration) + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + on rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detResidImfile + ON detRun.det_id = detResidImfile.det_id + AND detRun.iteration = detResidImfile.iteration + AND detInputExp.exp_id = detResidImfile.exp_id + AND rawImfile.class_id = detResidImfile.class_id + LEFT JOIN detResidExp + ON detResidImfile.det_id = detResidExp.det_id + AND detResidImfile.iteration = detResidExp.iteration + AND detResidImfile.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detResidExp.det_id IS NULL + AND detResidExp.iteration IS NULL + AND detResidExp.exp_id IS NULL + GROUP BY + detInputExp.exp_id, + detRun.iteration, + detRun.det_id + HAVING + COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault) = 0 + UNION + SELECT + detRun.det_id AS det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detInputExp.exp_id, + detInputExp.include, + rawExp.camera, + detRun.workdir, + rawExp.exp_tag + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + on rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detResidImfile + ON detRun.det_id = detResidImfile.det_id + AND detRun.iteration = detResidImfile.iteration + AND detInputExp.exp_id = detResidImfile.exp_id + AND rawImfile.class_id = detResidImfile.class_id + LEFT JOIN detResidExp + ON detResidImfile.det_id = detResidExp.det_id + AND detResidImfile.iteration = detResidExp.iteration + AND detResidImfile.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + AND detRun.mode = 'verify' + AND detResidExp.det_id IS NULL + AND detResidExp.iteration IS NULL + AND detResidExp.exp_id IS NULL + GROUP BY + detInputExp.exp_id, + detRun.iteration, + detRun.det_id + HAVING + COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault) = 0 +) as temp Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_toresidimfile.sql (revision 41217) @@ -0,0 +1,154 @@ +-- which returns a list of processed imfiles (with corresponding detrend id, +-- iteration, class id, uri, type) for which the appropriate stacked imfile has +-- been normalised, and which have not been masked out by detResidImfileAnalysis + +SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detRun.reduction, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + detProcessedImfile.uri, +-- detNormalizedImfile.uri AS det_uri, + detStackedImfile.uri AS det_uri, + detRun.det_id AS ref_det_id, + detRun.iteration AS ref_iter, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detInputExp +-- USING(det_id, iteration) + ON detInputExp.det_id = detRun.det_id + AND detInputExp.iteration = detRun.iteration +-- AND detInputExp.iteration = 0 +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN detProcessedImfile + ON detRun.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id +JOIN detStackedImfile + ON detRun.det_id = detStackedImfile.det_id + AND detRun.iteration = detStackedImfile.iteration + AND detProcessedImfile.class_id = detStackedImfile.class_id +-- EAM : replacing detNormalizedImfile with detStackedImfile to change the sequencing +-- JOIN detNormalizedImfile +-- ON detRun.det_id = detNormalizedImfile.det_id +-- AND detRun.iteration = detNormalizedImfile.iteration +-- AND detProcessedImfile.class_id = detNormalizedImfile.class_id +-- EAM : we there is no reason to wait for all stacks to complete before continuing +-- JOIN detNormalizedExp +-- ON detRun.det_id = detNormalizedExp.det_id +-- AND detRun.iteration = detNormalizedExp.iteration +LEFT JOIN detResidImfile + ON detRun.det_id = detResidImfile.det_id + AND detRun.iteration = detResidImfile.iteration + AND detProcessedImfile.exp_id = detResidImfile.exp_id + AND detProcessedImfile.class_id = detResidImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detStackedImfile.fault = 0 +-- AND detNormalizedImfile.fault = 0 +-- AND detNormalizedExp.fault = 0 + AND detResidImfile.det_id IS NULL + AND detResidImfile.iteration IS NULL + AND detResidImfile.exp_id IS NULL + AND detResidImfile.class_id IS NULL +UNION +SELECT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detRun.reduction, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + detProcessedImfile.uri, + detNormalizedImfile.uri AS det_uri, + detRun.ref_det_id, + detRun.ref_iter, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detRun AS detRunRef + ON detRunRef.det_id = detRun.ref_det_id +JOIN detInputExp + ON detInputExp.det_id = detRun.det_id + AND detInputExp.iteration = detRun.iteration +-- AND detInputExp.iteration = 0 +JOIN rawExp + ON rawExp.exp_id = detInputExp.exp_id +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id + AND detProcessedImfile.exp_id = detInputExp.exp_id +JOIN detNormalizedImfile + ON detNormalizedImfile.det_id = detRun.ref_det_id + AND detNormalizedImfile.iteration = detRun.ref_iter + AND detNormalizedImfile.class_id = detProcessedImfile.class_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detRun.det_id + AND detResidImfile.iteration = detRun.iteration + AND detResidImfile.exp_id = detProcessedImfile.exp_id + AND detResidImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'verify' + AND detRunRef.state = 'stop' + AND detNormalizedImfile.fault = 0 + AND detProcessedImfile.fault = 0 + AND detResidImfile.det_id IS NULL + AND detResidImfile.iteration IS NULL + AND detResidImfile.exp_id IS NULL + AND detResidImfile.class_id IS NULL +UNION +SELECT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detRun.reduction, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + detProcessedImfile.uri, + detRegisteredImfile.uri AS det_uri, + detRun.ref_det_id, + detRun.ref_iter, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detRun AS detRunRef + ON detRunRef.det_id = detRun.ref_det_id +JOIN detInputExp + ON detInputExp.det_id = detRun.det_id + AND detInputExp.iteration = detRun.iteration +-- AND detInputExp.iteration = 0 +JOIN rawExp + ON rawExp.exp_id = detInputExp.exp_id +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id + AND detProcessedImfile.exp_id = detInputExp.exp_id +JOIN detRegisteredImfile + ON detRegisteredImfile.det_id = detRun.ref_det_id + AND detRegisteredImfile.iteration = detRun.ref_iter + AND detRegisteredImfile.class_id = detProcessedImfile.class_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detRun.det_id + AND detResidImfile.iteration = detRun.iteration + AND detResidImfile.exp_id = detProcessedImfile.exp_id + AND detResidImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'verify' + AND detRunRef.state = 'stop' + AND detRegisteredImfile.fault = 0 + AND detProcessedImfile.fault = 0 + AND detResidImfile.det_id IS NULL + AND detResidImfile.iteration IS NULL + AND detResidImfile.exp_id IS NULL + AND detResidImfile.class_id IS NULL +ORDER BY exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/dettool_tostacked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dettool_tostacked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dettool_tostacked.sql (revision 41217) @@ -0,0 +1,52 @@ +-- select detRun.iteration +-- select detProcessedImfile.det_id +-- select detProcessedImfile.class_id +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all rawImfiles in the current exp_ids +-- compare to detProcessedImfiles by det_id/exp_id +-- found how many imfile there are in each class_id +-- and: +-- det_id is not in detStackedImfile; +-- iteration is not in detStackedImfile; +-- class_id is not in detStackedImfile; + +SELECT + detProcessedImfile.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + detRun.reduction, + detProcessedImfile.class_id, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +LEFT JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id +LEFT JOIN detStackedImfile + ON detInputExp.det_id = detStackedImfile.det_id + AND detInputExp.iteration = detStackedImfile.iteration + AND rawImfile.class_id = detStackedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detStackedImfile.det_id IS NULL + AND detStackedImfile.iteration IS NULL + AND detStackedImfile.class_id IS NULL + AND detInputExp.include = 1 +GROUP BY + rawImfile.class_id, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) + AND SUM(detProcessedImfile.fault) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_advance.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_advance.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_advance.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT + diff_phot_id, + diffPhotSkyfile.magicked +FROM diffPhotRun +JOIN diffSkyfile USING(diff_id) +LEFT JOIN diffPhotSkyfile USING(diff_phot_id, skycell_id) +WHERE diffPhotRun.state = 'new' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 +-- WHERE hook %s +GROUP BY diff_phot_id +HAVING COUNT(diffPhotSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) + AND SUM(IF(diffPhotSkyfile.fault > 0, 1, 0)) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_data.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_data.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_data.sql (revision 41217) @@ -0,0 +1,63 @@ +SELECT DISTINCT + diffPhotSkyfile.*, + diffRun.tess_id, + diffPhotRun.state, + diffPhotRun.workdir, + diffPhotRun.label, + diffPhotRun.diff_id, + diffRun.bothways, + diffRun.diff_mode, + -- The following are only valid for warps + -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs + camProcessedInput.zpt_obs, + camProcessedInput.zpt_stdev, + camProcessedInput.zpt_lq, + camProcessedInput.zpt_uq, + rawInput.comment, + rawInput.exp_time, + rawInput.camera, + rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + warp1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, + rawTemplate.exp_name AS exp_name_2, + rawTemplate.exp_id AS exp_id_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + warp2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 +FROM diffPhotRun +JOIN diffPhotSkyfile USING(diff_phot_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +LEFT JOIN warpRun AS warpInput + ON warpInput.warp_id = diffInputSkyfile.warp1 +LEFT JOIN fakeRun AS fakeInput + ON fakeInput.fake_id = warpInput.fake_id +LEFT JOIN camRun AS camInput + ON camInput.cam_id = fakeInput.cam_id +LEFT JOIN camProcessedExp AS camProcessedInput + ON camProcessedInput.cam_id = camInput.cam_id +LEFT JOIN chipRun AS chipInput + ON chipInput.chip_id = camInput.chip_id +LEFT JOIN rawExp AS rawInput + ON rawInput.exp_id = chipInput.exp_id +LEFT JOIN warpRun AS warpTemplate + ON warpTemplate.warp_id = diffInputSkyfile.warp2 +LEFT JOIN fakeRun AS fakeTemplate + ON fakeTemplate.fake_id = warpTemplate.fake_id +LEFT JOIN camRun AS camTemplate + ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id +LEFT JOIN chipRun AS chipTemplate + ON chipTemplate.chip_id = camTemplate.chip_id +LEFT JOIN rawExp AS rawTemplate + ON rawTemplate.exp_id = chipTemplate.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_definerun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_definerun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_definerun.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT DISTINCT + diffRun.* +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- only JOINing inputs, not templates! +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE diffRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_input.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_input.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_input.sql (revision 41217) @@ -0,0 +1,16 @@ +SELECT + diff_phot_id, + skycell_id, + diffSkyfile.path_base, + diffSkyfile.magicked, + diffRun.bothways, + rawExp.camera +FROM diffPhotRun +JOIN diffRun USING(diff_id) +JOIN diffSkyfile USING(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN warpRun ON warp1 = warp_id -- only JOINing on input warps! +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_pending.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_pending.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_pending.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + diffPhotRun.*, + diffRun.tess_id, + diffSkyfile.skycell_id +FROM diffPhotRun +JOIN diffRun USING(diff_id) +JOIN diffSkyfile USING(diff_id) +LEFT JOIN diffPhotSkyfile USING(diff_phot_id, skycell_id) +WHERE diffPhotSkyfile.skycell_id IS NULL + AND diffRun.state = 'full' + AND diffSkyfile.magicked >= 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_revert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_revert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/diffphottool_revert.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE diffPhotSkyfile +FROM diffPhotRun +JOIN diffPhotSkyfile USING(diff_phot_id) +WHERE fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_addsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_addsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_addsummary.sql (revision 41217) @@ -0,0 +1,5 @@ +INSERT INTO diffSummary (diff_id,projection_cell,path_base) VALUES ( + %lld, -- warp_id + '%s', -- projection_cell + '%s' -- path_base + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_run_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_run_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_run_state.sql (revision 41217) @@ -0,0 +1,17 @@ +-- change state of ddiffRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant skyfiles are in the end state +-- arguments are new state (cleaned or purged) diff_id and new state again for +-- the chipProcessedImfile sub query +UPDATE diffRun + SET state = '%s' + -- set magicked hook %s + WHERE + diffRun.diff_id = %lld + AND (SELECT + COUNT(diff_id) + FROM diffSkyfile + WHERE + diffSkyfile.diff_id = diffRun.diff_id + AND data_state != '%s' + AND diffSkyfile.quality = 0 + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_skyfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_skyfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_change_skyfile_data_state.sql (revision 41217) @@ -0,0 +1,10 @@ +-- handle changes in diffSkyfile.data_state. +-- Used for the modes tocleanedskyfile and topurgedskyfile +-- args are new data_state, diff_id, skycell_id and current expected state for diffRun +UPDATE diffSkyfile + SET + data_state = '%s' + -- magicked hook %s +WHERE + diff_id = %lld + AND skycell_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_coalesce_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_coalesce_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_coalesce_run.sql (revision 41217) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from diffSkyfile where diff_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_completed_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_completed_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_completed_runs.sql (revision 41217) @@ -0,0 +1,16 @@ +SELECT + COUNT(diffInputSkyfile.skycell_id), + COUNT(diffSkyfile.skycell_id), + diffSkyfile.*, + SUM(!diffSkyfile.magicked) = 0 as all_magicked +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +LEFT JOIN diffSkyfile USING(diff_id, skycell_id) +WHERE + diffRun.state = 'new' +-- WHERE hook %s +GROUP BY + diffInputSkyfile.diff_id +HAVING + COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) + AND SUM(diffSkyfile.fault) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part0.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part0.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part0.sql (revision 41217) @@ -0,0 +1,46 @@ +SELECT DISTINCT + stackRun.data_group AS INPUT_data_group, + stackRun.dist_group AS INPUT_dist_group, + stackRun.filter AS INPUT_filter, + stackRun.label AS INPUT_label, + stackRun.tess_id AS INPUT_tess_id +-- , +-- stackRun.stack_id AS INPUT_stack_id, +-- template.max_stack_id, +-- diffExp.diff_id +FROM stackRun JOIN stackSumSkyfile USING(stack_id) +JOIN ( + SELECT + MAX(stackRun.stack_id) as max_stack_id, + stackRun.filter, + stackRun.label as template_label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac + FROM stackRun JOIN stackSumSkyfile USING(stack_id) + WHERE + stackRun.state = 'full' AND + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + @STACK2_QUERY@ -- template condition + GROUP BY + skycell_id, + filter +) AS template ON stackRun.filter = template.filter + AND stackRun.tess_id = template.tess_id + AND stackRun.skycell_id = template.skycell_id +LEFT JOIN ( + SELECT + diffRun.diff_id, + diffInputSkyfile.stack1, + diffInputSkyfile.stack2 + FROM diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN stackRun ON stackRun.stack_id = diffInputSkyfile.stack1 + JOIN stackSumSkyfile USING(stack_id) + WHERE 1 + @STACK1_QUERY@ -- input condition +) AS diffExp ON diffExp.stack1 = stackRun.stack_id +WHERE + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + @DIFF0_QUERY@ -- diff constraint + @STACK1_QUERY@ -- input constraint +ORDER BY stackRun.data_group,stackRun.filter Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part1.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part1.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definestackstack_part1.sql (revision 41217) @@ -0,0 +1,56 @@ +SELECT distinct + stackRun.stack_id, + stackRun.data_group, + stackRun.filter, + stackRun.label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac, + template.max_stack_id, + template.template_label, + count(diffExp.diff_id) as n_diff +FROM stackRun JOIN stackSumSkyfile USING(stack_id) +JOIN ( + SELECT + MAX(stackRun.stack_id) as max_stack_id, + stackRun.filter, + stackRun.label as template_label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac + FROM stackRun JOIN stackSumSkyfile USING(stack_id) + WHERE + stackRun.state = 'full' AND + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + @STACK2_QUERY@ -- template constraint + GROUP BY + skycell_id, + filter +) AS template ON stackRun.filter = template.filter + AND stackRun.tess_id = template.tess_id + AND stackRun.skycell_id = template.skycell_id +LEFT JOIN ( + SELECT + diffRun.diff_id, + diffInputSkyfile.stack1, + diffInputSkyfile.stack2 + FROM diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN stackRun ON stackRun.stack_id = diffInputSkyfile.stack1 + JOIN stackSumSkyfile USING(stack_id) + WHERE 1 + @STACK1_QUERY@ -- input constraint +) AS diffExp ON diffExp.stack1 = stackRun.stack_id +WHERE + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + @STACK1_QUERY@ -- input constraint +group by stackRun.stack_id, + stackRun.data_group, + stackRun.filter, + stackRun.label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac, + template.max_stack_id, + template.template_label + @DIFF1_QUERY@ -- diff constraint +ORDER BY stackRun.data_group,stackRun.filter,stackRun.skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack.sql (revision 41217) @@ -0,0 +1,73 @@ +-- Get list of warp-stack pairs that can be diffed +-- and check results against existing diffs +SELECT + exp_id, + warp_id, + filter, + warpLabel, + warpDataGroup, + Inputs.tess_id, + Inputs.skycell_id, + stack_id, + stackLabel, + stackDataGroup, + diff_id, + diffLabel +FROM +( +SELECT + exp_id, + warp_id, + rawExp.filter, + warpRun.label AS warpLabel, + warpRun.data_group AS warpDataGroup, + warpRun.tess_id, + warpSkyfile.skycell_id, + MAX(stack_id) AS stack_id, + stackRun.label AS stackLabel, + stackRun.data_group AS stackDataGroup +FROM warpRun + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN warpSkyfile USING(warp_id) + JOIN stackRun ON + (stackRun.skycell_id = warpSkyfile.skycell_id AND + stackRun.filter = rawExp.filter AND + stackRun.tess_id = warpRun.tess_id) + JOIN stackSumSkyfile USING(stack_id) +WHERE + warpRun.state = 'full' + AND stackRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND exp_id IS NOT NULL +-- %s +GROUP BY exp_id,warp_id,skycell_id +) AS Inputs +LEFT JOIN +( +SELECT +diff_id, +diffRun.label AS diffLabel, +warp1, +stack2, +skycell_id, +diffInputSkyfile.tess_id +FROM +diffRun JOIN diffInputSkyfile USING(diff_id,tess_id) +WHERE +diff_mode = 2 +-- %s +) AS Diffs ON + (warp_id = warp1 AND + stack_id = stack2 AND + Diffs.skycell_id = Inputs.skycell_id AND + Diffs.tess_id = Inputs.tess_id) +WHERE +1 +-- %s + Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_old.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_old.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_old.sql (revision 41217) @@ -0,0 +1,42 @@ +-- Get list of warp-stack pairs that can be diffed +-- and check results against existing diffs +SELECT + exp_id, + warp_id, + rawExp.filter, + warpRun.label AS warpLabel, + warpRun.data_group AS warpDataGroup, + warpRun.tess_id, + warpSkyfile.skycell_id, + MAX(stack_id) AS stack_id, + stackRun.label AS stackLabel, + stackRun.data_group AS stackDataGroup, + diff_id, + diffRun.label AS diffLabel +FROM warpRun + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN warpSkyfile USING(warp_id) + JOIN stackRun ON + (stackRun.skycell_id = warpSkyfile.skycell_id AND + stackRun.filter = rawExp.filter AND + stackRun.tess_id = warpRun.tess_id) + JOIN stackSumSkyfile USING(stack_id) + LEFT JOIN diffInputSkyfile ON + (warp_id = diffInputSkyfile.warp1 AND + stack_id = diffInputSkyfile.stack2 AND + diffInputSkyfile.skycell_id = stackRun.skycell_id AND + diffInputSkyfile.tess_id = stackRun.tess_id) + LEFT JOIN diffRun USING(diff_id) +WHERE + warpRun.state = 'full' + AND stackRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND exp_id IS NOT NULL +-- %s +GROUP BY exp_id,warp_id,skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part1.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part1.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part1.sql (revision 41217) @@ -0,0 +1,38 @@ +SELECT + exp_id, + warp_id, + rawExp.filter, + warpRun.label, + warpRun.tess_id, + warpRun.data_group, + COUNT(skycell_id) as skycell_count +FROM warpRun +JOIN warpSkyfile USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN ( + SELECT DISTINCT + diffRun.*, + warp_id, + exp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + WHERE warp1 IS NOT NULL + AND warpRun.state = 'full' + -- warp where hook %s +) AS diffExp USING(exp_id, warp_id) +WHERE + warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + -- warp where hook %s + -- exp where hook %s + -- diff where hook %s +GROUP BY exp_id, warp_id DESC Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part2.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part2.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_part2.sql (revision 41217) @@ -0,0 +1,37 @@ +-- insert skycells to diff +INSERT INTO skycellsToDiff +SELECT + 0, -- diff_id + warpSkyfile.skycell_id, + warpSkyfile.warp_id, -- warp1 + NULL, -- stack1 + NULL, -- warp2 + max_stack_id, -- stack2 + warpSkyfile.tess_id, + 0 +FROM warpSkyfile +JOIN warpRun USING(warp_id) +LEFT JOIN ( + SELECT + MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment + stackRun.skycell_id, + stackRun.tess_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND stackRun.state = 'full' + -- stacks where hook %s + GROUP BY + skycell_id, + filter + ) as bestStacks USING(skycell_id) +WHERE + warpSkyfile.warp_id = %lld + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND filter = '%s' + AND warpSkyfile.tess_id = bestStacks.tess_id +-- warp where hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_temp_create.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_temp_create.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpstack_temp_create.sql (revision 41217) @@ -0,0 +1,10 @@ +CREATE TEMPORARY TABLE skycellsToDiff ( +diff_id BIGINT, +skycell_id VARCHAR(64), +warp1 BIGINT, +stack1 BIGINT, +warp2 BIGINT, +stack2 BIGINT, +tess_id VARCHAR(64), +diff_skycell_id BIGINT +) ENGINE=MEMORY; Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_insert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_insert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_insert.sql (revision 41217) @@ -0,0 +1,16 @@ +INSERT INTO diffInputSkyfile +SELECT + %s, -- diff_id + skycell_id, + %s, -- input warp_id + NULL, -- input stack_id + %s, -- template warp_id + NULL, -- template stack_id + tess_id, + 0 -- diff_skyfile_id +FROM warpSkyfile AS inputWarpSkyfile +JOIN warpSkyfile AS templateWarpSkyfile USING(skycell_id, tess_id) +WHERE inputWarpSkyfile.quality = 0 + AND templateWarpSkyfile.quality = 0 + AND inputWarpSkyfile.warp_id = %s + AND templateWarpSkyfile.warp_id = %s Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_select.sql (revision 41217) @@ -0,0 +1,33 @@ +SELECT + inputWarpRun.warp_id AS input_warp_id, + inputWarpRun.tess_id AS tess_id, + inputRawExp.exp_id AS input_exp_id, + inputWarpRun.data_group AS input_data_group, + -- The following trick pulls out the warp_id that has the smallest distance + SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id +FROM warpRun AS inputWarpRun +JOIN fakeRun AS inputFakeRun USING(fake_id) +JOIN camRun AS inputCamRun USING(cam_id) +JOIN chipRun AS inputChipRun USING(chip_id) +JOIN rawExp AS inputRawExp USING(exp_id) +-- To find exposures that haven't been diffed, insert newline here:%s LEFT JOIN diffs USING(exp_id) +JOIN warpRun AS templateWarpRun + ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template! + AND templateWarpRun.tess_id = inputWarpRun.tess_id -- Ensure using same tessellation +JOIN fakeRun AS templateFakeRun + ON templateFakeRun.fake_id = templateWarpRun.fake_id + AND templateFakeRun.fake_id != inputFakeRun.fake_id -- Don't use self as template! +JOIN camRun AS templateCamRun + ON templateCamRun.cam_id = templateFakeRun.cam_id + AND templateCamRun.cam_id != inputCamRun.cam_id -- Don't use self as template! +JOIN chipRun AS templateChipRun + ON templateChipRun.chip_id = templateCamRun.chip_id + AND templateChipRun.chip_id != inputChipRun.chip_id -- Don't use self as template! +JOIN rawExp AS templateRawExp + ON templateRawExp.exp_id = templateChipRun.exp_id + AND templateRawExp.filter = inputRawExp.filter + AND templateRawExp.exp_id != inputRawExp.exp_id -- Don't use self as template! +WHERE inputWarpRun.state = 'full' + AND templateWarpRun.state = 'full' +-- WHERE hook %s +GROUP BY input_warp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_create.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_create.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_create.sql (revision 41217) @@ -0,0 +1,8 @@ +-- Exposures that have diffs +-- Temporary table has useful indices that make future query faster! +CREATE TEMPORARY TABLE diffs ( + diff_id BIGINT, + exp_id BIGINT, + PRIMARY KEY(diff_id, exp_id), + KEY(exp_id) +) ENGINE=MEMORY; Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_insert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_insert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_temp_insert.sql (revision 41217) @@ -0,0 +1,32 @@ +-- Get list of exposures that have diffs +-- Only interested in whole exposures (diffRun.exposure = 1) +INSERT INTO diffs +SELECT DISTINCT + diffWarps.diff_id, + exp_id +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + -- WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + -- WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_test.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_test.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_definewarpwarp_test.sql (revision 41217) @@ -0,0 +1,82 @@ +DROP TABLE IF EXISTS diffs; + +-- Create a temporary table with useful indices +CREATE TEMPORARY TABLE diffs ( + diff_id BIGINT, + exp_id BIGINT, + PRIMARY KEY(diff_id, exp_id), + KEY(exp_id) +) ENGINE=MEMORY; + + +-- Get list of exposures that have diffs +INSERT INTO diffs +SELECT DISTINCT + diffWarps.diff_id, + exp_id +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE warp1 IS NOT NULL + -- WHERE hook %s + AND diffRun.label = 'ThreePi.Run2.r.v0' + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id + FROM diffRun + JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffRun.bothways = 1 + WHERE warp2 IS NOT NULL + -- WHERE hook %s + AND diffRun.label = 'ThreePi.Run2.r.v0' + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +; + + + +-- EXPLAIN +SELECT + inputWarpRun.warp_id AS input_warp_id, + inputWarpRun.tess_id AS tess_id, + inputRawExp.exp_id AS input_exp_id, + SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id +FROM warpRun AS inputWarpRun +JOIN fakeRun AS inputFakeRun USING(fake_id) +JOIN camRun AS inputCamRun USING(cam_id) +JOIN chipRun AS inputChipRun USING(chip_id) +JOIN rawExp AS inputRawExp USING(exp_id) +-- To find exposures that haven't been diffed, insert newline here:%s + LEFT JOIN diffs USING(exp_id) +JOIN warpRun AS templateWarpRun + ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template! + AND templateWarpRun.tess_id = inputWarpRun.tess_id -- Ensure using same tessellation +JOIN fakeRun AS templateFakeRun + ON templateFakeRun.fake_id = templateWarpRun.fake_id +JOIN camRun AS templateCamRun + ON templateCamRun.cam_id = templateFakeRun.cam_id +JOIN chipRun AS templateChipRun + ON templateChipRun.chip_id = templateCamRun.chip_id +JOIN rawExp AS templateRawExp + ON templateRawExp.exp_id = templateChipRun.exp_id + AND templateRawExp.filter = inputRawExp.filter +-- WHERE hook %s +WHERE (inputWarpRun.label = 'ThreePi.Run2.r.v0') + AND ((DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2)))) < 0.10000000 + 0.00000119)) + AND ((TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)) > 0.00000000 - 0.00000119)) + AND (templateWarpRun.label = 'ThreePi.Run2.r.v0') +-- AND (diffs.diff_id IS NULL) + AND inputWarpRun.state = 'full' +-- WHERE done +GROUP BY input_warp_id +; Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + diffRun.* +FROM diffRun +WHERE + diffRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_input_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_input_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_input_skyfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT diffInputSkyfile.* FROM diffInputSkyfile Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_run.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT diffRun.* from diffRun Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_export_skyfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT diffSkyfile.* FROM diffSkyfile Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_inputskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_inputskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_inputskyfile.sql (revision 41217) @@ -0,0 +1,126 @@ +SELECT * FROM + (SELECT + -- warp input + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + 0 as stack_id, + warpSkyfile.warp_id, + warpSkyfile.uri, + warpSkyfile.path_base, + warpSkyfile.magicked as magicked, + 0 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp1 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON diffInputSkyfile.warp1 = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + -- where hook %s + UNION + SELECT + -- warp template + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + 0 as stack_id, + warpSkyfile.warp_id, + warpSkyfile.uri, + warpSkyfile.path_base, + warpSkyfile.magicked as magicked, + 1 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp2 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON diffInputSkyfile.warp2 = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + -- where hook %s + UNION + SELECT + -- stack input + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + stackSumSkyfile.stack_id, + 0 as warp_id, + stackSumSkyfile.uri, + stackSumSkyfile.path_base, + 0 as magicked, + 0 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack1 = stackInputSkyfile.stack_id + JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + -- where hook %s + UNION + SELECT + -- stack template + diffRun.diff_id, + diffInputSkyfile.skycell_id, + diffInputSkyfile.tess_id, + stackSumSkyfile.stack_id, + 0 as warp_id, + stackSumSkyfile.uri, + stackSumSkyfile.path_base, + 0 as magicked, + 1 as template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack2 = stackInputSkyfile.stack_id + JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + -- where hook %s + ) as Foo +-- template where hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_listrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_listrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_listrun.sql (revision 41217) @@ -0,0 +1,74 @@ +SELECT DISTINCT + diffRun.diff_id, + diffRun.state, + diffRun.workdir, + diffRun.label, + diffRun.data_group, + diffRun.dist_group, + diffRun.reduction, + diffRun.note, + diffRun.magicked, + diffRun.diff_mode, + diffRun.tess_id, + diffRun.bothways, + warp1, + warp2, + -- The following are only valid for warps + -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs + camProcessedInput.zpt_obs, + camProcessedInput.zpt_stdev, + camProcessedInput.zpt_lq, + camProcessedInput.zpt_uq, + camProcessedInput.fwhm_major, + rawInput.comment, + rawInput.exp_time, + rawInput.camera, + rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + rawInput.filter AS filter_1, + rawInput.comment AS comment_1, + rawInput.dateobs AS dateobs_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, + rawTemplate.exp_name AS exp_name_2, + rawTemplate.exp_id AS exp_id_2, + rawInput.filter AS filter_2, + rawTemplate.comment AS comment_2, + rawTemplate.dateobs AS dateobs_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 + -- ,stack1, + -- stack2 +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +LEFT JOIN warpRun AS warpInput + ON warpInput.warp_id = diffInputSkyfile.warp1 +LEFT JOIN fakeRun AS fakeInput + ON fakeInput.fake_id = warpInput.fake_id +LEFT JOIN camRun AS camInput + ON camInput.cam_id = fakeInput.cam_id +LEFT JOIN camProcessedExp AS camProcessedInput + ON camProcessedInput.cam_id = camInput.cam_id +LEFT JOIN chipRun AS chipInput + ON chipInput.chip_id = camInput.chip_id +LEFT JOIN rawExp AS rawInput + ON rawInput.exp_id = chipInput.exp_id +LEFT JOIN warpRun AS warpTemplate + ON warpTemplate.warp_id = diffInputSkyfile.warp2 +LEFT JOIN fakeRun AS fakeTemplate + ON fakeTemplate.fake_id = warpTemplate.fake_id +LEFT JOIN camRun AS camTemplate + ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id +LEFT JOIN chipRun AS chipTemplate + ON chipTemplate.chip_id = camTemplate.chip_id +LEFT JOIN rawExp AS rawTemplate + ON rawTemplate.exp_id = chipTemplate.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_listssrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_listssrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_listssrun.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT DISTINCT + diffRun.diff_id, + diffRun.state, + diffRun.workdir, + diffRun.label, + diffRun.data_group, + diffRun.dist_group, + diffRun.reduction, + diffRun.note, + diffRun.tess_id +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +JOIN stackRun AS stackInputRun + ON stackInputRun.stack_id = diffInputSkyfile.stack1 +JOIN stackSumSkyfile AS stackInput + ON stackInputRun.stack_id = stackInput.stack_id +JOIN stackRun AS stackTemplateRun + ON stackTemplateRun.stack_id = diffInputSkyfile.stack2 +JOIN stackSumSkyfile AS stackTemplate + ON stackTemplateRun.stack_id = stackTemplate.stack_id +WHERE diff_mode = 4 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,128 @@ +-- does this result in too many entries (one for each diffInputSkyfile?) +-- all of this is just to get the camera used for the diff run +SELECT DISTINCT * FROM + (SELECT DISTINCT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.workdir, + diffRun.tess_id, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp1 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + @INNERCONSTRAINT@ + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + @INNERLIMITS@ + UNION + SELECT DISTINCT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.workdir, + diffRun.tess_id, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp2 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + @INNERCONSTRAINT@ + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + @INNERLIMITS@ + UNION + SELECT DISTINCT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.workdir, + diffRun.tess_id, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) +-- JOIN stackSumSkyfile +-- ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack1 = stackInputSkyfile.stack_id + JOIN warpRun + ON warpRun.warp_id = stackInputSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + @INNERCONSTRAINT@ + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + @INNERLIMITS@ + UNION + SELECT DISTINCT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.workdir, + diffRun.tess_id, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) +-- JOIN stackSumSkyfile +-- ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack2 = stackInputSkyfile.stack_id + JOIN warpRun + ON warpRun.warp_id = stackInputSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + @INNERCONSTRAINT@ + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + @INNERLIMITS@ + ) as Foo + WHERE 1 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + diffSkyfile.*, + diffRun.state, + diffRun.workdir, + diffRun.dvodb, + diffRun.tess_id +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE + ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update') AND diffSkyfile.quality = 0) + OR + (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed') + OR + (diffRun.state = 'goto_purged' AND diffSkyfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile_all.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile_all.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_pendingcleanupskyfile_all.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + diffSkyfile.*, + diffRun.state, + diffRun.workdir, + diffRun.dvodb, + diffRun.tess_id +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE ( diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertcleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertcleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertcleanup.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE diffSkyfile + JOIN diffRun using(diff_id) +SET diffSkyfile.data_state = 'full', + diffRun.state = '%s' +WHERE + diffRun.state = '%s' + AND diffSkyfile.data_state = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_delete.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_delete.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_delete.sql (revision 41217) @@ -0,0 +1,6 @@ +DELETE FROM diffSkyfile +USING diffSkyfile, diffRun +WHERE + diffRun.diff_id = diffSkyfile.diff_id + AND diffRun.state = 'new' + AND fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_update.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_update.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_update.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE diffRun +JOIN diffSkyfile USING(diff_id) +SET diffRun.state = 'new' +WHERE + diffSkyfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_updated.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_updated.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_revertdiffskyfile_updated.sql (revision 41217) @@ -0,0 +1,6 @@ +UPDATE diffSkyfile + JOIN diffRun USING(diff_id) +SET diffSkyfile.fault = 0 +WHERE diffRun.state = 'update' + AND diffSkyfile.data_state = 'update' + AND diffSkyfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_setskyfiletoupdate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_setskyfiletoupdate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_setskyfiletoupdate.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE diffRun + JOIN diffSkyfile USING(diff_id) +SET diffRun.state = 'update', + diffSkyfile.data_state = 'update', + diffSkyfile.fault = 0 + -- set hook %s +WHERE diff_id = %lld + AND (diffRun.state = 'cleaned' OR diffRun.state = 'update') + AND (diffSkyfile.data_state = 'cleaned' OR diffSkyfile.data_state = 'update') + AND (diffSkyfile.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_skyfile.sql (revision 41217) @@ -0,0 +1,66 @@ +SELECT + diffSkyfile.*, + diffInputSkyfile.diff_skyfile_id, + diffRun.tess_id, + diffRun.state, + diffRun.workdir, + diffRun.label, + diffRun.bothways, + diffRun.diff_mode, + warp1, + stack1, + warp2, + stack2, + -- The following are only valid for warps + -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs + camProcessedInput.zpt_obs, + camProcessedInput.zpt_stdev, + camProcessedInput.zpt_lq, + camProcessedInput.zpt_uq, + rawInput.comment, + rawInput.exp_time, + rawInput.camera, + rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + rawInput.filter AS filter_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, + rawTemplate.exp_name AS exp_name_2, + rawTemplate.exp_id AS exp_id_2, + rawTemplate.filter AS filter_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 +FROM diffRun +JOIN diffSkyfile USING(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +LEFT JOIN warpRun AS warpInput + ON warpInput.warp_id = diffInputSkyfile.warp1 +LEFT JOIN fakeRun AS fakeInput + ON fakeInput.fake_id = warpInput.fake_id +LEFT JOIN camRun AS camInput + ON camInput.cam_id = fakeInput.cam_id +LEFT JOIN camProcessedExp AS camProcessedInput + ON camProcessedInput.cam_id = camInput.cam_id +LEFT JOIN chipRun AS chipInput + ON chipInput.chip_id = camInput.chip_id +LEFT JOIN rawExp AS rawInput + ON rawInput.exp_id = chipInput.exp_id +LEFT JOIN warpRun AS warpTemplate + ON warpTemplate.warp_id = diffInputSkyfile.warp2 +LEFT JOIN fakeRun AS fakeTemplate + ON fakeTemplate.fake_id = warpTemplate.fake_id +LEFT JOIN camRun AS camTemplate + ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id +LEFT JOIN chipRun AS chipTemplate + ON chipTemplate.chip_id = camTemplate.chip_id +LEFT JOIN rawExp AS rawTemplate + ON rawTemplate.exp_id = chipTemplate.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_todiffskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_todiffskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_todiffskyfile.sql (revision 41217) @@ -0,0 +1,97 @@ +-- This query is a little complicated, since it contains multiple self-joins. +-- This is because of the options involved: both templates and inputs can be either warps or stacks. +-- All of the templates and inputs need to exist, so we query against all of them. +SELECT DISTINCT + diffRun.diff_id, + diffInputSkyfile.diff_skyfile_id, + diffRun.workdir, + diffInputSkyfile.skycell_id, + diffRun.tess_id, + diffRun.label, + diffRun.state, + diffRun.reduction, + diffRun.bothways, + diffRun.diff_mode, + diffSkyfile.path_base, + IFNULL(priority, 10000) AS priority +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) + +-- Get list of templates for each diffRun +-- JOIN diffInputSkyfile AS diffTemplateSkyfile +-- ON diffRun.diff_id = diffTemplateSkyfile.diff_id +-- AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id +-- AND diffTemplateSkyfile.template = 1 +-- Get list of inputs for each diffRun +-- JOIN diffInputSkyfile +-- ON diffRun.diff_id = diffInputSkyfile.diff_id +-- AND diffRun.skycell_id = diffInputSkyfile.skycell_id +-- AND diffInputSkyfile.template = 0 + +-- Get warp templates +LEFT JOIN warpRun AS warpTemplateRun + ON warpTemplateRun.warp_id = diffInputSkyfile.warp2 + AND diffInputSkyfile.warp2 IS NOT NULL +LEFT JOIN warpSkyfile AS warpTemplateSkyfile + ON warpTemplateSkyfile.warp_id = warpTemplateRun.warp_id + AND warpTemplateSkyfile.skycell_id = diffInputSkyfile.skycell_id + +-- Get warp inputs +LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 + AND diffInputSkyfile.warp1 IS NOT NULL +LEFT JOIN warpSkyfile + ON warpSkyfile.warp_id = warpRun.warp_id + AND warpSkyfile.skycell_id = diffInputSkyfile.skycell_id + +-- Get stack templates +LEFT JOIN stackRun AS stackTemplateRun + ON stackTemplateRun.stack_id = diffInputSkyfile.stack2 + AND diffInputSkyfile.stack2 IS NOT NULL +LEFT JOIN stackSumSkyfile AS stackTemplateSkyfile + ON stackTemplateSkyfile.stack_id = stackTemplateRun.stack_id + +-- Get stack inputs +LEFT JOIN stackRun + ON stackRun.stack_id = diffInputSkyfile.stack1 + AND diffInputSkyfile.stack1 IS NOT NULL +LEFT JOIN stackSumSkyfile + ON stackSumSkyfile.stack_id = stackRun.stack_id + +-- Get what's already been processed +LEFT JOIN diffSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id +LEFT JOIN Label ON Label.label = diffRun.label +WHERE +-- Ready to be processed + ((diffRun.state = 'new' + AND diffSkyfile.diff_id IS NULL) + OR (diffRun.state = 'update' + AND diffSkyfile.fault = 0 + AND diffSkyfile.data_state = 'update') + ) + AND (Label.active OR Label.active IS NULL) +-- Ensure input warps are available + AND (diffInputSkyfile.warp1 IS NULL + OR (warpSkyfile.data_state = 'full' + AND warpSkyfile.fault = 0 +-- AND warpSkyfile.magicked >= 0 + AND warpSkyfile.quality = 0)) +-- Ensure input stacks are available + AND (diffInputSkyfile.stack1 IS NULL + OR (stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0)) +-- Ensure template warps are available + AND (diffInputSkyfile.warp2 IS NULL + OR (warpTemplateSkyfile.data_state = 'full' + AND warpTemplateSkyfile.fault = 0 +-- AND warpSkyfile.magicked >= 0 + AND warpTemplateSkyfile.quality = 0)) +-- Ensure template stacks are available + AND (diffInputSkyfile.stack2 IS NULL + OR (stackTemplateRun.state = 'full' + AND stackTemplateSkyfile.fault = 0 + AND stackTemplateSkyfile.quality = 0)) + Index: /branches/ipp-148_http_503_filesset_revert_fix/difftool_tosummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/difftool_tosummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/difftool_tosummary.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT DISTINCT diff_id,"GPC1" as camera,diffRun.workdir,diffRun.tess_id,diffRun.diff_mode,diffRun.state + FROM diffRun + LEFT JOIN diffSummary USING(diff_id) +WHERE diffRun.state = 'full' AND + diffSummary.projection_cell IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_SSdiff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_SSdiff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_SSdiff.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT DISTINCT + 'SSdiff' as stage, + diffRun.diff_id AS stage_id, + diffRun.magicked, + CONCAT_WS('.','SSdiff',convert(diffRun.diff_id,CHAR)) as run_tag, + diffRun.label, + diffRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM diffRun +JOIN diffInputSkyfile using(diff_id) +JOIN stackRun ON diffInputSkyfile.stack1 = stackRun.stack_id +JOIN distTarget ON distTarget.stage = 'SSdiff' + AND distTarget.filter = stackRun.filter + AND distTarget.dist_group = diffRun.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = diff_id) + AND distTarget.target_id = distRun.target_id + -- JOIN hook %s +WHERE diffInputSkyfile.stack1 IS NOT NULL AND diffInputSkyfile.stack2 IS NOT NULL + AND distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned')) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_camera.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT DISTINCT + 'camera' as stage, + camRun.cam_id as stage_id, + rawExp.exp_name as run_tag, + camRun.magicked, + camRun.label, + camRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM camRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'camera' + AND rawExp.filter = distTarget.filter + AND camRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON camRun.cam_id = distRun.stage_id + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned')) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT DISTINCT + 'chip' as stage, + chipRun.chip_id as stage_id, + chipRun.magicked, + rawExp.exp_name as run_tag, + chipRun.label, + chipRun.data_group, + chipRun.dist_group, + distTarget.target_id, + distTarget.clean +FROM chipRun +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'chip' + AND chipRun.dist_group = distTarget.dist_group + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage_id = chipRun.chip_id + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned')) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_chip_bg.sql (revision 41217) @@ -0,0 +1,25 @@ +SELECT DISTINCT + 'chip_bg' as stage, + chipBackgroundRun.chip_bg_id as stage_id, + chipBackgroundRun.magicked, + rawExp.exp_name as run_tag, + chipBackgroundRun.label, + chipBackgroundRun.data_group, + chipBackgroundRun.dist_group, + distTarget.target_id, + distTarget.clean +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget + ON distTarget.stage = 'chip_bg' + AND chipBackgroundRun.dist_group = distTarget.dist_group + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun + ON distRun.stage_id = chipBackgroundRun.chip_bg_id + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((chipBackgroundRun.state = 'full') OR (distTarget.clean AND chipBackgroundRun.state = 'cleaned')) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_diff.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT DISTINCT + 'diff' as stage, + diffRun.diff_id AS stage_id, + diffRun.magicked, + rawExp.exp_name as run_tag, + diffRun.label, + diffRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM diffRun +JOIN diffInputSkyfile using(diff_id) +JOIN warpRun on diffInputSkyfile.warp1 = warpRun.warp_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'diff' + AND distTarget.filter = rawExp.filter + AND distTarget.dist_group = diffRun.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = diff_id) + AND distTarget.target_id = distRun.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned')) + + + Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_fake.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_fake.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_fake.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT DISTINCT + 'fake' as stage, + fakeRun.fake_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + rawExp.exp_name as run_tag, + fakeRun.label, + fakeRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM fakeRun +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'fake' AND fakeRun.dist_group = distTarget.dist_group + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = fake_id) + AND distRun.target_id = distTarget.target_id +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned')) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_ff.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT DISTINCT + 'ff' as stage, + fullForceRun.ff_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'ff.$skycell_id.$ff_id' + CONCAT_WS('.', 'ff', stackRun.skycell_id, convert(fullForceRun.ff_id, CHAR)) as run_tag, + fullForceRun.label, + fullForceRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM fullForceRun +JOIN skycalRun USING(skycal_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'ff' + AND fullForceRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = ff_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND fullForceRun.state = 'full' + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + -- AND (skycalResult.fault = 0 AND skycalResult.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawExp.exp_name AS run_tag, + rawExp.magicked, + camRun.label, + camRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM rawExp +JOIN magicDSRun ON stage = 'raw' AND magicDSRun.stage_id = exp_id +JOIN camRun USING(cam_id) +JOIN distTarget ON distTarget.dist_group = camRun.dist_group AND distTarget.stage = 'raw' + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id +WHERE distTarget.state = 'enabled' -- target and intrest are enabled + AND rcInterest.state = 'enabled' + AND magicDSRun.state = 'full' -- destreaked files are available Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw_no_magic.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw_no_magic.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_raw_no_magic.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawExp.exp_name AS run_tag, + rawExp.magicked, + CAST(NULL AS CHAR(255)) AS label, + CAST(NULL AS CHAR(255)) AS data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM rawExp +JOIN distTarget ON distTarget.stage = 'raw' + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id +WHERE distTarget.state = 'enabled' -- target and intrest are enabled + AND rcInterest.state = 'enabled' Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT DISTINCT + 'sky' as stage, + staticskyRun.sky_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'sky.$skycell_id.$sky_id' + CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag, + staticskyRun.label, + staticskyRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM staticskyRun +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'sky' + AND staticskyRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = sky_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distTarget.filter = 'multi' + AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky_singlefilter.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky_singlefilter.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_sky_singlefilter.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT + 'sky' as stage, + staticskyRun.sky_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'sky.$skycell_id.$stack_id' + CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag, + staticskyRun.label, + staticskyRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM staticskyRun +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'sky' + AND staticskyRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = sky_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_skycal.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT DISTINCT + 'skycal' as stage, + skycalRun.skycal_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'skycal.$skycell_id.$skycal_id' + CONCAT_WS('.', 'skycal', stackRun.skycell_id, convert(skycalRun.skycal_id, CHAR)) as run_tag, + skycalRun.label, + skycalRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM skycalRun +JOIN skycalResult USING(skycal_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'skycal' + AND skycalRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = skycal_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((skycalRun.state = 'full') OR (distTarget.clean AND skycalRun.state = 'cleaned')) + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (skycalResult.fault = 0 AND skycalResult.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_stack.sql (revision 41217) @@ -0,0 +1,25 @@ +SELECT DISTINCT + 'stack' as stage, + stackRun.stack_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'stack.$skycell_id.$stack_id' + CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag, + stackRun.label, + stackRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM stackRun +JOIN stackSumSkyfile USING(stack_id) +JOIN distTarget ON distTarget.stage = 'stack' + AND stackRun.dist_group = distTarget.dist_group + AND stackRun.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = stack_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned')) + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT DISTINCT + 'warp' as stage, + warpRun.warp_id AS stage_id, + warpRun.magicked, + rawExp.exp_name as run_tag, + warpRun.label, + warpRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun ON fakeRun.cam_id = camRun.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget ON distTarget.stage = 'warp' + AND warpRun.dist_group = distTarget.dist_group + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = warp_id) + AND distRun.target_id = distTarget.target_id +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned')) + + + Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_definebyquery_warp_bg.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT DISTINCT + 'warp_bg' as stage, + warpBackgroundRun.warp_bg_id AS stage_id, + warpBackgroundRun.magicked, + rawExp.exp_name as run_tag, + warpBackgroundRun.label, + warpBackgroundRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun ON fakeRun.cam_id = camRun.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN distTarget + ON distTarget.stage = 'warp_bg' + AND warpBackgroundRun.dist_group = distTarget.dist_group + AND rawExp.filter = distTarget.filter +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = warp_bg_id) + AND distRun.target_id = distTarget.target_id +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND ((warpBackgroundRun.state = 'full') OR (distTarget.clean AND warpBackgroundRun.state = 'cleaned')) + + + Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_defineinterest.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_defineinterest.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_defineinterest.sql (revision 41217) @@ -0,0 +1,6 @@ +INSERT INTO rcInterest + SELECT 0, rcDestination.dest_id, distTarget.target_id, '@STATE@' + FROM distTarget + JOIN rcDestination + LEFT JOIN rcInterest USING(dest_id, target_id) + WHERE (int_id IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_listfilesets.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_listfilesets.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_listfilesets.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + fs_id, + dist_id, + distRun.label, + rcDSFileset.state AS fileset_state, + rcDSFileset.name AS fileset, + rcDestination.name AS product, + rcDestination.dbname, + rcDestination.dbhost +FROM rcDSFileset +JOIN distRun USING(dist_id) +JOIN distTarget USING(target_id, stage, clean) +JOIN rcInterest USING(target_id, dest_id) +JOIN rcDestination USING(dest_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_listinterests.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_listinterests.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_listinterests.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + int_id, + rcInterest.state AS state, + dest_id, + name AS dest_name, + rcDestination.state as dest_state, + dist_group, + filter, + stage, + clean, + target_id, + distTarget.state as target_state +FROM rcInterest + JOIN distTarget USING(target_id) + JOIN rcDestination USING(dest_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_SSdiff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_SSdiff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_SSdiff.sql (revision 41217) @@ -0,0 +1,53 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + diffSkyfile.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + diffSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + diffRun.state, + -- data_state doesn't exist yet + -- diffSkyfile.data_state, + 'full' AS data_state, + diffSkyfile.quality, + distRun.no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN diffRun ON stage_id = diff_id +JOIN diffSkyfile using(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN stackRun + ON stackRun.stack_id = diffInputSkyfile.stack1 +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND diffSkyfile.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'SSdiff' + AND distComponent.dist_id IS NULL + AND ((diffRun.magicked > 0) OR distRun.no_magic) + AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_camera.sql (revision 41217) @@ -0,0 +1,36 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + 'exposure' AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + camProcessedExp.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + camRun.state, + camRun.state AS data_state, + camProcessedExp.quality, + distRun.no_magic, + chipRun.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN camRun ON camRun.cam_id = distRun.stage_id +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +LEFT JOIN Label + ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'camera' + AND distComponent.dist_id IS NULL + AND (((clean OR (chipRun.magicked != 0)) AND (camRun.magicked > 0)) OR distRun.no_magic) + AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip.sql (revision 41217) @@ -0,0 +1,36 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + chipProcessedImfile.class_id AS component, + exp_type, + distRun.clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + chipProcessedImfile.path_base, + chipProcessedImfile.path_base as chip_path_base, + chipRun.state, + chipProcessedImfile.data_state, + chipProcessedImfile.quality, + distRun.no_magic, + chipProcessedImfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN chipRun ON chipRun.chip_id = distRun.stage_id +JOIN rawExp using(exp_id) +JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND chipProcessedImfile.class_id = distComponent.component +LEFT JOIN Label + ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'chip' + AND distComponent.dist_id IS NULL + AND ((chipRun.magicked > 0) OR distRun.no_magic) + AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_chip_bg.sql (revision 41217) @@ -0,0 +1,36 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + chipBackgroundImfile.class_id AS component, + exp_type, + distRun.clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + chipBackgroundImfile.path_base, + chipBackgroundImfile.path_base as chip_path_base, + chipBackgroundRun.state, + chipBackgroundRun.state AS data_state, + chipBackgroundImfile.quality AS quality, + distRun.no_magic, + chipBackgroundImfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN chipBackgroundRun ON chipBackgroundRun.chip_bg_id = distRun.stage_id +JOIN chipRun using(chip_id) +JOIN rawExp using(exp_id) +JOIN chipBackgroundImfile ON chipBackgroundImfile.chip_bg_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND chipBackgroundImfile.class_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'chip_bg' + AND distComponent.dist_id IS NULL + AND ((chipBackgroundRun.magicked > 0) OR distRun.no_magic) + AND (chipBackgroundRun.state = 'full') + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_diff.sql (revision 41217) @@ -0,0 +1,43 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + diffSkyfile.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + diffSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + diffRun.state, + -- data_state doesn't exist yet + -- diffSkyfile.data_state, + 'full' AS data_state, + diffSkyfile.quality, + distRun.no_magic, + diffSkyfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN diffRun ON stage_id = diff_id +JOIN diffSkyfile using(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND diffSkyfile.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'diff' + AND distComponent.dist_id IS NULL + AND ((diffRun.magicked > 0) OR distRun.no_magic) + AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_fake.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_fake.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_fake.sql (revision 41217) @@ -0,0 +1,35 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + fakeProcessedImfile.class_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fakeProcessedImfile.path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + fakeRun.state, + fakeRun.state AS data_state, + 0 as quality, + distRun.no_magic, + 0 as magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id +JOIN fakeProcessedImfile USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND fakeProcessedImfile.class_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'fake' + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_ff.sql (revision 41217) @@ -0,0 +1,86 @@ +-- look for fullForceResult components and a summary components +SELECT * FROM +( +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + CONCAT_WS('.', warp_id, stackRun.skycell_id) AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fullForceResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + fullForceRun.state, + 'full' AS data_state, + fullForceResult.quality, + distRun.no_magic, + warpRun.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN fullForceRun ON stage_id = ff_id +JOIN skycalRun USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN fullForceResult USING(ff_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND CONCAT_WS('.', warp_id, stackRun.skycell_id) = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'ff' + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) +UNION +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + 'summary' AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fullForceSummary.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + fullForceRun.state, + 'full' AS data_state, + fullForceSummary.quality, + distRun.no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN fullForceRun ON stage_id = ff_id +JOIN fullForceSummary USING(ff_id) +JOIN skycalRun USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND 'summary' = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'ff' + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) +) as distRun +-- where arguments are appended here +WHERE 1 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_raw.sql (revision 41217) @@ -0,0 +1,172 @@ +SELECT * FROM ( +-- rawExp magicked with re_place +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawImfile.class_id AS component, + rawExp.exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + -- XXX: replace this with rawImfile.path_base once it exists + TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, + CAST(NULL AS CHAR(255)) AS alt_path_base, + -- pass camera stage path base since we want the camera mask file. The script knows what to do + camProcessedExp.path_base as chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN rawExp ON rawExp.exp_id = stage_id AND distTarget.stage = 'raw' AND distRun.alternate = 0 +JOIN rawImfile USING(exp_id) +JOIN chipRun ON chipRun.exp_id = rawExp.exp_id AND chipRun.magicked +JOIN chipProcessedImfile + USING(chip_id, class_id) +JOIN camRun USING(chip_id) +JOIN camProcessedExp using(cam_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.stage = 'raw' + AND distComponent.dist_id IS NULL + -- AND (rawExp.magicked OR distRun.no_magic) + AND rawExp.magicked + AND rawImfile.ignored = 0 + -- need to have magicked the chip image which makes the camera mask + AND chipProcessedImfile.magicked != 0 + AND camRun.magicked > 0 + AND (Label.active OR Label.active IS NULL) +UNION +SELECT + -- raw images no_magic required + distRun.dist_id, + distRun.label, + distTarget.dist_group, + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawImfile.class_id AS component, + rawExp.exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + -- XXX: replace this with rawImfile.path_base once it exists + TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, + CAST(NULL AS CHAR(255)) AS alt_path_base, + -- pass camera stage path base since we want the camera mask file. The script knows what to do + CAST(NULL AS CHAR(255)) AS chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN rawExp ON rawExp.exp_id = stage_id + AND distTarget.stage = 'raw' AND distRun.alternate = 0 +JOIN rawImfile USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.stage = 'raw' + AND distComponent.dist_id IS NULL + AND (distRun.no_magic) + AND rawImfile.ignored = 0 + AND (Label.active OR Label.active IS NULL) +UNION + -- raw stage alternate inputs +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + 'raw' AS stage, + rawExp.exp_id AS stage_id, + rawImfile.class_id AS component, + rawExp.exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + -- XXX: replace this with rawImfile.path_base once it exists + TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, + magicDSFile.backup_path_base AS alt_path_base, + -- pass camera stage path base since we want the camera mask file. The script knows what to do + camProcessedExp.path_base AS chip_path_base, + CAST('full' AS CHAR(255)) AS state, + CAST('full' AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN rawExp ON rawExp.exp_id = stage_id AND distTarget.stage = 'raw' AND distRun.alternate = 1 +JOIN rawImfile USING(exp_id) +JOIN magicDSRun ON magicDSRun.stage = 'raw' AND magicDSRun.stage_id = rawExp.exp_id +JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND rawImfile.class_id = magicDSFile.component +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipProcessedImfile USING(chip_id, class_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distComponent.dist_id IS NULL + AND chipProcessedImfile.magicked != 0 + AND camRun.magicked > 0 + AND rawImfile.ignored = 0 + AND (Label.active OR Label.active IS NULL) +UNION + -- raw stage clean (dbinfo only) +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + 'raw' AS stage, + rawExp.exp_id AS stage_id, + 'exposure' AS component, + rawExp.exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + CAST(NULL AS CHAR(255)) AS path_base, + CAST(NULL AS CHAR(255)) AS alt_path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawExp.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN rawExp ON rawExp.exp_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'raw' + AND distRun.clean + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) +) AS distRun +WHERE 1 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_sky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_sky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_sky.sql (revision 41217) @@ -0,0 +1,54 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + stackRun.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + staticskyResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + staticskyRun.state, + staticskyRun.state AS data_state, + staticskyResult.quality, + 1 AS no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN staticskyRun + ON stage_id = sky_id +JOIN staticskyResult + USING(sky_id) +JOIN staticskyInput + USING(sky_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'sky' + AND distComponent.dist_id IS NULL + AND (staticskyRun.state = 'full' OR (distRun.clean AND staticskyRun.state = 'cleaned')) + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_skycal.sql (revision 41217) @@ -0,0 +1,52 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + stackRun.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + skycalResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + skycalRun.state, + skycalRun.state AS data_state, + skycalResult.quality, + 1 AS no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN skycalRun + ON stage_id = skycal_id +JOIN skycalResult + USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'skycal' + AND distComponent.dist_id IS NULL + AND (skycalRun.state = 'full' OR (distRun.clean AND skycalRun.state = 'cleaned')) + AND (skycalResult.fault = 0 AND skycalResult.quality = 0) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_stack.sql (revision 41217) @@ -0,0 +1,54 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + stackRun.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + stackSumSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + stackRun.state, + -- data_state doesn't exist yet + -- stackSumSkyfile.data_state, + 'full' AS data_state, + stackSumSkyfile.quality, + 1 AS no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN stackRun + ON stage_id = stack_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'stack' + AND distComponent.dist_id IS NULL + AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp.sql (revision 41217) @@ -0,0 +1,38 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + warpSkyfile.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + warpSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + warpRun.state, + warpSkyfile.data_state, + warpSkyfile.quality, + distRun.no_magic, + warpSkyfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN warpRun ON stage_id = warp_id +JOIN warpSkyfile using(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND warpSkyfile.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'warp' + AND distComponent.dist_id IS NULL + AND ((warpRun.magicked > 0) OR distRun.no_magic) + AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pending_warp_bg.sql (revision 41217) @@ -0,0 +1,39 @@ +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + warpBackgroundSkyfile.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + warpBackgroundSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + warpBackgroundRun.state, + warpBackgroundRun.state as data_state, + 0 as quality, + distRun.no_magic, + warpBackgroundSkyfile.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN warpBackgroundRun ON stage_id = warp_bg_id +JOIN warpBackgroundSkyfile using(warp_bg_id) +JOIN warpRun using(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND warpBackgroundSkyfile.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'warp_bg' + AND distComponent.dist_id IS NULL + AND ((warpBackgroundRun.magicked > 0) OR distRun.no_magic) + AND (warpBackgroundRun.state = 'full') + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingcleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingcleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingcleanup.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + dist_id, + stage, + stage_id, + label, + outdir +FROM distRun +WHERE state = 'goto_cleaned' + AND distRun.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingdest.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingdest.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingdest.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT DISTINCT + rcDestination.*, + count(fs_id) as pending_fs +FROM rcDestination +JOIN rcRun using(dest_id) +WHERE rcRun.state = 'new' + AND rcDestination.state = 'enabled' + AND status_uri IS NOT NULL + -- where hook %s +GROUP BY dest_id +HAVING count(fs_id) > 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingfileset.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingfileset.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_pendingfileset.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + dist_id, + target_id, + distRun.stage, + distRun.outdir as dist_dir, + stage_id, + distRun.data_group, + distTarget.filter, + rcDestination.name AS product_name, + rcDestination.dest_id, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost +FROM rcDestination +JOIN rcInterest USING(dest_id) +JOIN distTarget USING(target_id) +JOIN distRun USING(target_id) +LEFT JOIN rcDSFileset USING(dest_id, dist_id) +WHERE distRun.state = 'full' + AND rcDestination.state = 'enabled' + AND distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND rcDSFileset.fs_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_processedcomponent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_processedcomponent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_processedcomponent.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + dist_id, + target_id, + stage, + stage_id, + component, + distComponent.outdir, + bytes, + md5sum, + name, + distComponent.state, + no_magic, + distComponent.fault +FROM distRun +JOIN distComponent USING(dist_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_queuercrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_queuercrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_queuercrun.sql (revision 41217) @@ -0,0 +1,20 @@ +-- create rcRun's for all destinations that have filesets that they are interested in available +INSERT INTO rcRun +SELECT -- rows in this select must match rcRun + 0, -- rc_id (auto-increment) + fs_id, + dest_id, + 'new', + NULL, -- status_fs + NULL, -- registered (database sets this to CURRENT_TIMESTAMP) + 0 -- fault + +FROM rcDestination +JOIN rcInterest USING(dest_id) +JOIN distTarget USING(target_id) +JOIN distRun USING(target_id, stage) +JOIN rcDSFileset USING(dest_id, dist_id) +LEFT JOIN rcRun using(fs_id, dest_id) +WHERE rcRun.rc_id IS NULL + AND rcDSFileset.state = 'full' + AND rcDSFileset.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_rerun_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_rerun_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_rerun_select.sql (revision 41217) @@ -0,0 +1,2 @@ +SELECT * FROM distRun +WHERE state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertcomponent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertcomponent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertcomponent.sql (revision 41217) @@ -0,0 +1,5 @@ +DELETE FROM distComponent +USING distComponent, distRun +WHERE distComponent.dist_id = distRun.dist_id + AND distRun.state = 'new' + AND distComponent.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertfileset.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertfileset.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertfileset.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE FROM rcDSFileset +USING rcDSFileset, distRun +WHERE distRun.dist_id = rcDSFileset.dist_id + AND rcDSFileset.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrcrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrcrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrcrun.sql (revision 41217) @@ -0,0 +1,3 @@ +UPDATE rcRun SET fault = 0 +WHERE fault != 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_revertrun.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE distRun +SET distRun.fault = 0 +WHERE (distRun.state = 'new' OR distRun.state = 'goto_cleaned') + AND distRun.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_toadvance.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_toadvance.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_toadvance.sql (revision 41217) @@ -0,0 +1,340 @@ +SELECT DISTINCT + dist_id, + stage, + stage_id, + CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir, + label, + clean +FROM + ( +-- raw stage not clean +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN rawImfile ON stage_id = rawImfile.exp_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = rawImfile.class_id + WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.fault = 0 + AND distRun.stage = 'raw' + AND rawImfile.ignored = 0 + GROUP BY + distRun.dist_id, + rawImfile.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- clean distribution of raw files (dbinfo only) only 1 component +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.clean + AND distRun.fault = 0 + AND distRun.stage = 'raw' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION +-- chip stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN chipProcessedImfile ON stage_id = chipProcessedImfile.chip_id + LEFT JOIN distComponent + ON distComponent.dist_id = distRun.dist_id + AND distComponent.component = chipProcessedImfile.class_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'chip' + GROUP BY + dist_id, + chipProcessedImfile.chip_id + HAVING + COUNT(chipProcessedImfile.class_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- chip_bg stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN chipBackgroundImfile ON stage_id = chipBackgroundImfile.chip_bg_id + LEFT JOIN distComponent + ON distComponent.dist_id = distRun.dist_id + AND distComponent.component = chipBackgroundImfile.class_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'chip_bg' + GROUP BY + dist_id, + chipBackgroundImfile.chip_bg_id + HAVING + COUNT(chipBackgroundImfile.class_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- camera stage +SELECT distRun.dist_id, + stage, + stage_id, + outroot, + distRun.label, + clean + FROM distRun + JOIN camRun ON stage_id = cam_id + JOIN chipRun USING(chip_id) + LEFT JOIN distComponent USING(dist_id) + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distComponent.fault = 0 + AND distRun.stage = 'camera' +-- AND ((chipRun.magicked AND camRun.state = 'full') OR distRun.no_magic) +-- AND (camRun.state = 'full' OR (distRun.clean and camRun.state = 'cleaned')) +UNION +-- fake stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN fakeProcessedImfile ON stage_id = fakeProcessedImfile.fake_id + LEFT JOIN distComponent + ON distComponent.dist_id = distRun.dist_id + AND distComponent.component = fakeProcessedImfile.class_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'fake' + GROUP BY + dist_id, + fakeProcessedImfile.fake_id + HAVING + COUNT(fakeProcessedImfile.class_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- warp stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN warpSkyfile on stage_id = warp_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = warpSkyfile.skycell_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'warp' +-- AND warpSkyfile.fault = 0 +-- AND warpSkyfile.quality = 0 + GROUP BY + distRun.dist_id, + warp_id + HAVING + COUNT(warpSkyfile.skycell_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- warp_bg stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN warpBackgroundSkyfile on stage_id = warp_bg_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = warpBackgroundSkyfile.skycell_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'warp_bg' +-- AND warpSkyfile.fault = 0 +-- AND warpSkyfile.quality = 0 + GROUP BY + distRun.dist_id, + warp_bg_id + HAVING + COUNT(warpBackgroundSkyfile.skycell_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- diff stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + distRun.label, + clean + FROM distRun + JOIN diffSkyfile + ON stage_id = diffSkyfile.diff_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = diffSkyfile.skycell_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'diff' +-- AND diffSkyfile.fault = 0 + GROUP BY + distRun.dist_id, + diff_id + HAVING + COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- stack stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN stackSumSkyfile on stage_id = stack_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'stack' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION +-- staticsky stage +-- NOTE this assumes that there is only one component per staticskyRun +-- (one skycell) +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN staticskyResult on stage_id = sky_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'sky' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION +-- skycal stage +-- NOTE this assumes that there is only one component per skycalRun +-- (one skycell) +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN skycalResult on stage_id = skycal_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'skycal' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION +-- SSdiff stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + distRun.label, + clean + FROM distRun + JOIN diffSkyfile + ON stage_id = diffSkyfile.diff_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = diffSkyfile.skycell_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'SSdiff' +-- AND diffSkyfile.fault = 0 + GROUP BY + distRun.dist_id, + diff_id + HAVING + COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +UNION +-- ff stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + distRun.label, + clean + -- ,COUNT(warp_id) + -- ,COUNT(ff_id) + FROM distRun + JOIN fullForceRun ON stage_id = ff_id + JOIN fullForceResult USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = CONCAT_WS('.', warp_id, stackRun.skycell_id) + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'ff' + GROUP BY + distRun.dist_id, + ff_id + HAVING + -- number of dist components is the number of warps plus 1 (for the summary component) + COUNT(fullForceResult.ff_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 +) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_updateinterest.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_updateinterest.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_updateinterest.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE rcInterest +JOIN distTarget USING(target_id) +JOIN rcDestination USING(dest_id) +SET rcInterest.state = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/disttool_updatercrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/disttool_updatercrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/disttool_updatercrun.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE rcRun +JOIN rcDestination USING(dest_id) +JOIN rcDSFileset using(dest_id, fs_id) +SET + -- set hook %s +WHERE + rcRun.state = 'new' + Index: /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_createbundle.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_createbundle.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_createbundle.sql (revision 41217) @@ -0,0 +1,7 @@ +-- Select the information we want from the various tables +SELECT @COLLIST@ FROM rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN camProcessedExp USING(cam_id) + LEFT OUTER JOIN fakeRun USING(cam_id) + LEFT OUTER JOIN warpRun USING(fake_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_definebyquery.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_definebyquery.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_definebyquery.sql (revision 41217) @@ -0,0 +1,10 @@ +-- query to identify unique data to bundle + +SELECT DISTINCT exp_id,chip_id,cam_id,warpRun.warp_id,invalid from rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN camProcessedExp USING(cam_id) + LEFT OUTER JOIN fakeRun USING(cam_id) + LEFT OUTER JOIN warpRun USING(fake_id) + LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id) +WHERE camRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_contents.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_contents.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_contents.sql (revision 41217) @@ -0,0 +1,2 @@ +SELECT dqstats_id,exp_id,chip_id,cam_id,warp_id from dqstatsContent + WHERE invalid = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/dqstatstool_get_run.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT DISTINCT dqstatsRun.*,rawExp.camera from dqstatsRun + JOIN dqstatsContent USING(dqstats_id) + JOIN rawExp USING(exp_id) + WHERE dqstatsRun.state = 'new' AND (dqstatsRun.fault = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_exp_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_exp_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_exp_state.sql (revision 41217) @@ -0,0 +1,15 @@ +-- change state of fakeRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the constituant imfiles are in the right state +-- arguments are new state (cleaned or purged) fake_id and new state again for +-- the fakeProcessedImfile sub query +UPDATE fakeRun + SET state = '%s' + WHERE + fakeRun.fake_id = %lld + AND (SELECT + COUNT(fake_id) + FROM fakeProcessedImfile + WHERE + fakeProcessedImfile.fake_id = fakeRun.fake_id + AND data_state != '%s' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_imfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_imfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_change_imfile_data_state.sql (revision 41217) @@ -0,0 +1,14 @@ +-- handle changes in data_state. Used for the modes tocleanedimfile and topurgedimfile +-- args are new data_state, fake_id, class_id, and current expected state for fakeRun +UPDATE fakeProcessedImfile + SET + data_state = '%s' +WHERE + fake_id = %lld + AND class_id = '%s' + -- only update if fakeRun.state has the expected value + AND ( + SELECT state from fakeRun where fakeRun.fake_id = fakeProcessedImfile.fake_id + ) = '%s' + + Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_completely_processed_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_completely_processed_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_completely_processed_exp.sql (revision 41217) @@ -0,0 +1,44 @@ +-- the output of this query must match the format of fakeRun row +SELECT DISTINCT + fake_id, + cam_id, + state, + workdir, + label, + data_group, + dist_group, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + epoch, + note +FROM + (SELECT + fakeRun.*, + rawImfile.class_id as rawimfile_class_id, + fakeProcessedImfile.class_id + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + JOIN rawImfile + ON rawImfile.exp_id = rawExp.exp_id + AND rawImfile.ignored = 0 + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND rawImfile.exp_id = fakeProcessedImfile.exp_id + AND rawImfile.class_id = fakeProcessedImfile.class_id + WHERE + fakeRun.state = 'new' + GROUP BY + fakeRun.fake_id, + rawExp.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(fakeProcessedImfile.class_id) + AND SUM(fakeProcessedImfile.fault) = 0 + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + fakeRun.* +FROM fakeRun +WHERE + fakeRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_processed_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_processed_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_processed_imfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT fakeProcessedImfile.* FROM fakeProcessedImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_export_run.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT fakeRun.* FROM fakeRun Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_camrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_camrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_camrun.sql (revision 41217) @@ -0,0 +1,37 @@ +SELECT + * +FROM + (SELECT DISTINCT + camRun.cam_id, + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.sun_angle, + rawExp.comment + FROM camRun + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + camRun.state = 'full' + AND chipRun.state = 'full' +) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_pendingexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_pendingexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_find_pendingexp.sql (revision 41217) @@ -0,0 +1,34 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + fakeRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + USING(fake_id) + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeRun.state = 'new' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND fakeMask.label IS NULL + AND fakeProcessedImfile.fake_id IS NOT NULL + GROUP BY + fakeRun.fake_id + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanupimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanupimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanupimfile.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + fakeProcessedImfile.*, + fakeRun.state, + fakeRun.workdir, + fakeRun.label, + fakeRun.reduction, + fakeRun.expgroup, + fakeRun.dvodb, + fakeRun.tess_id, + fakeRun.end_stage +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +WHERE + ((fakeRun.state = 'goto_cleaned' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_scrubbed' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_purged' AND fakeProcessedImfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + fakeRun.fake_id, + rawExp.camera, + fakeRun.state +FROM fakeRun +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (fakeRun.state = 'goto_cleaned' OR fakeRun.state = 'goto_scrubbed' OR fakeRun.state = 'goto_purged') + Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_pendingimfile.sql (revision 41217) @@ -0,0 +1,46 @@ +-- the subselect is so where criteria can be specified without knowing +-- which table the field came from +SELECT DISTINCT + fakeRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, + rawExp.exp_name, + rawExp.exp_tag, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN chipProcessedImfile USING(chip_id,exp_id) +JOIN rawExp USING(exp_id) +-- ON chipProcessedImfile.exp_id = rawExp.exp_id +LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND chipProcessedImfile.class_id = fakeProcessedImfile.class_id +LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label +WHERE + ((fakeRun.state = 'new' + AND fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.class_id IS NULL + ) + OR (fakeRun.state = 'update' + AND fakeProcessedImfile.data_state = 'cleaned') + ) + AND fakeMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_processedimfile.sql (revision 41217) @@ -0,0 +1,18 @@ +SELECT DISTINCT + fakeProcessedImfile.*, + fakeRun.workdir, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +JOIN rawExp + ON fakeProcessedImfile.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append +-- conditionals too + fakeProcessedImfile.exp_id is NOT NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_queue_cam_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_queue_cam_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_queue_cam_id.sql (revision 41217) @@ -0,0 +1,20 @@ +INSERT INTO fakeRun + SElECT + 0, -- fake_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- label + '%s', -- data_group + '%s', -- dist_group + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s', -- end_stage + NULL, -- epoch + '%s' -- note + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_revertprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_revertprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_revertprocessedimfile.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM fakeProcessedImfile +USING fakeProcessedImfile, fakeRun, camRun, chipRun, rawExp +WHERE + fakeProcessedImfile.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND fakeProcessedImfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/faketool_unmasked.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/faketool_unmasked.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/faketool_unmasked.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + * +FROM + (SELECT + fakeRun.label, + count(fakeRun.fake_id) as n_fakeruns + FROM fakeRun + LEFT JOIN fakeMask + USING (label) + WHERE fakeMask.label IS NULL + GROUP BY fakeRun.label) as fakeUnmask Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + skycalRun.skycal_id, + skycalResult.path_base, + skycalRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter +FROM skycalRun + JOIN skycalResult USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + -- join hook %s +WHERE + skycalRun.state = 'full' AND skycalResult.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery_select_warps.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery_select_warps.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_definebyquery_select_warps.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT warp_id +FROM warpRun join warpSkyfile using(warp_id, tess_id) + JOIN fakeRun using(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN skycell ON skycell.tess_id = warpSkyfile.tess_id AND skycell.skycell_id = warpSkyfile.skycell_id +WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' + AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_input.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_input.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_input.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + fullForceInput.* +FROM fullForceInput Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_result.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_result.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_result.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + fullForceResult.* +FROM fullForceResult Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_run.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + fullForceRun.* +FROM fullForceRun Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_summary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_summary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_export_summary.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + fullForceSummary.* +FROM fullForceSummary Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_result.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_result.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_result.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT + fullForceResult.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + fullForceRun.workdir, + fullForceRun.sources_path_base, + CONCAT_WS('.', warp_id, skycell_id) AS dist_component, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + warpImfile.warp_skyfile_id, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera +FROM fullForceRun + JOIN fullForceResult USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN warpImfile USING(warp_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_revert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_revert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_revert.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM fullForceResult +USING fullForceResult, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceResult.ff_id + AND fullForceResult.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_revertsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_revertsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_revertsummary.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM fullForceSummary +USING fullForceSummary, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceSummary.ff_id + AND fullForceSummary.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_summary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_summary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_summary.sql (revision 41217) @@ -0,0 +1,25 @@ +SELECT DISTINCT + fullForceSummary.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + fullForceRun.workdir, + fullForceRun.sources_path_base, + 'summary' AS dist_component, + stackRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceSummary USING(ff_id) + JOIN fullForceInput USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_summaryinputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_summaryinputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_summaryinputs.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT + fullForceResult.* +FROM fullForceRun + JOIN fullForceResult USING(ff_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_toadvance.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_toadvance.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_toadvance.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + fullForceRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) + LEFT JOIN fullForceSummary USING(ff_id) +WHERE fullForceRun.state = 'new' AND fullForceSummary.ff_id IS NULL + -- WHERE hook %s + GROUP BY ff_id + HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id) + AND SUM(fullForceResult.fault) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/fftool_todo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/fftool_todo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/fftool_todo.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + fullForceRun.*, + fullForceInput.warp_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera, + warpSkyfile.path_base AS warp_path_base +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) +WHERE fullForceRun.state = 'new' + AND fullForceResult.ff_id IS NULL + AND warpSkyfile.data_state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_camerarundone.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_camerarundone.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_camerarundone.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + corr_id, + camRun.* +FROM flatcorrRun +JOIN flatcorrCamLink + USING(corr_id) +JOIN camRun + USING(cam_id) +LEFT JOIN flatcorrAddstarLink + using(corr_id, cam_id) +WHERE + flatcorrRun.state = 'new' + AND flatcorrCamLink.include = 1 + AND camRun.state = 'full' + AND flatcorrAddstarLink.corr_id is NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_chiprundone.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_chiprundone.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_chiprundone.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + corr_id, + chipRun.* +FROM flatcorrRun +JOIN flatcorrChipLink + USING(corr_id) +JOIN chipRun + USING(chip_id) +LEFT JOIN flatcorrCamLink + using(corr_id, chip_id) +WHERE + flatcorrRun.state = 'new' + AND flatcorrChipLink.include = 1 + AND chipRun.state = 'full' + AND flatcorrCamLink.corr_id is NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_completely_processed_chiprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_completely_processed_chiprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_completely_processed_chiprun.sql (revision 41217) @@ -0,0 +1,37 @@ +INSERT INTO chipRunDone +SELECT + chip_id, + exp_id +FROM (SELECT DISTINCT + chip_id, + exp_id, + state, + workdir, + workdir_state, + label, + reduction, + expgroup, + dvodb + FROM + (SELECT + chipRun.*, + rawImfile.class_id as rawimfile_class_id, + chipProcessedImfile.class_id + FROM chipRun + JOIN rawImfile + USING(exp_id) + LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'full' + GROUP BY + chipRun.chip_id, + chipRun.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + AND SUM(chipProcessedImfile.quality > 0) != COUNT(*) + ) as Foo + ) as Bar Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_create_tmp_chiprundone.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_create_tmp_chiprundone.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_create_tmp_chiprundone.sql (revision 41217) @@ -0,0 +1,3 @@ +CREATE TEMPORARY TABLE chipRunDone +(chip_id BIGINT, exp_id VARCHAR(64), PRIMARY KEY(chip_id)) ENGINE=MEMORY + Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropcamera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropcamera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropcamera.sql (revision 41217) @@ -0,0 +1,5 @@ + +UPDATE flatcorrCamLink + SET include = 0 +WHERE corr_id = %lld + AND cam_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropchip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropchip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_dropchip.sql (revision 41217) @@ -0,0 +1,5 @@ + +UPDATE flatcorrChipLink + SET include = 0 +WHERE corr_id = %lld + AND chip_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_completed_floatcorruns.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_completed_floatcorruns.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_completed_floatcorruns.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + flatcorrRun.* +FROM flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRunDone + USING(chip_id) +WHERE + flatcorrRun.state != "stop" +GROUP BY + corr_id, chip_id +HAVING + COUNT(flatcorrExp.chip_id) = COUNT(chipRunDone.chip_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_processedimfiles.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_processedimfiles.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_find_processedimfiles.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + corr_id, + chipProcessedImfile.* +FROM flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +WHERE + flatcorrRun.state = 'run' + AND chipRun.state = 'new' + Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_inputimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_inputimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_inputimfile.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + chipProcessedImfile.*, + rawExp.filelevel , + rawExp.camera, + rawExp.telescope, + rawExp.filter +FROM chipProcessedImfile +JOIN rawExp +USING (exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pending.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pending.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pending.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + * +FROM +flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRun + USING(chip_id) +JOIN rawImfile + USING(exp_id) +LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id +WHERE + flatcorrRun.state = 'run' + AND chipRun.state = 'full' +GROUP BY + chipRun.chip_id, + chipRun.exp_id +HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault > 0) = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pendingprocess.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pendingprocess.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/flatcorr_pendingprocess.sql (revision 41217) @@ -0,0 +1,39 @@ +SELECT + corr_id, + det_type, + dvodb, + camera, + filter, + state, + make_corr, + workdir, + label, + reduction, + region, + cam_count, + add_count +FROM + (SELECT + flatcorrRun.*, + flatcorrCamLink.corr_id as cam_corr_id, + count(flatcorrCamLink.cam_id) as cam_count + FROM flatcorrRun + JOIN flatcorrCamLink + USING (corr_id) + WHERE flatcorrCamLink.include = 1 + GROUP BY + flatcorrCamLink.corr_id) AS t1 +LEFT JOIN + (SELECT + flatcorrAddstarLink.corr_id as add_corr_id, + count(flatcorrAddstarLink.add_id) as add_count + FROM flatcorrAddstarLink + JOIN addRun + USING (add_id) + WHERE addRun.state = 'full' + AND flatcorrAddstarLink.include = 1 + GROUP BY + flatcorrAddstarLink.corr_id) AS t2 +ON t1.cam_corr_id = t2.add_corr_id +WHERE cam_count = add_count +AND state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_WSdiff_check.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_WSdiff_check.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_WSdiff_check.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT warp_id,warp_state,W.skycell_id,stack_id,stack_state,S.seq_id + FROM +( SELECT warp_id,skycell_id,seq_id,warpRun.state AS warp_state + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN lapExp USING(exp_id,chip_id) + JOIN lapRun USING(lap_id) + WHERE 1 + @WHERE@ +) AS W +LEFT JOIN +( SELECT stack_id,skycell_id,seq_id,stackRun.state AS stack_state + FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN lapRun ON (stackAssociationMap.sass_id = lapRun.final_sass_id) + ) AS S +ON ( (W.seq_id = S.seq_id) + AND (W.skycell_id = S.skycell_id) ) + + Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerun.sql (revision 41217) @@ -0,0 +1,35 @@ +SELECT DISTINCT + want.exp_id, + have.chip_id, + COALESCE(have.private,false) as private, + true as active, + false as pairwise + FROM + (SELECT exp_id FROM rawExp + WHERE rawExp.exp_type= 'OBJECT' AND + rawExp.dateobs >= '2009-04-01T00:00:00.000000' AND +-- Position restriction goes here. + @WHERE@ + ) AS want + LEFT JOIN + (SELECT + exp_id, + MAX(chip_id) AS chip_id, + chipRun.state, + lapExp.private + FROM lapExp JOIN chipRun USING(exp_id,chip_id) + JOIN lapRun USING(lap_id) + where chip_id IS NOT NULL + AND +-- This is the restriction to only draw from the current lapSequence + @CHIPWHERE@ +-- ( + -- (active = TRUE) OR + -- (chipRun.state = 'full') OR + -- (chipRun.state = 'new') + -- when we can do updates, put that here. +-- ) + GROUP BY exp_id +) AS have USING(exp_id) + + Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerunbyrelease.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerunbyrelease.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_definerunbyrelease.sql (revision 41217) @@ -0,0 +1,23 @@ +select + rel.exp_id AS exp_id, + rel.chip_id AS chip_id, + false AS private, + false AS active, + false AS pairwise +FROM + (select exp_id,chip_id FROM ippRelease + JOIN relExp USING(rel_id) + JOIN rawExp USING(exp_id) + WHERE + @RELWHERE@ + ) AS rel +LEFT JOIN + (select MIN(lap_id) AS lap_id, + exp_id,chip_id FROM lapRun + JOIN lapExp USING(lap_id) + WHERE + @LAPWHERE@ + GROUP BY exp_id,chip_id + ) AS lap +USING(exp_id) +WHERE lap_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_exposures.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_exposures.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_exposures.sql (revision 41217) @@ -0,0 +1,49 @@ +-- Join the diff information +-- select DISTINCT V3.*, +-- diffRun.diff_id,diffRun.state as diffRun_state, +-- coalesce(CONVERT(sum(diffSkyfile.quality != 0),SIGNED),0) AS diff_bad_quality, +-- coalesce(CONVERT(count(diffSkyfile.diff_id),SIGNED),0) AS diff_component_count +-- FROM +select V3.*,-1 as diff_id ,"full" as diffRun_state,0 as diff_bad_quality,0 AS diff_component_count FROM +-- Join the warp information +( select V2.*, + warpRun.warp_id,warpRun.state as warpRun_state, + coalesce(CONVERT(sum(warpSkyfile.quality != 0),SIGNED),0) AS warp_bad_quality, + coalesce(CONVERT(count(warpSkyfile.warp_id),SIGNED),0) AS warp_component_count, + warpRun.magicked + FROM +-- Join the camera and fake information +( select V1.*, + camRun.cam_id,camRun.state as camRun_state, + coalesce(CONVERT(sum(camProcessedExp.quality != 0),SIGNED),0) AS cam_bad_quality, + coalesce(CONVERT(count(camProcessedExp.cam_id),SIGNED),0) AS cam_component_count, + fakeRun.fake_id,fakeRun.state as fakeRun_state FROM +-- Get all the lap, exposure, chip stage information +( SELECT DISTINCT + lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group, + lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state, + chipRun.state as chipRun_state, + coalesce(CONVERT(sum(chipProcessedImfile.quality != 0),SIGNED),0) AS chip_bad_quality, + coalesce(CONVERT(count(chipProcessedImfile.chip_id),SIGNED),0) AS chip_component_count + FROM lapRun JOIN lapExp USING(lap_id) + LEFT JOIN chipRun USING(chip_id) LEFT JOIN chipProcessedImfile USING(chip_id) +WHERE @WHERE@ + GROUP BY lap_id,exp_id + ) AS V1 +-- End lap/exposure/chip + LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id) + LEFT JOIN fakeRun USING(cam_id) + GROUP BY lap_id,exp_id + ) AS V2 +-- End camera/fake + LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id) + GROUP BY lap_id,exp_id +) AS V3 +-- End warp +-- LEFT JOIN +-- (SELECT DISTINCT diff_id,warp1,warp2 FROM diffInputSkyfile) AS DI ON +-- (DI.warp1 = warp_id OR DI.warp2 = warp_id) +-- LEFT JOIN diffRun USING(diff_id) +-- LEFT JOIN diffSkyfile USING(diff_id) +GROUP BY lap_id,exp_id + Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_filtersforgroup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_filtersforgroup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_filtersforgroup.sql (revision 41217) @@ -0,0 +1,2 @@ +SELECT DISTINCT filter +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_inactiveexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_inactiveexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_inactiveexp.sql (revision 41217) @@ -0,0 +1,41 @@ +SELECT DISTINCT + D.*,diffRun.state AS diff_state,coalesce(CONVERT(sum(others.active),SIGNED),0) AS is_in_use FROM ( + SELECT DISTINCT + W.*,IFNULL(diff1.diff_id,diff2.diff_id) AS diff_id FROM ( + SELECT DISTINCT + lapRun.lap_id, lapRun.seq_id, lapRun.tess_id, lapRun.projection_cell, lapRun.filter, lapRun.state, lapRun.label, + lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, + lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state, + chipRun.state as chipRun_state, + CONVERT(sum(chipProcessedImfile.fault),SIGNED) as chip_faults, + CONVERT(sum(chipProcessedImfile.quality),SIGNED) as chip_quality, + camRun.cam_id, camRun.state as camRun_state, + CONVERT(sum(camProcessedExp.fault),SIGNED) AS cam_faults, + CONVERT(sum(camProcessedExp.quality),SIGNED) AS cam_quality, + fakeRun.fake_id, fakeRun.state as fakeRun_state, + CONVERT(sum(fakeProcessedImfile.fault),SIGNED) as fake_faults, + warpRun.warp_id, warpRun.state as warpRun_state, + CONVERT(sum(warpSkyfile.fault),SIGNED) as warp_faults, + CONVERT(sum(warpSkyfile.quality),SIGNED) as warp_quality, + warpRun.magicked + FROM lapRun JOIN lapExp USING(lap_id) + LEFT JOIN chipRun USING(chip_id) + LEFT JOIN chipProcessedImfile USING(chip_id) + LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id) + LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id) + LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id) + WHERE lapExp.active = FALSE + AND @WHERE@ + AND (warpSkyfile.quality IS NULL OR + (warpSkyfile.quality != 8007 -- known cases where quality != 0, but everything's fine. + AND warpSkyfile.quality != 3006 -- known cases where quality != 0, but everything's fine. + )) + GROUP BY lap_id,exp_id + ) AS W +-- This was unreasonably slow in testing, so that's why I'm using a subquery here. + LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1) + LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2) +) AS D +LEFT JOIN diffRun USING(diff_id) +LEFT JOIN lapExp AS others ON (D.chip_id = others.chip_id AND D.lap_id != others.lap_id) +GROUP BY lap_id,exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_listsequence.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_listsequence.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_listsequence.sql (revision 41217) @@ -0,0 +1,1 @@ +select * from lapSequence Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingchipexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingchipexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingchipexp.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT re_id,projection_cell,registered,reprocRun.state as reproc_state, tess_id, + exp_id,private,active, + chip_id,chipRun.state AS chip_state, + cam_id,camRun.state AS cam_state, + fake_id,fakeRun.state AS fake_state, + warp_id,warpRun.state AS warp_state + FROM reprocRun JOIN reprocExp USING(re_id) + LEFT JOIN chipRun USING(chip_id,tess_id) + LEFT JOIN camRun USING(chip_id,tess_id) + LEFT JOIN fakeRun USING(fake_id,tess_id) + LEFT JOIN warpRun USING(fake_id,tess_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingexp.sql (revision 41217) @@ -0,0 +1,12 @@ +select lapRun.lap_id, lapRun.seq_id, lapRun.tess_id, lapRun.projection_cell, lapRun.filter, + lapRun.state, lapRun.label, + lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, + exp_id, chip_id, pair_id, private, pairwise, active, lapExp.data_state, + dateobs, object, comment, + chipRun.state AS chip_state + FROM lapRun JOIN lapExp USING(lap_id) + JOIN rawExp USING(exp_id,filter) + LEFT JOIN chipRun USING(exp_id,chip_id) +WHERE active IS TRUE AND lapRun.fault = 0 +-- lap_id restriction here. +-- This probably needs to be sorted by dateobs. Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendinggroup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendinggroup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendinggroup.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT DISTINCT lapGroup.*, + lapRun.dist_group +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) +WHERE lapGroup.state ='new' AND lapGroup.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_pendingrun.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * from lapRun Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertgroup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertgroup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertgroup.sql (revision 41217) @@ -0,0 +1,3 @@ +UPDATE lapGroup +SET fault = 0 +WHERE fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertsass.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertsass.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_revertsass.sql (revision 41217) @@ -0,0 +1,1 @@ +UPDATE lapRun SET final_sass_id = NULL,quick_sass_id = NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/laptool_stacks.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/laptool_stacks.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/laptool_stacks.sql (revision 41217) @@ -0,0 +1,103 @@ +SELECT + lapRun.lap_id,lapRun.seq_id,lapRun.tess_id,lapRun.projection_cell,lapRun.filter, + lapRun.state,lapRun.label,lapRun.dist_group,lapRun.registered,lapRun.fault, + skycell_id, + lapRun.quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + lapRun.final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + lapRun LEFT JOIN + ( +SELECT DISTINCT + lap_id,tess_id,projection_cell,filter,skycell_id, + quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.quick_sass_id + ) AS quick LEFT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + UNION +SELECT DISTINCT + lap_id,tess_id,projection_cell,filter,skycell_id, + quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.quick_sass_id + ) AS quick RIGHT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + ) stacks ON (stacks.lap_id = lapRun.lap_id AND + stacks.projection_cell = lapRun.projection_cell AND + stacks.tess_id = lapRun.tess_id AND + stacks.filter = lapRun.filter AND + (lapRun.quick_sass_id IS NULL OR lapRun.quick_sass_id = stacks.quick_sass_id) AND + (lapRun.final_sass_id IS NULL OR lapRun.final_sass_id = stacks.final_sass_id)) +WHERE 1 @WHERE@ Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_file_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_file_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_file_data_state.sql (revision 41217) @@ -0,0 +1,8 @@ +-- handle changes in data_state. +UPDATE magicDSFile + JOIN magicDSRun USING(magic_ds_id) +SET + magicDSFile.data_state = '%s' +WHERE + magic_ds_id = %lld + AND component = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_run_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_run_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_change_run_state.sql (revision 41217) @@ -0,0 +1,15 @@ +-- change state of magicDSRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the constituant imfiles are in the right state +-- arguments are new state (cleaned or purged) chip_id and new state again for +-- the chipProcessedImfile sub query +UPDATE magicDSRun + SET magicDSRun.state = '%s' +WHERE + magicDSRun.magic_ds_id = %lld + AND (SELECT + COUNT(magic_ds_id) + FROM magicDSFile + WHERE + magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND data_state != '%s' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_clearstatefaults.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_clearstatefaults.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_clearstatefaults.sql (revision 41217) @@ -0,0 +1,1 @@ +UPDATE magicDSRun SET state = '%s', fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completed_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completed_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completed_runs.sql (revision 41217) @@ -0,0 +1,117 @@ +SELECT DISTINCT + magic_ds_id, + magicked, + re_place, + label +FROM + ( +-- raw stage +SELECT + magicDSRun.*, + rawExp.magicked + FROM magicDSRun + JOIN rawImfile ON stage_id = rawImfile.exp_id + JOIN rawExp using(exp_id) + LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id + WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + GROUP BY + magicDSRun.magic_ds_id, + stage_id + HAVING + COUNT(rawImfile.class_id) = COUNT(magicDSFile.component) + AND SUM(magicDSFile.fault) = 0 +UNION +-- chip stage +SELECT + magicDSRun.*, + chipRun.magicked + FROM magicDSRun + JOIN chipRun ON stage_id = chip_id + JOIN chipProcessedImfile USING(chip_id) + LEFT JOIN magicDSFile + ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id + WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'chip' + AND chipProcessedImfile.quality = 0 + GROUP BY + magic_ds_id, + stage_id + HAVING + COUNT(chipProcessedImfile.class_id) = COUNT(magicDSFile.component) + AND SUM(magicDSFile.fault) = 0 +UNION +-- camera stage +SELECT + magicDSRun.*, + camRun.magicked + FROM magicDSRun + JOIN camRun ON stage_id = camRun.cam_id + JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id + LEFT JOIN magicDSFile + ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND magicDSFile.component IS NOT NULL + WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camProcessedExp.quality = 0 + AND magicDSFile.fault = 0 + GROUP BY + magic_ds_id, + stage_id +UNION +-- warp stage +SELECT + magicDSRun.*, + warpRun.magicked + FROM magicDSRun + JOIN warpRun on stage_id = warp_id + JOIN warpSkyfile USING(warp_id) + LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id + WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + GROUP BY + magicDSRun.magic_ds_id, + stage_id + HAVING + COUNT(warpSkyfile.skycell_id) = COUNT(magicDSFile.component) + AND SUM(magicDSFile.fault) = 0 +UNION +-- diff stage +SELECT DISTINCT + magicDSRun.*, + diffRun.magicked + FROM magicDSRun + JOIN magicRun USING (magic_id) + JOIN magicInputSkyfile USING(magic_id) + JOIN diffRun + ON magicRun.diff_id = diffRun.diff_id + JOIN diffSkyfile + ON diffRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id + LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id + WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + GROUP BY + magicDSRun.magic_ds_id, + magicRun.magic_id + HAVING + COUNT(magicInputSkyfile.node) = COUNT(magicDSFile.component) + AND SUM(magicDSFile.fault) = 0 + + ) as magicDSRun Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completedrevert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completedrevert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_completedrevert.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + magic_ds_id, + state +FROM magicDSRun +LEFT JOIN magicDSFile USING(magic_ds_id) +WHERE + magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored' +GROUP BY magic_ds_id +HAVING COUNT(component) = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_camera.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.data_group, + camRun.cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n 1 " if rerun else "\n 0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND camRun.magicked = 0 + AND chipRun.magicked + AND chipRun.state = 'full' + AND camRun.state = 'full' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'chip' AS stage, + chip_id AS stage_id, + chipRun.data_group, + camRun.cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'chip' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND chipRun.magicked = 0 + AND chipRun.state = 'full' + AND camRun.state = 'full' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_chip_bg.sql (revision 41217) @@ -0,0 +1,33 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'chip_bg' AS stage, + chip_bg_id AS stage_id, + chipBackgroundRun.data_group, + camRun.cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun +JOIN magicMask USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile USING(diff_id) +JOIN warpRun + ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN chipBackgroundRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'chip_bg' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND chipBackgroundRun.magicked = 0 + AND chipBackgroundRun.state = 'full' + AND camRun.state = 'full' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_diff.sql (revision 41217) @@ -0,0 +1,71 @@ +SELECT * +FROM ( +SELECT + magicRun.magic_id, + exp_id, + 'diff' AS stage, + diff_id AS stage_id, + diffRun.data_group, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'diff' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND diffRun.bothways = 0 + AND diffRun.magicked = 0 + AND diffRun.state = 'full' + -- WHERE hook %s +UNION +SELECT + forwardRun.magic_id, + forwardRun.exp_id, + 'diff' AS stage, + forwardRun.diff_id AS stage_id, + forwardRun.data_group, + 0 AS cam_id, + forwardRun.label, + forwardRun.workdir, + inverseRun.magic_id AS inv_magic_id, + inverseRun.exp_id AS inv_exp_id +FROM ( + SELECT + magicRun.magic_id, + exp_id, + diff_id, + diffRun.data_group, + magicRun.label, + magicRun.workdir + FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'diff' + WHERE magicRun.state = 'full' + AND magicRun.inverse = 0 + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND diffRun.state = 'full' + AND diffRun.bothways + AND diffRun.magicked = 0 + -- WHERE hook %s +) AS forwardRun +JOIN ( + SELECT + magic_id, + exp_id, + diff_id + FROM magicRun + JOIN magicMask USING(magic_id) + WHERE magicRun.state = 'full' + AND magicRun.inverse = 1 +) AS inverseRun USING(diff_id) + +) AS Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_raw.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'raw' AS stage, + exp_id AS stage_id, + chipRun.data_group, + camRun.cam_id AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage_id = exp_id + AND magicDSRun.stage = 'raw' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND rawExp.magicked = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'warp' AS stage, + warp_id AS stage_id, + warpRun.data_group, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'warp' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND warpRun.magicked = 0 + AND warpRun.state = 'full' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp_bg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp_bg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definebyquery_warp_bg.sql (revision 41217) @@ -0,0 +1,32 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'warp_bg' AS stage, + warp_bg_id AS stage_id, + warpBackgroundRun.data_group, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun +JOIN magicMask USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile USING(diff_id) +JOIN warpRun + ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) +JOIN warpBackgroundRun USING(warp_bg_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'warp_bg' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND warpBackgroundRun.magicked = 0 + AND warpBackgroundRun.state = 'full' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_camera.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.data_group, + camRun.cam_id, + camRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun AS oldWarpRun + ON (!magicRun.inverse AND diffInputSkyfile.warp1 = oldWarpRun.warp_id) + OR ( magicRun.inverse AND diffInputSkyfile.warp2 = oldWarpRun.warp_id) + -- This ends the usual JOIN down to the raw level + JOIN rawExp + ON rawExp.exp_id = magicRun.exp_id + JOIN chipRun + ON chipRun.exp_id = rawExp.exp_id + JOIN camRun + ON camRun.chip_id = chipRun.chip_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND camRun.magicked = 0 + AND camRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_chip.sql (revision 41217) @@ -0,0 +1,35 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'chip' AS stage, + chipRun.chip_id AS stage_id, + chipRun.data_group, + camRun.cam_id, + chipRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun + ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun AS oldCamRun USING(cam_id) + -- This ends the usual JOIN down to the raw level + JOIN chipRun + ON chipRun.exp_id = magicRun.exp_id + AND chipRun.chip_id != oldCamRun.chip_id -- Not allowed to copy yourself + JOIN camRun ON + camRun.chip_id = chipRun.chip_id + JOIN rawExp + ON rawExp.exp_id = chipRun.exp_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'chip' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND chipRun.magicked = 0 + AND chipRun.state = 'full' + AND camRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_definecopy_warp.sql (revision 41217) @@ -0,0 +1,36 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'warp' AS stage, + warpRun.warp_id AS stage_id, + warpRun.data_group, + camRun.cam_id, + warpRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun AS oldWarpRun + ON (!magicRun.inverse AND diffInputSkyfile.warp1 = oldWarpRun.warp_id) + OR ( magicRun.inverse AND diffInputSkyfile.warp2 = oldWarpRun.warp_id) + -- This ends the usual JOIN down to the raw level + JOIN rawExp + ON rawExp.exp_id = magicRun.exp_id + JOIN chipRun + ON chipRun.exp_id = rawExp.exp_id + JOIN camRun + ON camRun.chip_id = chipRun.chip_id + JOIN fakeRun + ON fakeRun.cam_id = camRun.cam_id + JOIN warpRun + ON warpRun.fake_id = fakeRun.fake_id + AND warpRun.warp_id != oldWarpRun.warp_id -- Not allowed to copy yourself + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'warp' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND warpRun.magicked = 0 + AND warpRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_destreakedfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_destreakedfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_destreakedfile.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT + magicDSFile.*, + CONCAT_WS('.', CONCAT(outroot, '/', exp_id), 'mds', magic_ds_id, stage_id, component) AS path_base, + magicDSRun.magic_id, + magicDSRun.inv_magic_id, + magicDSRun.state, + magicDSRun.stage, + magicDSRun.stage_id, + magicDSRun.cam_id, + magicDSRun.label, + magicDSRun.data_group, + magicDSRun.outroot, + magicDSRun.re_place, + magicDSRun.fault as run_fault, + magicDSRun.note +FROM + magicDSRun JOIN magicDSFile USING(magic_ds_id) JOIN magicRun USING(magic_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getrunids.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getrunids.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getrunids.sql (revision 41217) @@ -0,0 +1,23 @@ +-- find the inputs for a new magicDSRun from an existing magicRun +SELECT DISTINCT + magic_id, + magicRun.exp_id, + chip_id, + cam_id, + warp_id, + diffSkyfile.diff_id +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id +JOIN warpRun + ON (diffInputSkyfile.warp1 = warp_id AND !magicRun.inverse) + OR (diffInputSkyfile.warp2 = warp_id AND magicRun.inverse) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +WHERE magic_id = %ld Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getskycells.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getskycells.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_getskycells.sql (revision 41217) @@ -0,0 +1,64 @@ +-- Input warps +SELECT DISTINCT + diffSkyfile.diff_id, + diffRun.tess_id, + diffSkyfile.skycell_id, + diffSkyfile.path_base, + diffSkyfile.data_state, + diffSkyfile.quality +FROM magicDSRun +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id + -- Want input warps only + AND diffInputSkyfile.warp1 IS NOT NULL + AND magicRun.inverse = 0 +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = diffInputSkyfile.warp1 + AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id +JOIN warpSkyfile + ON warpSkyfile.warp_id = warpSkyCellMap.warp_id + AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id + AND warpSkyfile.quality = 0 +WHERE + diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + -- WHERE hook %s +UNION +-- Reference warps +SELECT DISTINCT + diffSkyfile.diff_id, + diffRun.tess_id, + diffSkyfile.skycell_id, + diffSkyfile.path_base, + diffSkyfile.data_state, + diffSkyfile.quality +FROM magicDSRun +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id + AND diffInputSkyfile.warp2 IS NOT NULL + AND magicRun.inverse = 1 +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = diffInputSkyfile.warp2 + AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id +JOIN warpSkyfile + ON warpSkyfile.warp_id = warpSkyCellMap.warp_id + AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id + AND warpSkyfile.quality = 0 +WHERE + diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + -- WHERE hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_listrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_listrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_listrun.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT + magicDSRun.* +FROM + magicDSRun Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertdestreakedfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertdestreakedfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertdestreakedfile.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM magicDSFile USING magicDSFile, magicDSRun +WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) + AND ((magicDSRun.state = 'new' AND magicDSFile.fault != 0) + OR (magicDSRun.state = 'goto_restored' + OR magicDSRun.state = 'goto_censored' + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertupdated.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertupdated.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_revertupdated.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE magicDSRun + JOIN magicDSFile using(magic_ds_id) +SET magicDSFile.fault = 0 +WHERE + magicDSRun.state = 'update' + AND magicDSFile.data_state = 'update' + AND magicDSFile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_setfiletoupdate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_setfiletoupdate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_setfiletoupdate.sql (revision 41217) @@ -0,0 +1,9 @@ +UPDATE magicDSRun + JOIN magicDSFile USING(magic_ds_id) +SET magicDSRun.state = 'update', + magicDSFile.data_state = 'update', + magicDSFile.fault = 0 + -- set hook %s +WHERE magic_ds_id = %lld + AND (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state ='update') Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_tocleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_tocleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_tocleanup.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + magic_ds_id, + stage, + magicDSRun.state, + magicDSRun.outroot, + camera +FROM magicDSRun + JOIN magicRun USING(magic_id) + JOIN rawExp USING(exp_id) +WHERE magicDSRun.state = 'goto_cleaned' +-- XXX: need to add fault to magicDSRun +-- XXX: the database has been updated, but fault isn't yet used +-- AND magicDSRun.fault = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_camera.sql (revision 41217) @@ -0,0 +1,46 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + magicDSRun.state, + chipRun.exp_id, + camRun.magicked, + magicDSRun.label, + camera, + magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, + stage, + stage_id, + 'exposure' AS component, + 0 AS mismatched_tess, + CAST(NULL AS CHAR(255)) AS uri, + camProcessedExp.path_base, + magicRun.inverse, + camProcessedExp.path_base AS cam_path_base, + camRun.reduction AS cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun ON magicDSRun.stage_id = camRun.cam_id +JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id +LEFT JOIN Label ON magicDSRun.label = Label.label +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camRun.state = 'full' + AND ((chipRun.state = 'full' AND chipRun.magicked > 0) + OR (chipRun.state = 'cleaned' AND chipRun.magicked < 0)) + AND camProcessedExp.fault = 0 + AND camProcessedExp.quality = 0 + AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_chip.sql (revision 41217) @@ -0,0 +1,48 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + magicDSRun.state, + chipRun.exp_id, + chipRun.magicked, + magicDSRun.label, + camera, + magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, + stage, + stage_id, + class_id AS component, + 0 AS mismatched_tess, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + magicRun.inverse, + camProcessedExp.path_base AS cam_path_base, + camRun.reduction AS cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun ON chipRun.chip_id = stage_id +JOIN chipProcessedImfile ON chipRun.chip_id = chipProcessedImfile.chip_id +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id +LEFT JOIN Label + ON magicDSRun.label = Label.label +WHERE + ((magicDSRun.state = 'new' AND magicDSFile.component IS NULL) + OR (magicDSRun.state = 'update' AND magicDSFile.data_state = 'update' + AND magicDSFile.fault = 0)) + AND magicDSRun.stage = 'chip' + AND (chipRun.state = 'full' OR (chipRun.state = 'update' and chipProcessedImfile.data_state = 'full')) + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_diff.sql (revision 41217) @@ -0,0 +1,100 @@ +SELECT * FROM ( +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicDSRun.state, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, + stage, + magicRun.diff_id AS stage_id, + diffRun.magicked, + diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) AS cam_path_base, + CAST(NULL AS CHAR(255)) AS cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +LEFT JOIN Label ON magicDSRun.label = Label.label +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicDSRun.state, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base, + stage, + magicRun.diff_id AS stage_id, + diffRun.magicked, + diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) AS cam_path_base, + CAST(NULL AS CHAR(255)) AS cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +LEFT JOIN Label ON magicDSRun.label = Label.label +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) +) AS magicDSRun +-- we need the following so this query is compatible with the other stages +WHERE magic_ds_id IS NOT NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_raw.sql (revision 41217) @@ -0,0 +1,44 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicDSRun.state, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + magicMask.path_base as streaks_path_base, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, + stage, + stage_id, + rawExp.magicked, + class_id as component, + 0 AS mismatched_tess, + rawImfile.uri AS uri, + -- XXX: replace this with rawImfile.path_base once it exists + TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + camRun.reduction AS cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id +LEFT JOIN Label ON magicDSRun.label = Label.label +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) + Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_todestreak_warp.sql (revision 41217) @@ -0,0 +1,46 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicDSRun.state, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + magicMask.path_base as streaks_path_base, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, + stage, + stage_id, + warpRun.magicked, + warpSkyfile.skycell_id as component, + diffRun.tess_id != warpRun.tess_id AS mismatched_tess, + diffRun.tess_id as diff_tess_id, + warpSkyfile.uri, + warpSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + CAST(NULL AS CHAR(255)) as cam_reduction, + outroot, + recoveryroot, + re_place, + remove, + IFNULL(Label.priority, 10000) AS priority +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN diffRun USING(diff_id) +JOIN warpRun ON warp_id = stage_id +JOIN warpSkyfile USING(warp_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id +LEFT JOIN Label ON magicDSRun.label = Label.label +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_toremove.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_toremove.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_toremove.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + magic_ds_id, + stage, + stage_id, + component, + backup_path_base +FROM magicDSFile +JOIN magicDSRun USING(magic_ds_id) +WHERE magicDSRun.remove + AND magicDSRun.state = 'full' + AND backup_path_base IS NOT NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_camera.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT + magic_ds_id, + magicDSRun.state, + exp_id, + re_place, + camera, + stage, + stage_id, + component, + outroot, + camProcessedExp.path_base, + camProcessedExp.path_base AS cam_path_base, + recovery_path_base, + 0 AS bothways, + 0 AS bytes, + 0 AS md5sum, + camRun.magicked +FROM magicDSRun + JOIN magicDSFile using(magic_ds_id) + JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id + JOIN camRun ON camRun.cam_id = camProcessedExp.cam_id + JOIN chipRun USING(chip_id) + JOIN rawExp using(exp_id) +WHERE magicDSRun.stage = 'camera' + AND camProcessedExp.fault = 0 + AND camRun.state = 'full' + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND (backup_path_base IS NOT NULL) + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_chip.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT + magic_ds_id, + magicDSRun.state, + rawExp.exp_id, + re_place, + camera, + stage, + stage_id, + component, + outroot, + chipProcessedImfile.path_base, + camProcessedExp.path_base AS cam_path_base, + recovery_path_base, + 0 AS bothways, + 0 AS bytes, + 0 AS md5sum, + chipRun.magicked +FROM magicDSRun + JOIN magicDSFile using(magic_ds_id) + JOIN camProcessedExp using(cam_id) + JOIN chipRun ON (stage_id = chipRun.chip_id) + JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id) + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +WHERE magicDSRun.stage = 'chip' + AND (((magicDSRun.state = 'new' OR (magicDSRun.state = 'update')) + AND magicDSFile.fault > 0) + -- why don't we require a fault for these states? + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_diff.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT + magic_ds_id, + magicDSRun.state, + exp_id, + re_place, + camera, + stage, + stage_id, + component, + outroot, + diffSkyfile.path_base, + magicDSFile.backup_path_base, + magicDSFile.recovery_path_base, + "NULL" AS cam_path_base, + CAST(diffRun.bothways AS SIGNED) AS bothways, + 0 AS bytes, + 0 AS md5sum, + diffRun.magicked +FROM magicDSRun + JOIN magicRun using(magic_id) + JOIN magicDSFile USING(magic_ds_id) + JOIN diffSkyfile ON (stage_id = diffSkyfile.diff_id AND component = skycell_id) + JOIN diffRun ON diffSkyfile.diff_id = diffRun.diff_id + JOIN rawExp USING(exp_id) +WHERE magicDSRun.stage = 'diff' + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_raw.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT + magic_ds_id, + magicDSRun.state, + exp_id, + re_place, + rawImfile.camera, + stage, + stage_id, + component, + outroot, + rawImfile.uri AS path_base, + CAST(NULL AS CHAR(255)) AS cam_path_base, + recovery_path_base, + 0 AS bothways, + bytes, + md5sum, + 0 AS magicked +FROM magicDSRun + JOIN magicDSFile using(magic_ds_id) + JOIN rawImfile ON (stage_id = exp_id AND component = class_id) +WHERE magicDSRun.stage = 'raw' + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magicdstool_torevert_warp.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT + magic_ds_id, + magicDSRun.state, + exp_id, + re_place, + camera, + stage, + stage_id, + component, + outroot, + warpSkyfile.path_base, + "NULL" AS cam_path_base, + recovery_path_base, + 0 AS bothways, + 0 AS bytes, + 0 AS md5sum, + warpRun.magicked +FROM magicDSRun + JOIN magicDSFile USING(magic_ds_id) + JOIN warpSkyfile ON (stage_id = warp_id AND component = skycell_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun ON fakeRun.cam_id = camRun.cam_id + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE magicDSRun.stage = 'warp' + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_chipprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_chipprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_chipprocessedimfile.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT DISTINCT + chip_id, + class_id, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + camProcessedExp.path_base as cam_path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffInputSkyfile.skycell_id = diffRun.skycell_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = warpRun.warp_id + AND warpSkyCellMap.skycell_id = diffRun.skycell_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN chipProcessedImfile USING(chip_id,class_id) +WHERE + chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 +-- AND magicRun.state = 'full' +-- AND magicMask.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_create_tmp_warpcomplete.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_create_tmp_warpcomplete.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_create_tmp_warpcomplete.sql (revision 41217) @@ -0,0 +1,3 @@ +CREATE TEMPORARY TABLE warpComplete + (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), + PRIMARY KEY(warp_id, skycell_id, tess_id)) ENGINE=MEMORY; Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_insert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_insert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_insert.sql (revision 41217) @@ -0,0 +1,10 @@ +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + skycell_id +FROM diffSkyfile +WHERE + diff_id = @DIFF_ID@ -- Update this with the appropriate diff_id + AND fault = 0 + AND quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select.sql (revision 41217) @@ -0,0 +1,52 @@ +-- Get a list of exposures on which magic may be performed +SELECT DISTINCT + chipRun.exp_id, + MAX(diffWarps.diff_id) AS diff_id, + -- The following trick pulls out the 'inverse' value for the maximum diff_id + CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id DESC), ',', 1), UNSIGNED) AS inverse, + diff_data_group +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id, + 0 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id, + 1 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN + (SELECT magic_id, exp_id, label + FROM magicRun + -- rerun hook %s + ) AS oldMagicRun + ON oldMagicRun.exp_id = chipRun.exp_id +-- WHERE hook %s +GROUP BY chipRun.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select_multidiff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select_multidiff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_definebyquery_select_multidiff.sql (revision 41217) @@ -0,0 +1,53 @@ +-- Get a list of exposures on which magic may be performed +SELECT DISTINCT + chipRun.exp_id, + MAX(diffWarps.diff_id) AS diff_id, + -- The following trick pulls out the 'inverse' value for the maximum diff_id + CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id DESC), ',', 1), UNSIGNED) AS inverse, + diff_data_group +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id, + 0 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id, + 1 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN + (SELECT magic_id, exp_id, diff_id, label + FROM magicRun + -- rerun hook %s + ) AS oldMagicRun + ON oldMagicRun.exp_id = chipRun.exp_id + AND oldMagicRun.diff_id = diffWarps.diff_id +-- WHERE hook %s +GROUP BY chipRun.exp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_deletemask.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_deletemask.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_deletemask.sql (revision 41217) @@ -0,0 +1,9 @@ +DELETE magicMask +FROM magicRun + JOIN magicNodeResult USING(magic_id) + JOIN magicMask USING(magic_id) +WHERE + magicRun.state = 'new' + AND magicNodeResult.node = 'root' + AND magicNodeResult.fault != 0 + AND magicRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_diffskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_diffskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_diffskyfile.sql (revision 41217) @@ -0,0 +1,28 @@ +SELECT DISTINCT + diffSkyfile.diff_id, + diffRun.skycell_id, + diffSkyfile.uri, + diffSkyfile.path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffInputSkyfile.skycell_id = diffRun.skycell_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = diffInputSkyfile.warp_id + AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id +JOIN warpSkyfile + ON warpSkyfile.warp_id = warpSkyCellMap.warp_id + AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id + AND warpSkyfile.quality = 0 +WHERE + diffRun.state = 'full' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 +-- AND magicRun.state = 'full' +-- AND magicMask.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_exposure.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_exposure.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_exposure.sql (revision 41217) @@ -0,0 +1,54 @@ +SELECT * +FROM ( + -- Forward diff + SELECT DISTINCT + magicRun.magic_id, + rawExp.*, + camProcessedExp.path_base, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev, + camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, + camProcessedExp.iq_m2, + camProcessedExp.iq_m3, + camProcessedExp.iq_m4 + FROM + magicRun + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 + AND magicRun.inverse = 0 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id,exp_id) + JOIN rawExp USING(exp_id) + -- WHERE hook %s + UNION + -- Backward diff + SELECT DISTINCT + magicRun.magic_id, + rawExp.*, + camProcessedExp.path_base, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev, + camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, + camProcessedExp.iq_m2, + camProcessedExp.iq_m3, + camProcessedExp.iq_m4 + FROM + magicRun + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp2 + AND magicRun.inverse = 1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id,exp_id) + JOIN rawExp USING(exp_id) + -- WHERE hook %s +) AS magicExposures Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputs.sql (revision 41217) @@ -0,0 +1,73 @@ +-- Get input details for a magic node +SELECT + magicRun.magic_id, + magicRun.inverse, -- Using the inverse subtraction? + magicRun.diff_id, + magicInputs.node, + -- Only one of diff_path_base and magic_path_base should be non-NULL + -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL + magicInputs.diff_path_base, -- path_base for the diff (if any) + magicInputs.warp_path_base, -- path_base for the template warp (if any) + magicInputs.stack_path_base, -- path_base for the template stack (if any) + magicInputs.magic_path_base -- path_base for child nodes (if any) +FROM magicRun +JOIN ( + -- Single skycells: have uri=NULL + SELECT + magic_id, + magicRun.diff_id, + skycell_id AS node, + diffSkyfile.path_base AS diff_path_base, + warpSkyfile.path_base AS warp_path_base, + stackSumSkyfile.path_base AS stack_path_base, + NULL AS magic_path_base + FROM magicRun + JOIN magicTree USING(magic_id) + JOIN magicInputSkyfile USING(magic_id, node) + JOIN diffRun USING(diff_id) + JOIN diffSkyfile + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node + JOIN ( + -- Template for non-inverse + SELECT + magic_id, + skycell_id, + warp2 AS warp_id, + stack2 AS stack_id + FROM magicRun + JOIN diffInputSkyfile USING(diff_id) + WHERE magicRun.inverse = 0 + -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s + UNION + -- Template for inverse + SELECT + magic_id, + skycell_id, + warp1 AS warp_id, + stack1 AS stack_id + FROM magicRun + JOIN diffInputSkyfile USING(diff_id) + WHERE magicRun.inverse = 1 + -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s + ) AS diffTemplates USING(magic_id, skycell_id) + LEFT JOIN warpSkyfile USING(warp_id, skycell_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + -- WHERE hook (magicRun.magic_id, magicTree.node) %s + UNION + -- Merged skycells: have diff_id=0, various_path_base=NULL + SELECT + magicTree.magic_id, + 0 AS diff_id, + magicTree.dep, + NULL AS diff_path_base, + NULL AS warp_path_base, + NULL AS stack_path_base, + magicNodeResult.path_base AS magic_path_base + FROM magicTree + JOIN magicRun USING(magic_id) + JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.dep = magicNodeResult.node + -- WHERE hook (magicRun.magic_id, magicTree.node) %s + ) AS magicInputs USING(magic_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_inputskyfile.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + magicInputSkyfile.*, + magicRun.inverse, + diffSkyfile.path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffSkyfile + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node +WHERE + magicRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_mask.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_mask.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_mask.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + magicMask.*, + magicRun.exp_id +FROM magicMask +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'full' + AND magicMask.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_rawimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_rawimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_rawimfile.sql (revision 41217) @@ -0,0 +1,27 @@ +SELECT DISTINCT + exp_id, + class_id, + rawImfile.uri, + camProcessedExp.path_base as cam_path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = warpRun.warp_id + AND warpSkyCellMap.skycell_id = diffRun.skycell_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id,exp_id) +JOIN rawExp USING(exp_id) +JOIN rawImfile USING(exp_id,class_id) +WHERE + rawImfile.fault = 0 +-- AND magicRun.state = 'full' +-- AND magicMask.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_camera.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_camera.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_camera.sql (revision 41217) @@ -0,0 +1,6 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN camRun ON stage = 'camera' AND stage_id = camRun.cam_id +SET camRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_chip.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN chipRun ON stage = 'chip' AND stage_id = chip_id + JOIN chipProcessedImfile USING(chip_id) +SET chipProcessedImfile.magicked = 0, + chipRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_diff.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN diffRun ON stage = 'diff' AND stage_id = diffRun.diff_id + JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id +SET diffRun.magicked = 0, + diffSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_raw.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_raw.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_raw.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN rawExp ON stage = 'raw' AND stage_id = rawExp.exp_id + JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id +SET rawImfile.magicked = 0, + rawExp.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_restore_warp.sql (revision 41217) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN warpRun ON stage = 'warp' AND stage_id = warp_id + JOIN warpSkyfile using(warp_id) +SET warpRun.magicked = 0, + warpSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_revertnode.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_revertnode.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_revertnode.sql (revision 41217) @@ -0,0 +1,5 @@ +DELETE magicNodeResult +FROM magicNodeResult +JOIN magicRun USING(magic_id) +WHERE magicNodeResult.fault != 0 + AND magicRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_setfull.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_setfull.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_setfull.sql (revision 41217) @@ -0,0 +1,6 @@ +UPDATE + magicRun +SET + state = 'full' +WHERE + state != 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_setgotocleaned.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_setgotocleaned.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_setgotocleaned.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE magicRun + SET workdir_state = 'goto_cleaned' +WHERE workdir_state = 'dirty' + Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_tocleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_tocleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_tocleanup.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT magic_id, + magicRun.exp_id, + rawExp.camera, + magicRun.workdir, + IFNULL(priority, 10000) as priority +FROM magicRun + JOIN rawExp USING(exp_id) + LEFT JOIN Label ON magicRun.label = Label.label +WHERE magicRun.workdir_state = 'goto_cleaned' + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_tomask.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_tomask.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_tomask.sql (revision 41217) @@ -0,0 +1,18 @@ +SELECT + magic_id, + exp_id, + camera, + magicRun.workdir, + uri +FROM magicRun +JOIN rawExp USING(exp_id) +JOIN magicTree USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +WHERE + magicRun.state = 'new' + AND magicNodeResult.node = 'root' + AND magicNodeResult.fault = 0 +GROUP BY + magic_id +HAVING + COUNT(magicTree.node) = COUNT(magicNodeResult.node) Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_inputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_inputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_inputs.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT + magicTree.*, + magicRun.workdir, + rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, + rawExp.camera, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE AS done, + magicNodeResult.fault IS TRUE AS bad, + IFNULL(Label.priority, 10000) AS priority +FROM magicRun +JOIN magicTree USING(magic_id) +JOIN magicInputSkyfile USING(magic_id, node) +JOIN rawExp USING(exp_id) +JOIN diffSkyfile -- only get nodes that match a skycell + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node +JOIN diffRun + ON diffRun.diff_id = diffSkyfile.diff_id +LEFT JOIN magicNodeResult USING(magic_id, node) +LEFT JOIN Label ON magicRun.label = Label.label +WHERE + magicRun.state = 'new' + AND magicNodeResult.magic_id IS NULL + AND magicNodeResult.node IS NULL + AND diffRun.state = 'full' + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_runs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_runs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_runs.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + magic_id, + IFNULL(priority, 10000) AS priority +FROM magicTree +JOIN magicRun USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +LEFT JOIN Label ON magicRun.label = Label.label +WHERE + magicRun.state = 'new' +AND magicNodeResult.magic_id IS NULL +-- WHERE hook %s +ORDER BY + priority DESC, magicRun.magic_id Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_tree.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_tree.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_toprocess_tree.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + magicTree.*, + magicRun.workdir, + rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, + rawExp.camera, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE AS done, + magicNodeResult.fault IS TRUE AS bad, + IFNULL(Label.priority, 10000) AS priority +FROM magicTree +JOIN magicRun USING(magic_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +LEFT JOIN Label ON magicRun.label = Label.label +WHERE + magicRun.state = 'new' + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s +ORDER BY + priority, magicRun.magic_id Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_toskyfilemask.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_toskyfilemask.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_toskyfilemask.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT + magicMask.* +FROM magicRun +JOIN magicMask + USING(magic_id) +LEFT JOIN magicSkyfileMask + USING(magic_id) +WHERE + magicRun.state = 'new' + AND magicSkyfileMask.magic_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_totree.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_totree.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_totree.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + magic_id, + exp_id, + camera, + magicRun.workdir, + ra, + decl, + diffRun.tess_id, + IFNULL(priority, 10000) AS priority +FROM magicRun +JOIN diffRun USING(diff_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicTree + USING(magic_id) +LEFT JOIN Label ON magicRun.label = Label.label +WHERE + magicRun.state = 'new' + AND magicTree.node IS NULL + AND magicRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/magictool_warpskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/magictool_warpskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/magictool_warpskyfile.sql (revision 41217) @@ -0,0 +1,25 @@ +SELECT DISTINCT + warpSkyfile.warp_id, + warpSkyfile.skycell_id, + warpSkyfile.uri, + warpSkyfile.path_base +FROM magicRun +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffInputSkyfile.skycell_id = diffRun.skycell_id + -- Want input warps only + AND diffInputSkyfile.warp_id IS NOT NULL + AND diffInputSkyfile.template = 0 +JOIN warpSkyfile + ON warpSkyfile.warp_id = diffInputSkyfile.warp_id + AND warpSkyfile.skycell_id = diffRun.skycell_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +WHERE + warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 +-- AND magicRun.state = 'full' +-- AND magicMask.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_merge_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_merge_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_merge_id_dvo.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT mergedvodbRun.* FROM mergedvodbRun +JOIN mergedvodbProcessed USING(merge_id) +WHERE ( mergedvodbRun.state = 'merged' OR mergedvodbRun.state = 'full') + AND mergedvodbProcessed.fault = 0 + AND merge_id NOT IN (SELECT merge_id + FROM mergedvodbCopy + JOIN mergedvodbRun USING(mergedvodb_id) + WHERE %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_mergedvodbcopy.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_mergedvodbcopy.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_mergedvodbcopy.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT + mergedvodbCopy.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbCopy + JOIN mergedvodbRun using (mergedvodb_id) + + Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_minidvodb_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_minidvodb_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_minidvodb_id.sql (revision 41217) @@ -0,0 +1,8 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM mergedvodbRun + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pending_merge.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pending_merge.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pending_merge.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT + mergedvodbRun.*, + minidvodbRun.minidvodb_path +FROM minidvodbRun +JOIN mergedvodbRun + USING(minidvodb_id) +LEFT JOIN mergedvodbProcessed + USING(merge_id) +WHERE + (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) + AND mergedvodbRun.mergedvodb IS NOT NULL + AND mergedvodbRun.mergedvodb_path IS NOT NULL + + Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pendingmergeprocess.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pendingmergeprocess.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_pendingmergeprocess.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT mergedvodbRun.* +FROM mergedvodbRun +LEFT JOIN mergedvodbProcessed +USING(merge_id) +WHERE (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_processed.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_processed.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_find_processed.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT + mergedvodbProcessed.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbProcessed +JOIN mergedvodbRun + USING(merge_id) + Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_merge_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_merge_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_merge_id.sql (revision 41217) @@ -0,0 +1,11 @@ +INSERT INTO mergedvodbRun + SELECT + 0, -- merge_id + minidvodb_id, -- minidvodb_id + '%s', -- mergedvodb + '%s', -- mergedvodb_path + '%s', -- state + FROM minidvodbRun join minidvodbProcessed using (minidvodb_id) + WHERE + minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodbRun.minidvodb_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_mergecopy_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_mergecopy_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_mergecopy_id.sql (revision 41217) @@ -0,0 +1,16 @@ +INSERT INTO mergedvodbCopy + SELECT + 0, -- mergedvodbcopy_id + merge_id, -- merge_id + '%s', -- mergedvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM mergedvodbRun + WHERE + (mergedvodbRun.state = 'merged' + OR mergedvodbRun.state = 'full') + AND mergedvodbRun.merge_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_minidvodb_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_minidvodb_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_queue_minidvodb_id.sql (revision 41217) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbcopy.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbcopy.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbcopy.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE mergedvodbCopy + -- SET minidvodbCopy.state = 'new', + SET mergedvodbCopy.fault = 0 +WHERE + mergedvodbCopy.state = 'new' + AND mergedvodbCopy.fault >0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbprocessed.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbprocessed.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/mergetool_revertmergedvodbprocessed.sql (revision 41217) @@ -0,0 +1,6 @@ +DELETE FROM mergedvodbProcessed +USING mergedvodbProcessed, mergedvodbRun +WHERE + mergedvodbProcessed.merge_id = mergedvodbRun.merge_id + AND mergedvodbProcessed.fault != 0 + AND mergedvodbRun.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodb_id_dvo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodb_id_dvo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodb_id_dvo.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' + AND minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM minidvodbCopy + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodbcopy.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodbcopy.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_find_minidvodbcopy.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT + minidvodbCopy.*, + minidvodbRun.minidvodb_path +FROM minidvodbCopy + JOIN minidvodbRun using (minidvodb_id) + + Index: /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_queue_minidvodb_id.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_queue_minidvodb_id.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_queue_minidvodb_id.sql (revision 41217) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_revertminidvodbcopy.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_revertminidvodbcopy.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/minidvodbtool_revertminidvodbcopy.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE minidvodbCopy + -- SET minidvodbCopy.state = 'new', + SET minidvodbCopy.fault = 0 +WHERE + minidvodbCopy.state = 'new' + AND minidvodbCopy.fault >0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/parse_pxadmin_create_tables.pl =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/parse_pxadmin_create_tables.pl (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/parse_pxadmin_create_tables.pl (revision 41217) @@ -0,0 +1,34 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +my $ctr=0; +my $old=""; +while (<>) { + s/^\s+//; #remove whitespace from front + s/\s*--.*\n$//; # remove all comments + s/\s+/ /g; # kill the middle whitespace + s/\s*$//; # kill the whitespace at the end of the line +# if ($_ !~ /;$/ ) {chomp $_}; + s/;$/;\n/; # add back in the \n that was removed as whitespace but only for ; + s/,$/, /; # add space at end of , if at end of line + if (m/CREATE\s+TABLE\s+receive/) {$ctr = 1;} + if (m/;/) {$ctr = 0;} + if ($ctr == 0) { + s/AUTO_INCREMENT// ; + if (/FOREIGN\s+KEY/) { + if (!/, $/) { #if FOREIGN KEY line doesn't have , + $old =~ s/,\s*$//; #remove , from previous line + print $old; + $old=""; + } + } else { + print $old; + $old=$_; + } + } else { print $_; + } +} +print $old; +print "-- This comment line is here to avoid empty query error.\n"; Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_addaccesslevel.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_addaccesslevel.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_addaccesslevel.sql (revision 41217) @@ -0,0 +1,4 @@ +INSERT + INTO pstampAccessLevel + SELECT proj_id, %d, %f, %f FROM pstampProject + WHERE name = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_completedreq.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_completedreq.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_completedreq.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT pstampRequest.* , + IFNULL(Label.priority, 0) AS priority +FROM pstampRequest + LEFT JOIN Label USING(label) +WHERE state = 'run' + AND pstampRequest.fault = 0 + AND ( + SELECT count(*) FROM pstampJob + WHERE pstampJob.req_id = pstampRequest.req_id + AND pstampJob.state != 'stop' + AND pstampJob.state != 'cancel' + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_datastore.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_datastore.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_datastore.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT pstampDataStore.* +FROM pstampDataStore +WHERE + state = 'enabled' Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_getdependent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_getdependent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_getdependent.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT DISTINCT pstampDependent.* +FROM pstampDependent +WHERE (pstampDependent.state = 'new' OR pstampDependent.state = 'hold') Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listfile.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + pstampFile.*, + pstampJob.req_id +FROM pstampFile + JOIN pstampJob using(job_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listjob.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listjob.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listjob.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT + pstampJob.*, + pstampRequest.name, + pstampRequest.outProduct +FROM pstampJob + JOIN pstampRequest USING(req_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listuser.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listuser.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_listuser.sql (revision 41217) @@ -0,0 +1,10 @@ +SELECT userName, + domainName, + pstampUser.accessLevel, + IFNULL(pstampUserDomain.accessLevel, 0) AS domainAccessLevel, + pstampUser.defaultProduct as userProduct, + pstampUser.defaultLabel as userLabel, + pstampUserDomain.defaultProduct as domainProduct, + pstampUserDomain.defaultLabel as domainLabel +FROM pstampUser + LEFT JOIN pstampUserDomain USING(domainName) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingcleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingcleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingcleanup.sql (revision 41217) @@ -0,0 +1,9 @@ +-- postage stamp requests pending cleanup +SELECT + pstampRequest.*, + IFNULL(Label.priority, 10000) AS priority +FROM pstampRequest + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'goto_cleaned' + AND pstampRequest.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingdependent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingdependent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingdependent.sql (revision 41217) @@ -0,0 +1,14 @@ +SELECT DISTINCT + pstampDependent.*, + pstampRequest.label, + MIN(req_id) as first_req_id, + concat( pstampDependent.outdir, '/checkdep.', dep_id, '.log') as logfile, + IFNULL(Label.priority, 10000) AS priority +FROM pstampDependent +JOIN pstampJob USING(dep_id) +JOIN pstampRequest USING(req_id) +LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampDependent.state = 'new' + AND pstampJob.state = 'run' + AND pstampRequest.state = 'run' + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingjob.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingjob.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingjob.sql (revision 41217) @@ -0,0 +1,48 @@ +SELECT * FROM +(SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority, + 0 AS myParent, + 0 AS runningChildren +FROM pstampJob + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 + AND NOT is_parent + AND (dep_id = 0 OR pstampDependent.state = 'full') + AND (Label.active OR Label.active IS NULL) + -- where hook 1 %s +UNION +SELECT * FROM + (SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority + FROM pstampJob + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) + LEFT JOIN Label ON pstampRequest.label = Label.label + WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 + AND is_parent + AND (dep_id = 0 OR pstampDependent.state = 'full') + AND (Label.active OR Label.active IS NULL) + -- where hook 2 %s + ) as parentJob + LEFT JOIN + (SELECT parent_id AS myParent, + count(job_id) as runningChildren + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state ='run' + AND pstampJob.state = 'run' + AND pstampJob.parent_id + GROUP BY parent_id + ) as childJobs + ON childJobs.myParent = parentJob.job_id + WHERE runningChildren IS NULL +) AS theQuery + + Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingreq.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingreq.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_pendingreq.sql (revision 41217) @@ -0,0 +1,12 @@ +-- postage stamp requests ready for download and parsing +SELECT + pstampRequest.*, + pstampDataStore.outProduct AS ds_outProduct, + IFNULL(pstampDataStore.need_magic, 1) AS need_magic, + IFNULL(Label.priority, 10000) AS priority +FROM pstampRequest + LEFT JOIN pstampDataStore USING(ds_id) + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'new' + AND pstampRequest.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_project.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_project.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_project.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * FROM pstampProject Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertdependent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertdependent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertdependent.sql (revision 41217) @@ -0,0 +1,9 @@ +UPDATE pstampDependent + JOIN pstampJob USING(dep_id) + JOIN pstampRequest USING(req_id) +SET pstampDependent.fault = 0 +-- fault count hook %s +WHERE pstampRequest.state ='run' + AND pstampJob.state ='run' + AND pstampDependent.state = 'new' + AND (pstampDependent.fault > 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertjob.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertjob.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertjob.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE pstampJob + JOIN pstampRequest USING(req_id) +SET pstampJob.fault = 0 + -- clear fault count clause goes here %s +WHERE pstampRequest.state = 'run' + AND pstampJob.state = 'run' + -- fault clause goes here %s Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE pstampRequest + SET fault = 0 +WHERE pstampRequest.state != 'stop' + Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq_deletejobs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq_deletejobs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_revertreq_deletejobs.sql (revision 41217) @@ -0,0 +1,6 @@ +-- delete jobs that got added by an incomplete pstamp parser run +DELETE pstampJob +FROM pstampJob + JOIN pstampRequest USING(req_id) +WHERE pstampRequest.state = 'new' + AND pstampRequest.fault > 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_stopdependentjob.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_stopdependentjob.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_stopdependentjob.sql (revision 41217) @@ -0,0 +1,10 @@ +-- stop jobs which have a faulted dependent +UPDATE pstampDependent + JOIN pstampJob USING(dep_id) + JOIN pstampRequest USING(req_id) +SET pstampJob.state = 'stop', + pstampJob.fault = %d, + pstampDependent.fault = %d, + pstampDependent.state = '%s' +WHERE pstampJob.state = 'run' + AND pstampDependent.state = 'new' Index: /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_updatejob.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_updatejob.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pstamptool_updatejob.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE pstampJob + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) +SET Index: /branches/ipp-148_http_503_filesset_revert_fix/pubtool_definerun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pubtool_definerun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pubtool_definerun.sql (revision 41217) @@ -0,0 +1,74 @@ +-- Get runs to publish +SELECT DISTINCT + client_id, + stage_id, + src_label, + output_format +FROM ( + -- Get diffs to publish + SELECT DISTINCT + client_id, + diff_id AS stage_id, + diffRun.label AS src_label, + output_format + FROM publishClient + JOIN diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile using(diff_id) + JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference! + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicDSRun.stage = 'diff' AND magicDSRun.stage_id = diff_id + WHERE publishClient.stage = 'diff' + AND publishClient.active = 1 + AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND ( diffRun.diff_mode = 4 + OR (publishClient.magicked AND diffRun.magicked != 0) + OR (publishClient.magicked = 0 AND + (diffRun.magicked = 0 OR (diffRun.magicked != 0 AND magic_ds_id IS NOT NULL)) + ) + ) + AND diffSkyfile.quality = 0 + -- WHERE hook %s + UNION + -- Get cameras to publish + SELECT + client_id, + cam_id AS stage_id, + camRun.label AS src_label, + output_format + FROM publishClient + JOIN camRun + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'camera' + AND publishClient.active = 1 + AND camRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND (camRun.magicked != 0 OR publishClient.magicked = 0) + -- WHERE hook %s + UNION + -- Get diffphots to publish + SELECT DISTINCT + client_id, + diff_phot_id AS stage_id, + diffPhotRun.label AS src_label, + output_format + FROM publishClient + JOIN diffPhotRun + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference! + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'diffphot' + AND publishClient.active = 1 + AND diffPhotRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND (diffPhotRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0) + -- WHERE hook %s + ) AS publishToDo +-- Only get stuff that hasn't been published +LEFT JOIN publishRun USING(client_id, stage_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/pubtool_pending.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pubtool_pending.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pubtool_pending.sql (revision 41217) @@ -0,0 +1,87 @@ +SELECT + publishToDo.* +FROM ( + -- Get the diffs + -- The following is only appropriate for a diff where warp1 is set; otherwise it's more difficult to get the camera name + SELECT DISTINCT + publishRun.pub_id, + publishClient.product, + publishClient.stage, + publishClient.workdir, + publishClient.output_format, + publishClient.magicked AS need_magic, + diffRun.diff_id AS stage_id, + rawExp.camera, + rawExp.exp_id + FROM publishRun + JOIN publishClient USING(client_id) + JOIN diffRun + ON diffRun.diff_id = publishRun.stage_id + JOIN diffInputSkyfile USING(diff_id) + -- Need to do something fancy here to get the camera name for a stack + LEFT JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'diff' + AND publishClient.active = 1 + AND publishRun.state = 'new' + AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND (diffRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0) + -- WHERE hook %s + UNION + SELECT + publishRun.pub_id, + publishClient.product, + publishClient.stage, + publishClient.workdir, + publishClient.output_format, + publishClient.magicked AS need_magic, + camRun.cam_id AS stage_id, + rawExp.camera, + rawExp.exp_id + FROM publishRun + JOIN publishClient USING(client_id) + JOIN camRun + ON camRun.cam_id = publishRun.stage_id + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'camera' + AND publishClient.active = 1 + AND publishRun.state ='new' + AND camRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND (camRun.magicked != 0 OR publishClient.magicked = 0) + -- WHERE hook %s + UNION + SELECT DISTINCT + publishRun.pub_id, + publishClient.product, + publishClient.stage, + publishClient.workdir, + publishClient.output_format, + publishClient.magicked AS need_magic, + diffPhotRun.diff_phot_id AS stage_id, + rawExp.camera, + rawExp.exp_id + FROM publishRun + JOIN publishClient USING(client_id) + JOIN diffPhotRun + ON diffPhotRun.diff_phot_id = publishRun.stage_id + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + -- Need to do something fancy here to get the camera name for a stack + LEFT JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'diffphot' + AND publishClient.active = 1 + AND publishRun.state = 'new' + AND diffPhotRun.state IN ('full', 'cleaned', 'goto_cleaned') + AND (diffPhotRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0) + -- WHERE hook %s +) AS publishToDo +LEFT JOIN publishDone USING(pub_id) +WHERE publishDone.pub_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/pubtool_revert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pubtool_revert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pubtool_revert.sql (revision 41217) @@ -0,0 +1,7 @@ +DELETE FROM publishDone +USING publishDone, publishRun, publishClient +WHERE publishDone.pub_id = publishRun.pub_id + AND publishRun.state = 'new' + AND publishRun.client_id = publishClient.client_id + AND publishClient.active = 1 + AND publishDone.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_create_tables.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_create_tables.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_create_tables.sql (revision 41217) @@ -0,0 +1,2430 @@ +CREATE TABLE dbversion ( + schema_version VARCHAR(64), + updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDataStore ( + camera VARCHAR(64), + telescope VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + use_compress SMALLINT, + PRIMARY KEY(camera, telescope) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitExp ( + summit_id BIGINT NOT NULL AUTO_INCREMENT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_type VARCHAR(64), + uri VARCHAR(255), + imfiles INT, + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(summit_id), + KEY(exp_name, camera, telescope), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitImfile ( + summit_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + file_id VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(summit_id, class, class_id), + KEY(exp_name, camera, telescope, class, class_id), + KEY(file_id), + FOREIGN KEY(summit_id) REFERENCES summitExp(summit_id), + FOREIGN KEY(exp_name, camera, telescope) REFERENCES summitExp(exp_name, camera, telescope) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadExp ( + summit_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + state VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(summit_id), + KEY(exp_name, camera, telescope), + KEY(state), + FOREIGN KEY(summit_id) REFERENCES summitExp(summit_id), + FOREIGN KEY(exp_name, camera, telescope) REFERENCES summitExp(exp_name, camera, telescope) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadImfile ( + summit_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + hostname VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), + PRIMARY KEY(summit_id, class, class_id), + KEY(exp_name, camera, telescope, class, class_id), + KEY(fault), + FOREIGN KEY(summit_id) REFERENCES pzDownloadExp(summit_id), + FOREIGN KEY(summit_id, class, class_id) REFERENCES summitImfile(summit_id, class, class_id), + FOREIGN KEY (exp_name, camera, telescope) REFERENCES pzDownloadExp(exp_name, camera, telescope), + FOREIGN KEY(exp_name, camera, telescope, class, class_id) REFERENCES summitImfile(exp_name, camera, telescope, class, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newExp ( + exp_id BIGINT AUTO_INCREMENT, + summit_id BIGINT, + tmp_exp_name VARCHAR(64), + tmp_camera VARCHAR(64), + tmp_telescope VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + label VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_id), + KEY(summit_id), + KEY(tmp_exp_name), + KEY(tmp_camera), + KEY(tmp_telescope), + KEY(state), + KEY(workdir_state), + KEY(end_stage), + KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newImfile ( + exp_id BIGINT, + tmp_class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + bytes INT, + md5sum VARCHAR(32), + PRIMARY KEY(exp_id, tmp_class_id), + FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawExp ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_tag VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + state VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + obs_mode VARCHAR(64), + obs_group VARCHAR(64), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + sun_angle FLOAT, + sun_alt FLOAT, + moon_angle FLOAT, + moon_alt FLOAT, + moon_phase FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + magicked BIGINT, + PRIMARY KEY(exp_id), + KEY(end_stage), + KEY(fault), + KEY(state), + KEY(exp_name), + FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawImfile ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + tmp_class_id VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + data_state VARCHAR(64), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + obs_mode VARCHAR(64), + obs_group VARCHAR(64), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + sun_angle FLOAT, + sun_alt FLOAT, + moon_angle FLOAT, + moon_alt FLOAT, + moon_phase FLOAT, + ignored TINYINT DEFAULT 0, + hostname VARCHAR(64), + quality SMALLINT NOT NULL DEFAULT 0, + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + raw_image_id BIGINT AUTO_INCREMENT, + magicked BIGINT, + bytes INT, + md5sum VARCHAR(32), + burntool_state SMALLINT, + video_cells TINYINT DEFAULT 0, + PRIMARY KEY(exp_id, class_id), + KEY(tmp_class_id), + KEY(fault), + KEY(raw_image_id), + KEY(quality), + KEY(dateobs), + UNIQUE KEY(exp_id, tmp_class_id), + FOREIGN KEY(exp_id, tmp_class_id) REFERENCES newImfile(exp_id, tmp_class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipRun ( + chip_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix FLOAT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + update_mode SMALLINT DEFAULT 0, + note VARCHAR(255), + PRIMARY KEY(chip_id), + KEY(chip_id), KEY(exp_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(expgroup), + KEY(end_stage), + INDEX(chip_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipImfile ( + chip_id BIGINT, + class_id VARCHAR(64), + chip_imfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(chip_id, class_id), + KEY(chip_imfile_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipProcessedImfile ( + chip_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + data_state VARCHAR(64), + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + fwhm_major FLOAT, + fwhm_major_lq FLOAT, + fwhm_major_uq FLOAT, + fwhm_minor FLOAT, + fwhm_minor_lq FLOAT, + fwhm_minor_uq FLOAT, + iq_fwhm_major FLOAT, + iq_fwhm_major_err FLOAT, + iq_fwhm_minor FLOAT, + iq_fwhm_minor_err FLOAT, + iq_m2 FLOAT, + iq_m2_err FLOAT, + iq_m2_lq FLOAT, + iq_m2_uq FLOAT, + iq_m2c FLOAT, + iq_m2c_err FLOAT, + iq_m2c_lq FLOAT, + iq_m2c_uq FLOAT, + iq_m2s FLOAT, + iq_m2s_err FLOAT, + iq_m2s_lq FLOAT, + iq_m2s_uq FLOAT, + iq_m3 FLOAT, + iq_m3_err FLOAT, + iq_m3_lq FLOAT, + iq_m3_uq FLOAT, + iq_m4 FLOAT, + iq_m4_err FLOAT, + iq_m4_lq FLOAT, + iq_m4_uq FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_total FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_psfstars INT, + n_iqstars INT, + n_extended INT, + n_cr INT, + path_base VARCHAR(255), + quality SMALLINT NOT NULL DEFAULT 0, + fault SMALLINT NOT NULL, + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + deteff_magref FLOAT, + PRIMARY KEY(chip_id, exp_id, class_id), + KEY(data_state), + KEY(fault), + KEY(quality), + KEY(data_state), + FOREIGN KEY(chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id), + FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camRun ( + cam_id BIGINT AUTO_INCREMENT, + chip_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_ref_npix FLOAT, + maskfrac_ref_static FLOAT, + maskfrac_ref_dynamic FLOAT, + maskfrac_ref_magic FLOAT, + maskfrac_ref_advisory FLOAT, + maskfrac_max_npix FLOAT, + maskfrac_max_static FLOAT, + maskfrac_max_dynamic FLOAT, + maskfrac_max_magic FLOAT, + maskfrac_max_advisory FLOAT, + note VARCHAR(255), + PRIMARY KEY(cam_id), + KEY(cam_id), + KEY(chip_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(expgroup), + KEY(end_stage), + INDEX(cam_id, chip_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camProcessedExp ( + cam_id BIGINT, + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + zpt_lq FLOAT, + zpt_uq FLOAT, + fwhm_major FLOAT, + fwhm_major_lq FLOAT, + fwhm_major_uq FLOAT, + fwhm_minor FLOAT, + fwhm_minor_lq FLOAT, + fwhm_minor_uq FLOAT, + iq_fwhm_major FLOAT, + iq_fwhm_major_err FLOAT, + iq_fwhm_minor FLOAT, + iq_fwhm_minor_err FLOAT, + iq_m2 FLOAT, + iq_m2_err FLOAT, + iq_m2_lq FLOAT, + iq_m2_uq FLOAT, + iq_m2c FLOAT, + iq_m2c_err FLOAT, + iq_m2c_lq FLOAT, + iq_m2c_uq FLOAT, + iq_m2s FLOAT, + iq_m2s_err FLOAT, + iq_m2s_lq FLOAT, + iq_m2s_uq FLOAT, + iq_m3 FLOAT, + iq_m3_err FLOAT, + iq_m3_lq FLOAT, + iq_m3_uq FLOAT, + iq_m4 FLOAT, + iq_m4_err FLOAT, + iq_m4_lq FLOAT, + iq_m4_uq FLOAT, + dtime_script FLOAT, + dtime_astrom FLOAT, + dtime_addstar FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_psfstars INT, + n_iqstars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + software_ver VARCHAR(16), + maskfrac_ref_npix INT, + maskfrac_ref_static FLOAT, + maskfrac_ref_dynamic FLOAT, + maskfrac_ref_magic FLOAT, + maskfrac_ref_advisory FLOAT, + maskfrac_max_npix INT, + maskfrac_max_static FLOAT, + maskfrac_max_dynamic FLOAT, + maskfrac_max_magic FLOAT, + maskfrac_max_advisory FLOAT, + deteff FLOAT, + deteff_err FLOAT, + deteff_lq FLOAT, + deteff_uq FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + background_model SMALLINT, + astrom_chips BIGINT, + ast_r0 FLOAT, + ast_d0 FLOAT, + ast_t0 FLOAT, + ast_s0 FLOAT, + ast_rs FLOAT, + ast_ds FLOAT, + PRIMARY KEY(cam_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + stage VARCHAR(64), + stage_id BIGINT, + stage_extra1 INT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + reduction VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + dvodb VARCHAR(255), + note VARCHAR(255), + image_only TINYINT, + minidvodb TINYINT, + minidvodb_group VARCHAR(64), + minidvodb_name VARCHAR(64), + PRIMARY KEY(add_id), + KEY(add_id), + KEY(stage_id), + KEY(stage), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(data_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + path_base VARCHAR(255), + dvodb_path VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + note VARCHAR(255), + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY(fake_id) REFERENCES fakeRun(fake_id), + FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRun ( + det_id BIGINT AUTO_INCREMENT, + iteration INT, + det_type VARCHAR(64), + mode VARCHAR(64), + state VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + ref_det_id BIGINT, + ref_iter INT, + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + INDEX(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detInputExp ( + det_id BIGINT, + iteration INT, + exp_id BIGINT, + include TINYINT, + PRIMARY KEY(det_id, iteration, exp_id), + INDEX(det_id, exp_id), + INDEX(det_id, iteration), + FOREIGN KEY(det_id) REFERENCES detRun(det_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedImfile ( + det_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id), + FOREIGN KEY(det_id, exp_id) REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedExp ( + det_id BIGINT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id), + KEY(fault), + FOREIGN KEY(det_id, exp_id) REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY(det_id, exp_id) REFERENCES detProcessedImfile(det_id, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detStackedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY(det_id, class_id) REFERENCES detProcessedImfile(det_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedStatImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY(det_id, iteration, class_id) REFERENCES detStackedImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY(det_id) REFERENCES detInputExp(det_id), + FOREIGN KEY(det_id, iteration, class_id) REFERENCES detNormalizedStatImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedExp ( + det_id BIGINT, + iteration INT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY(det_id, iteration) REFERENCES detNormalizedImfile(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidImfile ( + det_id BIGINT, + iteration INT, + ref_det_id BIGINT, + ref_iter INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, exp_id, class_id) REFERENCES detProcessedImfile(det_id, exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidExp ( + det_id BIGINT, + iteration INT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY(det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY(det_id, iteration, exp_id) REFERENCES detResidImfile(det_id, iteration, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRunSummary ( + det_id BIGINT, + iteration INT, + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY(det_id, iteration) REFERENCES detResidExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRegisteredImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY(det_id, iteration) REFERENCES detRun(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + reduction VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix FLOAT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + note VARCHAR(255), + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(end_stage), + INDEX(warp_id, fake_id), + FOREIGN KEY(fake_id) REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, class_id), + KEY(fault), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpImfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + warp_skyfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(warp_id, skycell_id), + KEY(warp_skyfile_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + quality SMALLINT NOT NULL DEFAULT 0, + fault SMALLINT, + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + background_model SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(fault), + KEY(quality), + KEY(data_state), + FOREIGN KEY(warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSummary ( + warp_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id, projection_cell), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackRun ( + stack_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(64), + software_ver VARCHAR(16), + note VARCHAR(255), + PRIMARY KEY(stack_id), + KEY(stack_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id), + KEY(label), + KEY(data_group), + KEY(dist_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackInputSkyfile ( + stack_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(stack_id, warp_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id), + FOREIGN KEY(warp_id) REFERENCES warpSkyfile(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSumSkyfile ( + stack_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_stack FLOAT, + dtime_match_mean FLOAT, + dtime_match_stdev FLOAT, + dtime_convolve FLOAT, + dtime_initial FLOAT, + dtime_reject FLOAT, + dtime_final FLOAT, + dtime_phot FLOAT, + dtime_script FLOAT, + match_mean FLOAT, + match_stdev FLOAT, + match_rms FLOAT, + stamps_mean FLOAT, + stamps_stdev FLOAT, + stamps_min INT, + reject_images INT, + reject_pix_mean FLOAT, + reject_pix_stdev FLOAT, + sources INT, + hostname VARCHAR(64), + good_frac FLOAT, + mjd_obs DOUBLE, + fault SMALLINT, + software_ver VARCHAR(16), + background_model SMALLINT, + quality SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(stack_id), + KEY(dtime_stack), + KEY(good_frac), + KEY(fault), + KEY(quality), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + projection_cell VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(projection_cell), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSummary ( + sass_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(sass_id, projection_cell), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffRun ( + diff_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + tess_id VARCHAR(64), + bothways TINYINT DEFAULT 0, + exposure TINYINT DEFAULT 0, + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix FLOAT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + diff_mode SMALLINT NOT NULL, + note VARCHAR(255), + PRIMARY KEY(diff_id), + KEY(diff_id), + KEY(state), + KEY(tess_id), + KEY(label), + KEY(data_group), + KEY(dist_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffInputSkyfile ( + diff_id BIGINT, + skycell_id VARCHAR(64), + warp1 BIGINT, + stack1 BIGINT, + warp2 BIGINT, + stack2 BIGINT, + tess_id VARCHAR(64), + diff_skyfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(diff_skyfile_id), + KEY(diff_id, skycell_id), + KEY(warp1), + KEY(warp2), + KEY(stack1), + KEY(stack2), + KEY(skycell_id), + KEY(tess_id), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id), + FOREIGN KEY(warp1, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id), + FOREIGN KEY(warp2, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id), + FOREIGN KEY(stack1) REFERENCES stackSumSkyfile(stack_id), + FOREIGN KEY(stack2) REFERENCES stackSumSkyfile(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSkyfile ( + diff_id BIGINT, + skycell_id VARCHAR(64), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + stamps_num INT, + stamps_mean FLOAT, + stamps_rms FLOAT, + norm FLOAT, + bg_diff FLOAT, + kernel_x FLOAT, + kernel_y FLOAT, + kernel_xx FLOAT, + kernel_xy FLOAT, + kernel_yy FLOAT, + deconv_max FLOAT, + sources INT, + dtime_diff FLOAT, + dtime_match FLOAT, + dtime_phot FLOAT, + dtime_script FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + fault SMALLINT, + magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, + PRIMARY KEY(diff_id, skycell_id), + KEY(good_frac), + KEY(fault), + KEY(quality), + KEY(data_state), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSummary ( + diff_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id, projection_cell), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicRun ( + magic_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + diff_id BIGINT, + inverse TINYINT NOT NULL DEFAULT 0, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + fault SMALLINT, + note VARCHAR(255), + PRIMARY KEY(magic_id), + KEY(magic_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(data_group), + KEY(fault), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicInputSkyfile ( + magic_id BIGINT, + node VARCHAR(64), + PRIMARY KEY(magic_id, node), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicTree ( + magic_id BIGINT, + node VARCHAR(64), + dep VARCHAR(64), + KEY(magic_id), + KEY(node), + KEY(dep), + INDEX(magic_id, node), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicNodeResult ( + magic_id BIGINT, + node VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(magic_id, node), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY(magic_id, node) REFERENCES magicTree(magic_id, node), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicMask ( + magic_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + streaks INT, + fault SMALLINT, + PRIMARY KEY(magic_id), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicDSRun ( + magic_ds_id BIGINT AUTO_INCREMENT, + magic_id BIGINT, + inv_magic_id BIGINT, + state VARCHAR(64), + stage VARCHAR(64), + stage_id BIGINT, + cam_id BIGINT, + label VARCHAR(64), + data_group VARCHAR(64), + outroot VARCHAR(255), + recoveryroot VARCHAR(255), + re_place TINYINT, + remove TINYINT, + fault SMALLINT, + note VARCHAR(255), + PRIMARY KEY(magic_ds_id), + KEY(magic_ds_id), + KEY(state), + KEY(magic_id), + KEY(label), + KEY(fault), + KEY(stage), + KEY(stage_id), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicDSFile ( + magic_ds_id BIGINT, + component VARCHAR(64), + backup_path_base VARCHAR(255), + recovery_path_base VARCHAR(255), + streak_frac FLOAT, + nondiff_frac FLOAT, + run_time FLOAT, + fault SMALLINT, + data_state VARCHAR(64), + PRIMARY KEY(magic_ds_id, component), + KEY(fault), + KEY(data_state), + FOREIGN KEY(magic_ds_id) REFERENCES magicDSRun(magic_ds_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calDB ( + cal_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calRun ( + cal_id BIGINT AUTO_INCREMENT, + region VARCHAR(64), + last_step VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id), + KEY(last_step), + FOREIGN KEY(cal_id) REFERENCES calDB(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + det_type VARCHAR(64), + dvodb VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + filter VARCHAR(64), + state VARCHAR(64), + make_corr TINYINT, + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + region VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- these three tables link the flatcorrRun to the associated chip, camera, and addstar runs +CREATE TABLE flatcorrChipLink ( + corr_id BIGINT, + chip_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY(corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrCamLink ( + corr_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, cam_id), + FOREIGN KEY(corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrAddstarLink ( + corr_id BIGINT, + cam_id BIGINT, + add_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, cam_id, add_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id), + FOREIGN KEY (add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDataStore ( + ds_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + lastFileset VARCHAR(64), + timestamp DATETIME, + label VARCHAR(64), + outProduct VARCHAR(64) UNIQUE, + uri VARCHAR(255), + pollInterval INTEGER DEFAULT 60, + need_magic TINYINT, + PRIMARY KEY(ds_id), + KEY(ds_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampProject ( + proj_id BIGINT AUTO_INCREMENT, + name VARCHAR(64) UNIQUE, + state VARCHAR(64), + dbname VARCHAR(64), + dvodb VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + need_magic TINYINT, + PRIMARY KEY(proj_id), + KEY(proj_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampRequest ( + req_id BIGINT AUTO_INCREMENT, + ds_id BIGINT, + state VARCHAR(64), + name VARCHAR(64) UNIQUE, + reqType VARCHAR(16), + label VARCHAR(64), + outProduct VARCHAR(64), + uri VARCHAR(255), + outdir VARCHAR(255), + username VARCHAR(255), + proj_id BIGINT, + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + fault SMALLINT, + PRIMARY KEY(req_id), + KEY(req_id), + KEY(state), + KEY(fault), + KEY(label), + KEY(username) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampJob ( + job_id BIGINT AUTO_INCREMENT, + req_id BIGINT, + rownum VARCHAR(64), + state VARCHAR(64), + jobType VARCHAR(16), + fault SMALLINT, + exp_id BIGINT, + outputBase VARCHAR(255), + options BIGINT, + dep_id BIGINT, + fault_count INT, + parent_id BIGINT DEFAULT NULL, + is_parent TINYINT DEFAULT 0, + PRIMARY KEY(job_id, req_id), + KEY(job_id), + KEY(req_id), + KEY(state), + KEY(fault), + KEY(dep_id), + KEY(parent_id), + KEY(is_parent), + FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDependent ( + dep_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + stage VARCHAR(64), + stage_id BIGINT, + component VARCHAR(64), + imagedb VARCHAR(64), + outdir VARCHAR(255), + rlabel VARCHAR(64), + need_magic TINYINT, + fault SMALLINT, + fault_count INT, + PRIMARY KEY(dep_id), + KEY(state), + KEY(stage), + KEY(stage_id), + KEY(component), + KEY(imagedb), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampFile ( + file_id BIGINT AUTO_INCREMENT, + job_id BIGINT NOT NULL, + path VARCHAR(255), + PRIMARY KEY(file_id), + KEY(job_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampUserDomain ( + domainName VARCHAR(64), + accessLevel INT, + defaultProduct VARCHAR(255), + defaultLabel VARCHAR(64), + PRIMARY KEY(domainName) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampUser ( + userName VARCHAR(64), + domainName VARCHAR(64), + accessLevel INT, + defaultProduct VARCHAR(255), + defaultLabel VARCHAR(64), + PRIMARY KEY (userName, domainName) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampAccessLevel ( + proj_id BIGINT, + accessLevel INT, + mjd_min FLOAT DEFAULT 0, + mjd_max FLOAT DEFAULT 0, + PRIMARY KEY (proj_id, accessLevel), + UNIQUE KEY (proj_id, accessLevel), + FOREIGN KEY(proj_id) REFERENCES pstampProject(proj_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampWebRequest ( + num BIGINT AUTO_INCREMENT, + PRIMARY KEY(num) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE distTarget ( + target_id BIGINT AUTO_INCREMENT, + dist_group VARCHAR(64), + filter VARCHAR(64), + stage VARCHAR(64), + clean TINYINT, + state VARCHAR(64), + comment VARCHAR(255), + PRIMARY KEY(target_id), + KEY(stage), + KEY(dist_group), + KEY(filter), + KEY(state), + CONSTRAINT UNIQUE (dist_group, filter, stage, clean) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE distRun ( + dist_id BIGINT AUTO_INCREMENT, + target_id BIGINT, + stage VARCHAR(64), + stage_id BIGINT, + magic_ds_id BIGINT, + label VARCHAR(64), + outroot VARCHAR(255), + outdir VARCHAR(255), + clean TINYINT, + no_magic TINYINT, + alternate TINYINT, + state VARCHAR(64), + time_stamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + fault SMALLINT, + data_group VARCHAR(64), + note VARCHAR(255), + PRIMARY KEY(dist_id), + KEY(state), + KEY(label), + KEY(stage), + KEY(stage_id), + KEY(magic_ds_id), + KEY(fault), + KEY(data_group), + FOREIGN KEY(target_id) REFERENCES distTarget(target_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE distComponent ( + dist_id BIGINT, + component VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), + state VARCHAR(64), + outdir VARCHAR(255), + name VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(dist_id, component), + KEY(state), + FOREIGN KEY(dist_id) REFERENCES distRun(dist_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +CREATE TABLE rcDestination ( + dest_id BIGINT AUTO_INCREMENT, + name VARCHAR(64), + status_uri VARCHAR(255), + comment VARCHAR(255), + last_fileset VARCHAR(255), + dbname VARCHAR(64), + dbhost VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(dest_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rcInterest ( + int_id BIGINT AUTO_INCREMENT, + dest_id BIGINT, + target_id BIGINT, + state VARCHAR(64), + PRIMARY KEY(int_id), + FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id), + FOREIGN KEY(target_id) REFERENCES distTarget(target_id), + CONSTRAINT UNIQUE (dest_id, target_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rcDSFileset ( + fs_id BIGINT AUTO_INCREMENT, + dist_id BIGINT, + dest_id BIGINT, + name VARCHAR(255), + state VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + fault SMALLINT DEFAULT 0, + PRIMARY KEY(dist_id, dest_id), + KEY(fs_id), + FOREIGN KEY(dist_id) REFERENCES distRun(dist_id), + FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rcRun ( + rc_id BIGINT AUTO_INCREMENT, + fs_id BIGINT, + dest_id BIGINT, + state VARCHAR(64), + status_fs_name VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + fault SMALLINT DEFAULT 0, + PRIMARY KEY(rc_id), + FOREIGN KEY(fs_id) REFERENCES rcDSFileset(fs_id), + FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Sources from which to receive files +CREATE TABLE receiveSource ( + source_id BIGINT AUTO_INCREMENT, -- unique identifier + source VARCHAR(128) NOT NULL, -- source URI + product VARCHAR(64) NOT NULL, -- product of interest + workdir VARCHAR(255) NOT NULL, -- where to extract + state VARCHAR(64) NOT NULL, -- state 'enabled' or 'disabled' + comment VARCHAR(255), -- for human memory + fileset_last VARCHAR(128), -- last fileset seen + status_product VARCHAR(64), -- status data store product + ds_dbname VARCHAR(64), -- status data store's database name + ds_dbhost VARCHAR(64), -- status data store's host name + PRIMARY KEY(source_id), + KEY(source), + KEY(product), + KEY(comment) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Filesets to receive +CREATE TABLE receiveFileset ( + fileset_id BIGINT AUTO_INCREMENT, -- unique identifier + source_id BIGINT NOT NULL, -- link to receiveSource + fileset VARCHAR(128) NOT NULL, -- fileset to receive + state VARCHAR(64), -- new or full + dirinfo VARCHAR(255), -- uri for directory info file for this run + dbinfo VARCHAR(255), -- uri for database dump file for this run + fault SMALLINT NOT NULL DEFAULT 0, -- Fault code + PRIMARY KEY(fileset_id), + KEY(source_id), + CONSTRAINT UNIQUE(source_id, fileset), + FOREIGN KEY(source_id) REFERENCES receiveSource(source_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Files to receive +CREATE TABLE receiveFile ( + file_id BIGINT AUTO_INCREMENT, -- unique identifier + fileset_id BIGINT NOT NULL, -- link to receiveFileset + file VARCHAR(128) NOT NULL, -- file to receive + bytes BIGINT, + md5sum VARCHAR(255), + file_type VARCHAR(64), + component VARCHAR(64), + PRIMARY KEY(file_id), + KEY(fileset_id), + CONSTRAINT UNIQUE(fileset_id, file), + FOREIGN KEY(fileset_id) REFERENCES receiveFileset(fileset_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of receiving files +CREATE TABLE receiveResult ( + file_id BIGINT AUTO_INCREMENT, -- link to receiveFile + dtime_copy FLOAT, -- Time to copy + dtime_extract FLOAT, -- Time to extract + fault SMALLINT NOT NULL DEFAULT 0, -- Fault code + PRIMARY KEY(file_id), + KEY(fault), + FOREIGN KEY(file_id) REFERENCES receiveFile(file_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + + +-- Tables to support publishing of detections to a Science Client + +-- Clients to which we send stuff +CREATE TABLE publishClient ( + client_id BIGINT AUTO_INCREMENT, -- unique identifier + active TINYINT DEFAULT 0, -- whether we should worry about this or not + product VARCHAR(64), -- product name + stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.) + magicked TINYINT DEFAULT 1, -- Require magicked data? + workdir VARCHAR(255) NOT NULL, -- working directory + comment VARCHAR(255), -- for human memory + name varchar(64) default NULL, -- unique client_id verbose identifier + output_format SMALLINT NOT NULL default 1, -- format output versioning + PRIMARY KEY(client_id), + UNIQUE KEY name (name) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Publishing a set of data (e.g., a specific diffRun) +CREATE TABLE publishRun ( + pub_id BIGINT AUTO_INCREMENT, -- unique identifier + client_id BIGINT NOT NULL, -- link to publishClient + stage_id BIGINT NOT NULL, -- link to various stage tables + label VARCHAR(64), -- label for run + state VARCHAR(64), -- state of run (new, full, etc.) + PRIMARY KEY(pub_id), + KEY(client_id), + KEY(stage_id), + KEY(label), + KEY(state), + FOREIGN KEY(client_id) REFERENCES publishClient(client_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Publishing a file within a set +CREATE TABLE publishDone ( + pub_id BIGINT AUTO_INCREMENT, -- link to publishRun + path_base VARCHAR(255), -- base path of output + hostname VARCHAR(64), -- name of host + dtime_script FLOAT, -- run time for script + fault SMALLINT NOT NULL DEFAULT 0, -- Fault code + PRIMARY KEY(pub_id), + KEY(fault), + FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Tables for static sky analysis + +-- A static sky analysis set +CREATE TABLE staticskyRun ( + sky_id BIGINT AUTO_INCREMENT, -- unique identifier + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(sky_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Inputs for static sky analysis +CREATE TABLE staticskyInput ( + sky_id BIGINT, -- static sky identifier + stack_id BIGINT, -- stack identifier + PRIMARY KEY(sky_id,stack_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackSumSkyfile(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of static sky analysis +CREATE TABLE staticskyResult ( + sky_id BIGINT, -- static sky identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + dtime_phot FLOAT, -- elapsed time for photometry + dtime_script FLOAT, -- elapsed time for script + sources INT, -- number of sources + num_inputs INT, -- number of input images + hostname VARCHAR(64) NOT NULL, -- host that executed script + good_frac FLOAT, -- good fraction of skycell + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + PRIMARY KEY(sky_id), + KEY(good_frac), + KEY(fault), + KEY(quality), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Processing labels with their priorities +CREATE TABLE Label ( + label VARCHAR(64), + priority INT, + active TINYINT, + comment VARCHAR(80), + PRIMARY KEY(label), + KEY(priority), + KEY(active) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Tables to support background restoration + +-- Background replacement on a chipRun +CREATE TABLE chipBackgroundRun ( + chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + chip_id BIGINT NOT NULL, -- link to chipRun + cam_id BIGINT DEFAULT 0, -- optional link to camRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + magicked BIGINT DEFAULT 0 NOT NULL, -- magic status + PRIMARY KEY(chip_bg_id), + KEY(chip_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from chipBackgroundRun +CREATE TABLE chipBackgroundImfile ( + chip_bg_id BIGINT NOT NULL, -- unique identifier + class_id VARCHAR(64) NOT NULL, -- class (component) identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(chip_bg_id,class_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Background replacement on a warpRun (utilising chipBackgroundRun) +CREATE TABLE warpBackgroundRun ( + warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + warp_id BIGINT NOT NULL, -- link to warpRun + chip_bg_id BIGINT NOT NULL, -- link to chipBackgroundRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + magicked BIGINT DEFAULT 0 NOT NULL, -- magic status + PRIMARY KEY(warp_bg_id), + KEY(warp_id), + KEY(chip_bg_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from warpBackgroundRun +CREATE TABLE warpBackgroundSkyfile ( + warp_bg_id BIGINT NOT NULL, -- unique identifier + skycell_id VARCHAR(64) NOT NULL, -- skycell identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(warp_bg_id,skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Tables to support (re-)photometry of a diff + +CREATE TABLE diffPhotRun ( + diff_phot_id BIGINT AUTO_INCREMENT, -- Identifier for diffPhotRun + diff_id BIGINT NOT NULL, -- Identifier for diffRun + state VARCHAR(64) NOT NULL, -- State of run + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied + PRIMARY KEY(diff_phot_id), + KEY(diff_id), + KEY(state), + KEY(label), + KEY(data_group), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffPhotSkyfile ( + diff_phot_id BIGINT AUTO_INCREMENT, -- Identifier for diffPhotRun + skycell_id VARCHAR(64) NOT NULL, -- Skycell identifier + path_base VARCHAR(255) NOT NULL, -- Base of path for output + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + fault SMALLINT NOT NULL, -- fault code + quality SMALLINT NOT NULL, -- bad quality flag + software_ver VARCHAR(16), -- software version + magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied + PRIMARY KEY(diff_phot_id, skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(diff_phot_id) REFERENCES diffPhotRun(diff_phot_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Tables for large area processing + +CREATE TABLE lapSequence ( + seq_id BIGINT AUTO_INCREMENT, -- Identifier for the processing sequence + name VARCHAR(64) NOT NULL, -- short name of the sequence + description VARCHAR(255) NOT NULL, -- longer description of the sequence + PRIMARY KEY(seq_id), + KEY(name) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapRun ( + lap_id BIGINT AUTO_INCREMENT, -- Identifier for the processing run + seq_id BIGINT NOT NULL, -- Identifier to match to the sequence + tess_id VARCHAR(64) NOT NULL, -- tessellation id to use + projection_cell VARCHAR(64) NOT NULL, -- projection cell from the tessellation to consider + filter VARCHAR(64) NOT NULL, -- filter + state VARCHAR(64) NOT NULL, -- state of run + label VARCHAR(64) NOT NULL, -- processing label + dist_group VARCHAR(64) NOT NULL, -- distribution group for products of this run + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + fault SMALLINT NOT NULL, -- fault code + quick_sass_id BIGINT, -- stackAssociation id for quick stack + final_sass_id BIGINT, -- stackAssociation id for final stack + PRIMARY KEY(lap_id), + KEY(seq_id), + KEY(projection_cell), + KEY(filter), + KEY(state), + KEY(label), + KEY(fault), + FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id), + FOREIGN KEY(quick_sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(final_sass_id) REFERENCES stackAssociation(sass_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapExp ( + lap_id BIGINT NOT NULL, -- Link back to processing run + exp_id BIGINT NOT NULL, -- exposure definition + chip_id BIGINT, -- processing id from chipRun + pair_id BIGINT, -- companion chip_id + private TINYINT DEFAULT 0, -- denotes this exposure is private + pairwise TINYINT DEFAULT 0, -- denotes if this exposure should be pairwise diffed + active TINYINT DEFAULT 0, -- denotes if this exposure is currently in use + data_state VARCHAR(64) NOT NULL, -- state of exposure + PRIMARY KEY (lap_id), + KEY (exp_id), + KEY (chip_id), + KEY (pair_id), + KEY (data_state), + FOREIGN KEY (lap_id) REFERENCES lapRun(lap_id), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY (chip_id,exp_id) REFERENCES chipRun(chip_id,exp_id), + FOREIGN KEY (pair_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE minidvodbRun ( +minidvodb_id BIGINT AUTO_INCREMENT, + minidvodb_name VARCHAR(64), + minidvodb_group VARCHAR(64) NOT NULL, + minidvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE minidvodbProcessed ( + minidvodb_id BIGINT, + dtime_resort FLOAT, + dtime_relphot FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (minidvodb_id), + KEY(minidvodb_id), + FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +CREATE TABLE minidvodbCopy ( + minidvodbcopy_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + minidvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(minidvodbcopy_id), + KEY(minidvodb_id), + FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + merge_order BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE skycalRun ( + skycal_id BIGINT AUTO_INCREMENT, -- unique identifier + sky_id BIGINT, + stack_id BIGINT, + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(skycal_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(sky_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of sky calibration analysis +CREATE TABLE skycalResult ( + skycal_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + dtime_astrom FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + n_astrom INT, + n_detections INT, + n_extended INT, + n_forced INT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(skycal_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE skycell ( + tess_id VARCHAR(64), + skycell_id VARCHAR(64), + radeg float, + decdeg float, + glong float, + glat float, + width float, + height float, + PRIMARY KEY (tess_id, skycell_id), + KEY(radeg), + KEY(decdeg), + KEY(glong), + KEY(glat) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE survey ( + surveyID INT, + surveyName VARCHAR(16) NOT NULL, + description VARCHAR(255), + PRIMARY KEY(surveyID), + UNIQUE KEY(surveyName) +) ENGINE=InnoDB CHARSET=latin1; + +CREATE TABLE ippRelease ( + rel_id INT AUTO_INCREMENT, + surveyID INT, + release_name VARCHAR(64), + release_state VARCHAR(16), -- active, pending, archive, drop + dataRelease INT, -- PSPS dataRelease + priority INT, + dvodb VARCHAR(255), + ubercal_file VARCHAR(255), + accessLevelMin INT DEFAULT 0, + PRIMARY KEY(rel_id), + UNIQUE KEY(surveyID, release_name), + KEY(release_name), + KEY(release_state), + KEY(priority), + FOREIGN KEY(surveyID) REFERENCES survey(surveyID) +) ENGINE=Innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relExp ( + relexp_id BIGINT AUTO_INCREMENT, + rel_id INT, + exp_id BIGINT, + chip_id BIGINT, -- links to the runs that supplied the data + cam_id BIGINT, -- for this release + group_id INT, -- id of relGroup that this exposure is contained in + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, -- flags for relphot, relastro, ?? + zpt_obs FLOAT, -- calibrated zero point for this release of + zpt_stdev FLOAT, -- this exposure + mcal FLOAT, -- this exposure + ubercal_dist INT, -- ubercal distance (from dvo calibration) + path_base VARCHAR(255), -- path_base of any supporting files for this + -- release of this exposure. + fault SMALLINT NOT NULL, + registered DATETIME, -- insertion time for this row + time_stamp DATETIME, -- time of last update for this row + PRIMARY KEY (relexp_id), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + UNIQUE KEY(rel_id, exp_id), + KEY (state), + KEY (fault), + KEY (group_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relStack ( + relstack_id BIGINT AUTO_INCREMENT, + rel_id INT, + stack_id BIGINT, -- id of the stackRun + skycal_id BIGINT, -- id of the sky calibration run that supplied the calibration + -- for this release of this skycell. + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(16), + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, + stack_type VARCHAR(16), -- nightly, deep, reference + zpt_obs FLOAT, -- calibrated zero point for this release of this skycell + zpt_stdev FLOAT, + mjd_obs INT UNSIGNED, -- for nightly stacks the mjd_obs, zero for deep and reference + path_base VARCHAR(255), -- path_base of any supporting files, + fault SMALLINT NOT NULL, + registered DATETIME, + time_stamp DATETIME, + PRIMARY KEY (relstack_id), + UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs), + KEY (tess_id, skycell_id), + KEY (stack_type), + KEY (mjd_obs), + KEY (state), + KEY (fault), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relGroup ( + group_id INT AUTO_INCREMENT, + rel_id INT, + group_type VARCHAR(16), + lap_id BIGINT, + group_name VARCHAR(16), + state VARCHAR(16), -- new full + label VARCHAR(64), + exp_list_path VARCHAR(255), + fault SMALLINT NOT NULL, + registered DATETIME, + PRIMARY KEY (group_id), + KEY (rel_id, group_name), + KEY (group_type), + KEY (lap_id), + KEY (group_name), + KEY (state), + KEY (label), + KEY (fault), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapGroup ( + seq_id BIGINT, + tess_id VARCHAR(64), + projection_cell VARCHAR(64), + state VARCHAR(16), + label VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered + fault SMALLINT, + PRIMARY KEY(seq_id, tess_id, projection_cell), + KEY(state), + KEY(fault), + FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE dqstatsRun ( + dqstats_id BIGINT NOT NULL AUTO_INCREMENT, + state VARCHAR(64) DEFAULT NULL, + registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + label VARCHAR(64) DEFAULT NULL, + fault SMALLINT NOT NULL, + PRIMARY KEY (dqstats_id), + KEY label (label), + KEY fault (fault) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE dqstatsContent ( + dqstats_id BIGINT, + exp_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + warp_id BIGINT, + invalid TINYINT DEFAULT NULL, + KEY dqstats_id (dqstats_id), + KEY exp_id (exp_id), + KEY chip_id (chip_id), + KEY cam_id (cam_id), + KEY warp_id (warp_id), + FOREIGN KEY (dqstats_id) REFERENCES dqstatsRun (dqstats_id), + FOREIGN KEY (exp_id) REFERENCES rawExp (exp_id), + FOREIGN KEY (chip_id) REFERENCES chipRun (chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun (cam_id), + FOREIGN KEY (warp_id) REFERENCES warpRun (warp_id) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceRun ( + ff_id BIGINT NOT NULL AUTO_INCREMENT, + skycal_id BIGINT, + sources_path_base VARCHAR(255), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + note VARCHAR(255), + reduction VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(ff_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(skycal_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( + ff_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE remoteRun ( + remote_id BIGINT NOT NULL AUTO_INCREMENT, + state VARCHAR(64) NOT NULL, + stage VARCHAR(64) NOT NULL, + label VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + job_id BIGINT, + last_poll TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + PRIMARY KEY (remote_id), + KEY (state), + KEY (stage), + KEY (label), + KEY (job_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE remoteComponent ( + remote_id BIGINT, + stage_id BIGINT, + jobs INT, + state VARCHAR(64) NOT NULL, + path_base VARCHAR(255), + KEY (stage_id), + FOREIGN KEY (remote_id) REFERENCES remoteRun(remote_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- These comment lines are here to avoid an empty query error. +-- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_drop_tables.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_drop_tables.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_drop_tables.sql (revision 41217) @@ -0,0 +1,133 @@ +SET FOREIGN_KEY_CHECKS=0; + +DROP TABLE IF EXISTS dbversion; +DROP TABLE IF EXISTS pzDataStore; +DROP TABLE IF EXISTS summitExp; +DROP TABLE IF EXISTS summitImfile; +DROP TABLE IF EXISTS pzDownloadExp; +DROP TABLE IF EXISTS pzDownloadImfile; +DROP TABLE IF EXISTS newExp; +DROP TABLE IF EXISTS newImfile; +DROP TABLE IF EXISTS rawExp; +DROP TABLE IF EXISTS rawImfile; +DROP TABLE IF EXISTS chipRun; +DROP TABLE IF EXISTS chipProcessedImfile; +DROP TABLE IF EXISTS chipImfile; +DROP TABLE IF EXISTS chipMask; +DROP TABLE IF EXISTS camRun; +DROP TABLE IF EXISTS camProcessedExp; +DROP TABLE IF EXISTS camMask; +DROP TABLE IF EXISTS detRun; +DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS addRun; +DROP TABLE IF EXISTS addProcessedExp; +DROP TABLE IF EXISTS addMask; +DROP TABLE IF EXISTS fakeRun; +DROP TABLE IF EXISTS fakeProcessedImfile; +DROP TABLE IF EXISTS fakeMask; +DROP TABLE IF EXISTS detProcessedImfile; +DROP TABLE IF EXISTS detProcessedExp; +DROP TABLE IF EXISTS detStackedImfile; +DROP TABLE IF EXISTS detNormalizedStatImfile; +DROP TABLE IF EXISTS detNormalizedImfile; +DROP TABLE IF EXISTS detNormalizedExp; +DROP TABLE IF EXISTS detResidImfile; +DROP TABLE IF EXISTS detResidExp; +DROP TABLE IF EXISTS detRunSummary; +DROP TABLE IF EXISTS detRegisteredImfile; +DROP TABLE IF EXISTS warpRun; +DROP TABLE IF EXISTS warpSkyCellMap; +DROP TABLE IF EXISTS warpImfile; +DROP TABLE IF EXISTS warpSkyfile; +DROP TABLE IF EXISTS warpMask; +DROP TABLE IF EXISTS warpSummary; +DROP TABLE IF EXISTS diffRun; +DROP TABLE IF EXISTS diffInputSkyfile; +DROP TABLE IF EXISTS diffSkyfile; +DROP TABLE IF EXISTS diffSummary; +DROP TABLE IF EXISTS stackRun; +DROP TABLE IF EXISTS stackInputSkyfile; +DROP TABLE IF EXISTS stackSumSkyfile; +DROP TABLE IF EXISTS stackSummary; +DROP TABLE IF EXISTS stackAssociation; +DROP TABLE IF EXISTS stackAssociationMap; +DROP TABLE IF EXISTS magicRun; +DROP TABLE IF EXISTS magicInputSkyfile; +DROP TABLE IF EXISTS magicTree; +DROP TABLE IF EXISTS magicNodeResult; +DROP TABLE IF EXISTS magicMask; +DROP TABLE IF EXISTS magicDSRun; +DROP TABLE IF EXISTS magicDSFile; +DROP TABLE IF EXISTS calDB; +DROP TABLE IF EXISTS calRun; +DROP TABLE IF EXISTS flatcorrRun; +DROP TABLE IF EXISTS flatcorrExp; +DROP TABLE IF EXISTS flatcorrChipLink; +DROP TABLE IF EXISTS flatcorrCamLink; +DROP TABLE IF EXISTS flatcorrAddstarLink; +DROP TABLE IF EXISTS pstampDataStore; +DROP TABLE IF EXISTS pstampProject; +DROP TABLE IF EXISTS pstampRequest; +DROP TABLE IF EXISTS pstampDependent; +DROP TABLE IF EXISTS pstampWebRequest; +DROP TABLE IF EXISTS pstampFile; +DROP TABLE IF EXISTS pstampJob; +DROP TABLE IF EXISTS pstampUserDomain; +DROP TABLE IF EXISTS pstampUser; +DROP TABLE IF EXISTS pstampAccessLevel; +DROP TABLE IF EXISTS distRun; +DROP TABLE IF EXISTS distTarget; +DROP TABLE IF EXISTS distComponent; +DROP TABLE IF EXISTS rcDSProduct; +DROP TABLE IF EXISTS rcDestination; +DROP TABLE IF EXISTS rcInterest; +DROP TABLE IF EXISTS rcDSFileset; +DROP TABLE IF EXISTS rcRun; +DROP TABLE IF EXISTS receiveSource; +DROP TABLE IF EXISTS receiveFileset; +DROP TABLE IF EXISTS receiveFile; +DROP TABLE IF EXISTS receiveResult; +DROP TABLE IF EXISTS publishClient; +DROP TABLE IF EXISTS publishRun; +DROP TABLE IF EXISTS publishDone; +DROP TABLE IF EXISTS staticskyInput; +DROP TABLE IF EXISTS staticskyResult; +DROP TABLE IF EXISTS staticskyRun; +DROP TABLE IF EXISTS Label; +DROP TABLE IF EXISTS chipBackgroundRun; +DROP TABLE IF EXISTS chipBackgroundImfile; +DROP TABLE IF EXISTS warpBackgroundRun; +DROP TABLE IF EXISTS warpBackgroundSkyfile; +DROP TABLE IF EXISTS diffPhotRun; +DROP TABLE IF EXISTS diffPhotSkyfile; +DROP TABLE IF EXISTS lapExp; +DROP TABLE IF EXISTS lapRun; +DROP TABLE IF EXISTS lapSequence; +DROP TABLE IF EXISTS minidvodbRun; +DROP TABLE IF EXISTS minidvodbCopy; +DROP TABLE IF EXISTS minidvodbProcessed; +DROP TABLE IF EXISTS vpProcessedCell; +DROP TABLE IF EXISTS vpRun; +DROP TABLE IF EXISTS skycalRun; +DROP TABLE IF EXISTS skycalResult; +DROP TABLE IF EXISTS skycell; +DROP TABLE IF EXISTS relStack; +DROP TABLE IF EXISTS relExp; +DROP TABLE IF EXISTS relGroup; +DROP TABLE IF EXISTS ippRelease; +DROP TABLE IF EXISTS survey; +DROP TABLE IF EXISTS lapGroup; +DROP TABLE IF EXISTS mergedvodbCopy; +DROP TABLE IF EXISTS mergedvodbProcessed; +DROP TABLE IF EXISTS mergedvodbRun; +DROP TABLE IF EXISTS dqstatsContent; +DROP TABLE IF EXISTS dqstatsRun; +DROP TABLE IF EXISTS fullForceRun; +DROP TABLE IF EXISTS fullForceInput; +DROP TABLE IF EXISTS fullForceResult; +DROP TABLE IF EXISTS fullForceSummary; +DROP TABLE IF EXISTS remoteComponent; +DROP TABLE IF EXISTS remoteRun; + + +SET FOREIGN_KEY_CHECKS=1 Index: /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_update_version.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_update_version.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pxadmin_update_version.sql (revision 41217) @@ -0,0 +1,1 @@ +UPDATE dbversion SET schema_version = '%s', updated = current_timestamp() Index: /branches/ipp-148_http_503_filesset_revert_fix/pztool_find_completed_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pztool_find_completed_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pztool_find_completed_exp.sql (revision 41217) @@ -0,0 +1,34 @@ +SELECT DISTINCT + summit_id, + exp_name, + camera, + telescope, + dateobs, + state +FROM ( + SELECT + pzDownloadImfile.*, + pzDownloadExp.state, + summitExp.imfiles, + summitExp.dateobs + FROM pzDownloadExp + JOIN summitExp + USING(summit_id) + LEFT JOIN pzDownloadImfile + USING(summit_id) + LEFT JOIN newExp + USING(summit_id) + WHERE + pzDownloadExp.state = 'run' + AND newExp.summit_id IS NULL + AND newExp.tmp_exp_name IS NULL + AND newExp.tmp_camera IS NULL + AND newExp.tmp_telescope IS NULL + GROUP BY + pzDownloadExp.summit_id + -- it doesn't matter which field in pzDownloadImfile we count as we've + -- already Download a group by + HAVING + COUNT(pzDownloadImfile.exp_name) = summitExp.imfiles + AND SUM(pzDownloadImfile.fault) = 0 + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/pztool_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pztool_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pztool_pendingimfile.sql (revision 41217) @@ -0,0 +1,46 @@ +SELECT * +FROM ( + SELECT * FROM + (SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(summit_id) + LEFT JOIN pzDownloadImfile + USING(summit_id, class, class_id) + JOIN summitExp + USING(summit_id) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.summit_id IS NULL + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name NOT LIKE 'c%' + ORDER BY dateobs) AS partA +UNION + SELECT * FROM + (SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(summit_id) + LEFT JOIN pzDownloadImfile + USING(summit_id, class, class_id) + JOIN summitExp + USING(summit_id) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.summit_id IS NULL + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name LIKE 'c%' + ORDER BY dateobs) AS partB +) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_downloadimfile_faults.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_downloadimfile_faults.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_downloadimfile_faults.sql (revision 41217) @@ -0,0 +1,21 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 503: try again + fault = 203 + -- HTTP 500: timeout + OR fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault = 115 + -- nebulous timout / ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + -- HTTP 502 Proxy Error (seen now at ITC) + OR fault = 202 + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_fileset_faults.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_fileset_faults.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pztool_revert_fileset_faults.sql (revision 41217) @@ -0,0 +1,19 @@ +UPDATE summitExp SET fault = 0 +WHERE + fault != 0 + -- fault is older than 10 minutes + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) > 60 * 10 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 500: timeout + fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + -- HTTP 502 Proxy Error (seen now at ITC) + OR fault = 202 + ) Index: /branches/ipp-148_http_503_filesset_revert_fix/pztool_revertcopied.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/pztool_revertcopied.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/pztool_revertcopied.sql (revision 41217) @@ -0,0 +1,3 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/rcserver_updatercrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/rcserver_updatercrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/rcserver_updatercrun.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE rcRun +JOIN rcDestination USING(dest_id) +JOIN rcDSFileset using(dest_id, fs_id) +SET Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_addfileset.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_addfileset.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_addfileset.sql (revision 41217) @@ -0,0 +1,4 @@ +SELECT fileset_id +FROM receiveFileset +WHERE source_id = %s + AND fileset = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_list.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_list.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_list.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT * +FROM receiveSource +WHERE receiveSource.state = 'enabled' Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfile.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + receiveFile.file_id, + source, + product, + workdir, + dirinfo, + fileset, + fileset_id, + file, + bytes, + md5sum, + file_type, + component +FROM receiveFile +JOIN receiveFileset USING(fileset_id) +JOIN receiveSource USING(source_id) +LEFT JOIN receiveResult + ON receiveResult.file_id = receiveFile.file_id +WHERE + receiveSource.state = 'enabled' + AND ((receiveFileset.state = 'new') + OR (receiveFileset.state = 'listed' AND receiveFile.component = 'dirinfo' AND receiveFileset.dirinfo IS NULL)) + AND receiveFileset.fault = 0 + AND receiveResult.file_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfileset.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfileset.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_pendingfileset.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT receiveFileset.*, + receiveSource.source, + receiveSource.product, + receiveSource.workdir +FROM receiveFileset +JOIN receiveSource USING(source_id) +LEFT JOIN receiveFile USING(fileset_id) +WHERE receiveSource.state = 'enabled' + AND receiveFileset.state = 'reg' + AND receiveFileset.fault = 0 + AND receiveFile.fileset_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_revert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_revert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_revert.sql (revision 41217) @@ -0,0 +1,6 @@ +DELETE FROM receiveResult +USING receiveResult, receiveFile, receiveFileset, receiveSource +WHERE receiveResult.file_id = receiveFile.file_id + AND receiveFile.fileset_id = receiveFileset.fileset_id + AND receiveFileset.source_id = receiveSource.source_id + AND receiveResult.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/receivetool_toadvance.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/receivetool_toadvance.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/receivetool_toadvance.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT + receiveFileset.fileset_id, + receiveFileset.fileset, + receiveFileset.dbinfo, + receiveSource.status_product, + receiveSource.ds_dbname, + receiveSource.ds_dbhost +FROM receiveFileset + JOIN receiveSource USING(source_id) + JOIN receiveFile USING(fileset_id) + LEFT JOIN receiveResult USING(file_id) +WHERE receiveSource.state = 'enabled' + AND receiveFileset.state = 'new' + AND receiveFileset.fault = 0 +GROUP BY fileset_id +HAVING COUNT(receiveFile.file_id) = COUNT(receiveResult.file_id) + AND SUM(receiveResult.fault) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkburntoolimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkburntoolimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkburntoolimfile.sql (revision 41217) @@ -0,0 +1,34 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + summitExp.exp_type, + rawImfile.dateobs, + rawImfile.obs_mode AS obs_mode, + rawImfile.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + summitImfile.class_id AS summit_class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + JOIN summitImfile USING(summit_id) + LEFT JOIN pzDownloadExp USING(summit_id) + LEFT JOIN newExp USING(summit_id) + LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id) + LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id + LEFT JOIN rawImfile ON (rawImfile.exp_id = newExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitExp.dateobs >= '@DATEOBS_BEGIN@' + AND summitExp.dateobs <= '@DATEOBS_END@' + AND summitImfile.class_id = '@CLASS_ID@' + AND (summitExp.exp_name <= '@EXP_NAME@') +ORDER BY summitExp.dateobs Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkstatus.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkstatus.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_checkstatus.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + summitImfile.class_id AS summit_class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM summitExp + JOIN summitImfile USING(summit_id) + LEFT JOIN pzDownloadExp USING(summit_id) + LEFT JOIN newExp USING(summit_id) + LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id) + LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id + LEFT JOIN rawImfile ON (rawImfile.exp_id = newExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitImfile.class_id = '@CLASS_ID@' Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_create_dup_table.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_create_dup_table.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_create_dup_table.sql (revision 41217) @@ -0,0 +1,2 @@ +CREATE TEMPORARY TABLE duplicate (exp_id BIGINT, PRIMARY KEY(exp_id)) +ENGINE=MEMORY Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_exp.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * FROM rawExp Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_export_imfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * FROM rawImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_finishcompressexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_finishcompressexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_finishcompressexp.sql (revision 41217) @@ -0,0 +1,82 @@ +select DISTINCT + exp_id, + exp_name, + camera, + telescope, + dateobs, + exp_tag, + exp_type, + filelevel, + workdir, + state, + reduction, + dvodb, + tess_id, + end_stage, + filter, + comment, + obs_mode, + obs_group, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temperature, + env_humidity, + env_wind_speed, + env_wind_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + sun_angle, + sun_alt, + moon_angle, + moon_alt, + moon_phase, + hostname, + fault, + epoch, + magicked +FROM + (SELECT V.*, + count(class_id) AS complete + FROM + (SELECT rawExp.*,count(newImfile.tmp_class_id) AS total + FROM rawExp + LEFT JOIN newImfile ON rawExp.exp_id = newImfile.exp_id + WHERE (rawExp.state = 'goto_compressed' OR rawExp.state = 'goto_lossy') + GROUP BY rawExp.exp_id) AS V + JOIN rawImfile ON V.exp_id = rawImfile.exp_id + WHERE ((V.state = 'goto_compressed' AND rawImfile.data_state = 'compressed') OR + (V.state = 'goto_lossy' AND rawImfile.data_state = 'lossy')) + GROUP BY V.exp_id) as foo +WHERE (foo.complete = foo.total) + Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingburntoolimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingburntoolimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingburntoolimfile.sql (revision 41217) @@ -0,0 +1,34 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + newImfile.tmp_class_id, + summitExp.exp_type, + rawImfile.camera AS camera, + rawImfile.dateobs, + rawImfile.obs_mode AS obs_mode, + rawImfile.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + summitImfile.class_id AS summit_class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + JOIN summitImfile USING(summit_id) + LEFT JOIN pzDownloadExp USING(summit_id) + LEFT JOIN newExp USING(summit_id) + LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id) + LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id + LEFT JOIN rawImfile ON (rawImfile.exp_id = newExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitExp.dateobs > '@DATEOBS_BEGIN@' + AND summitExp.dateobs <= '@DATEOBS_END@' +ORDER BY summitImfile.class_id,summitExp.dateobs Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingcompressimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingcompressimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingcompressimfile.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT rawImfile.*,rawExp.workdir,rawExp.exp_tag from rawImfile + JOIN rawExp USING(exp_id) + WHERE 1 +-- ((data_state = 'goto_compressed' AND state = 'goto_compressed') +-- OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) +-- where hook %s +-- limit hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingexp.sql (revision 41217) @@ -0,0 +1,56 @@ +SELECT DISTINCT + summit_id, + exp_id, + tmp_exp_name, + tmp_camera, + tmp_telescope, + state, + workdir, + workdir_state, + reduction, + dvodb, + tess_id, + end_stage, + label, + camera, + filter, + obs_mode, + obs_group, + epoch, + dateobs +FROM + ( + select DISTINCT * from + ( + select newExp.*,count(newImfile.tmp_class_id) AS new_count + from newExp + LEFT JOIN newImfile USING (exp_id) + LEFT JOIN rawExp USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + GROUP BY newExp.exp_id + ) AS NEWEXPOSURES + JOIN + ( + select DISTINCT newExp.exp_id, + rawImfile.camera, + rawImfile.filter, + rawImfile.dateobs, + rawImfile.obs_mode, + rawImfile.obs_group, + count(rawImfile.tmp_class_id) AS raw_count + from newExp + LEFT JOIN rawExp USING (exp_id) + LEFT JOIN rawImfile USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + AND rawImfile.data_state = 'full' + -- where hook %s + GROUP BY newExp.exp_id + HAVING SUM(rawImfile.fault) = 0 + LIMIT 256 + ) AS RAWEXPOSURES + USING (exp_id) + WHERE raw_count = new_count + ) AS Foo +-- limit hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_pendingimfile.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT DISTINCT + newExp.exp_id, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope, + newExp.workdir, + newImfile.tmp_class_id, + newImfile.uri, + newImfile.bytes, + newImfile.md5sum, + summitExp.dateobs AS summit_dateobs +FROM newImfile +JOIN newExp + USING(exp_id) +LEFT JOIN summitExp USING(summit_id) +LEFT JOIN rawImfile + ON newExp.exp_id = rawImfile.exp_id + AND newImfile.tmp_class_id = rawImfile.tmp_class_id +WHERE + newExp.state = 'run' + AND rawImfile.tmp_class_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_populate_dup_table.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_populate_dup_table.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_populate_dup_table.sql (revision 41217) @@ -0,0 +1,13 @@ +INSERT INTO duplicate +SELECT exp_id FROM + ( + SELECT + MAX(exp_id) AS exp_id, +-- tmp_exp_name, +-- tmp_camera, +-- tmp_telescope, + count(exp_id) AS count + FROM newExp + GROUP BY tmp_exp_name, tmp_camera, tmp_telescope + HAVING count > 1 + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedexp.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + *, + DEGREES(ra) / 15. AS rahours, + DEGREES(ra) AS radeg, + DEGREES(decl) AS decdeg +FROM rawExp +-- bogus where clause so there is already a where statement to append too +WHERE rawExp.exp_id IS NOT NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_processedimfile.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + rawImfile.*, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope +FROM rawImfile +JOIN newExp + USING(exp_id) +WHERE rawImfile.ignored = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedexp.sql (revision 41217) @@ -0,0 +1,3 @@ +DELETE FROM rawExp +WHERE + fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_revertprocessedimfile.sql (revision 41217) @@ -0,0 +1,3 @@ +DELETE FROM rawImfile +WHERE + fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyquery.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyquery.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyquery.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE rawExp + SET state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyqueryimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyqueryimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_updatebyqueryimfile.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE rawImfile JOIN rawExp USING(exp_id) + SET data_state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: /branches/ipp-148_http_503_filesset_revert_fix/regtool_updateprocessedimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/regtool_updateprocessedimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/regtool_updateprocessedimfile.sql (revision 41217) @@ -0,0 +1,6 @@ +UPDATE rawImfile + SET %s + WHERE + rawImfile.exp_id = %lld + AND + rawImfile.class_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelexp.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + ippRelease.rel_id, + ippRelease.release_name, + rawExp.exp_name, + rawExp.filter, + rawExp.dateobs, + rawExp.exp_id, + camRun.chip_id, + camRun.cam_id, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev +FROM ippRelease +JOIN camRun +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN relExp AS previousrelexp USING(exp_id, rel_id) + +WHERE previousrelexp.relexp_id IS NULL + AND camRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_data_group.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_data_group.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_data_group.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + ippRelease.rel_id, + count(exp_id) AS num_exposures +FROM ippRelease + LEFT JOIN relExp using(rel_id) + LEFT JOIN camRun using(cam_id) + WHERE relExp.group_id = 0 + AND relExp.state != 'drop' + Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_data_group.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_data_group.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_data_group.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT relExp.relexp_id +FROM relExp + JOIN camRun USING(cam_id) +WHERE + relExp.group_id = 0 + AND relExp.state != 'drop' Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_lap.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_lap.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_exp_lap.sql (revision 41217) @@ -0,0 +1,6 @@ +SELECT relExp.relexp_id +FROM relGroup + JOIN lapRun USING(lap_id) + JOIN lapExp USING(lap_id) + JOIN relExp using(rel_id, exp_id) +WHERE relExp.group_id = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_lap.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_lap.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelgroup_select_lap.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + lapRun.seq_id, + lapRun.lap_id, + ippRelease.rel_id +FROM lapRun + JOIN ippRelease + LEFT JOIN relGroup USING(lap_id) +WHERE (lapRun.state = 'done' or lapRun.state = 'full') + AND relGroup.group_id IS NULL +-- AND lapRun.seq_id = xxx + Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT + ippRelease.rel_id, + stackRun.stack_id, + 0 as skycal_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.filter, + stackSumSkyfile.mjd_obs, + 0 AS zpt_obs, + 0 AS zpt_stdev + +FROM ippRelease +JOIN stackRun +JOIN stackSumSkyfile using(stack_id) +LEFT JOIN relStack AS previousRelStack + ON previousRelStack.rel_id = ippRelease.rel_id + AND previousRelStack.tess_id = stackRun.tess_id + AND previousRelStack.skycell_id = stackRun.skycell_id + AND previousRelStack.filter = stackRun.filter + -- JOIN hook %s +WHERE previousRelStack.relstack_id IS NULL + AND stackRun.state ='full' + AND stackSumSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack_with_skycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack_with_skycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_definerelstack_with_skycal.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT + ippRelease.rel_id, + stackRun.stack_id, + IFNULL(skycalRun.skycal_id, 0) as skycal_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.filter, + stackSumSkyfile.mjd_obs, + skycalResult.zpt_obs, + skycalResult.zpt_stdev + +FROM ippRelease +JOIN stackRun +JOIN stackSumSkyfile USING(stack_id) +JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id +JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id +-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) +LEFT JOIN relStack AS previousRelStack + ON ippRelease.rel_id = previousRelStack.rel_id + AND stackRun.tess_id = previousRelStack.tess_id + AND stackRun.skycell_id = previousRelStack.skycell_id + AND stackRun.filter = previousRelStack.filter +-- JOIN Hook %s + +WHERE previousRelStack.relstack_id IS NULL + AND skycalRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelexp.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE FROM relExp USING relExp, ippRelease, rawExp, camRun +WHERE relExp.rel_id = ippRelease.rel_id + AND relExp.exp_id = rawExp.exp_id + AND relExp.cam_id = camRun.cam_id Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelstack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelstack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_deleterelstack.sql (revision 41217) @@ -0,0 +1,3 @@ +DELETE FROM relStack USING relStack, ippRelease, stackRun +WHERE relStack.rel_id = ippRelease.rel_id + AND relStack.stack_id = stackRun.stack_id Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelease.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelease.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelease.sql (revision 41217) @@ -0,0 +1,1 @@ +select * from ippRelease JOIN survey USING(surveyID) Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelexp.sql (revision 41217) @@ -0,0 +1,45 @@ +SELECT + relexp_id, + relExp.state, + relExp.fault, + ippRelease.release_name, + ippRelease.release_state, + ippRelease.rel_id, + ippRelease.priority, + ippRelease.dvodb, + ippRelease.ubercal_file, + survey.surveyName, + rawExp.exp_name, + relExp.flags, + relExp.zpt_obs, + relExp.zpt_stdev, + relExp.mcal, + relExp.ubercal_dist, + relExp.path_base, + relExp.registered, + relExp.time_stamp, + relExp.exp_id, + relExp.chip_id, + chipRun.state as chip_state, + relExp.cam_id, + camRun.state as cam_state, + camProcessedExp.path_base as cam_path_base, + camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, + warpRun.warp_id, + warpRun.state as warp_state, + warpRun.tess_id, + rawExp.filter, + rawExp.dateobs, + TRUNCATE(DEGREES(rawExp.ra), 4) AS radeg, + TRUNCATE(DEGREES(rawExp.decl), 4) AS decdeg, + rawExp.comment +FROM relExp +JOIN ippRelease USING(rel_id) +JOIN survey USING(surveyID) +JOIN rawExp using(exp_id) +JOIN chipRun using(chip_id, exp_id) +JOIN camRun using(cam_id, chip_id) +JOIN camProcessedExp using(cam_id) +JOIN fakeRun using(cam_id) +JOIN warpRun using(fake_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelstack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelstack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listrelstack.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + release_name, + surveyName, + relStack.*, + stackSumSkyfile.path_base AS stack_path_base, + stackSumSkyfile.quality, + stackRun.state AS stack_state, + stackRun.data_group AS stack_data_group, + skycalResult.fwhm_major, + skycalResult.quality as skycal_quality, + skycalResult.path_base AS skycal_path_base, + skycalRun.data_group AS skycal_data_group, + staticskyResult.sky_id AS sky_id, + staticskyResult.path_base AS staticsky_path_base, + 'GPC1' AS camera +FROM relStack +JOIN ippRelease USING(rel_id) +JOIN survey USING(surveyID) +JOIN stackRun USING(stack_id, filter, tess_id, skycell_id) +JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN skycalRun USING(skycal_id) +LEFT JOIN skycalResult USING(skycal_id) +LEFT JOIN staticskyResult USING(sky_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listsurvey.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listsurvey.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_listsurvey.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * from survey Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_pendingrelgroup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_pendingrelgroup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_pendingrelgroup.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT + relGroup.*, + ippRelease.release_name +FROM relGroup + JOIN ippRelease USING(rel_id) +WHERE relGroup.state = 'new' + AND relGroup.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_setrelstackcalibratedfromskycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_setrelstackcalibratedfromskycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_setrelstackcalibratedfromskycal.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE relStack + JOIN ippRelease USING(rel_id) + JOIN skycell USING(tess_id, skycell_id) + JOIN stackRun USING(stack_id, tess_id, skycell_id) + JOIN skycalRun USING(stack_id) -- join to skycal using stack_id not skycal_id ... + JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id +SET relStack.skycal_id = skycalRun.skycal_id, -- ... because we set skycal_id here + relStack.zpt_obs = skycalResult.zpt_obs, + relStack.zpt_stdev = skycalResult.zpt_stdev, + relStack.state = 'calibrated' Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_stacksummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_stacksummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_stacksummary.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT DISTINCT sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + survey.surveyName, + ippRelease.release_name, + ippRelease.rel_id + +FROM relStack + JOIN ippRelease USING(rel_id) + JOIN survey USING(surveyID) + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + +-- WHERE rel_id = 37 and skycell_id like 'skycell.0635.045' and filter = 'r.00000'\G +-- WHERE rel_id = 37 and projection_cell = 'skycell.0635' and filter = 'r.00000'\G Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_summary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_summary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_summary.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT DISTINCT + release_name, + surveyName, + sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + rawExp.camera +FROM relStack + JOIN ippRelease USING(rel_id) + JOIN survey USING(surveyID) + JOIN stackRun USING(stack_id, filter, tess_id, skycell_id) + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + JOIN stackInputSkyfile using(stack_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/releasetool_tocalibexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/releasetool_tocalibexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/releasetool_tocalibexp.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT relExp.*, +ippRelease.release_name, +ippRelease.release_state +FROM relExp JOIN ippRelease USING(rel_id) +WHERE relExp.state = 'goto_calib' AND relExp.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/reltool_pendinggroup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/reltool_pendinggroup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/reltool_pendinggroup.sql (revision 41217) @@ -0,0 +1,8 @@ +-- relGroups to process +SELECT relGroup.*, + ippRelease.release_id, + ippRelease.release_name +FROM relGroup + JOIN ippRelease USING(rel_id) +WHERE relGroup.state ='new' + AND relGroup.fault = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_cam.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_cam.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_cam.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT cam_id AS stage_id + FROM camRun +WHERE +camRun.state = 'new' +-- where hook %s +GROUP BY cam_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'camera' and remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_chip.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_chip.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_chip.sql (revision 41217) @@ -0,0 +1,26 @@ +SELECT stage_id FROM + +( SELECT chip_id AS stage_id + FROM chipRun +WHERE +chipRun.state = 'new' +-- where hook %s +GROUP BY chip_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'chip' and remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL + + + + + Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_diff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_diff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_diff.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT diff_id AS stage_id + FROM diffRun +WHERE +diffRun.state = 'new' +-- where hook %s +GROUP BY diff_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING(remote_id) +WHERE +remoteRun.stage = 'diff' AND remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING(stage_id) +WHERE R.remote_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_ff.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_ff.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_ff.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT ff_id AS stage_id + FROM fullForceRun +WHERE +fullForceRun.state = 'new' +-- where hook %s +GROUP BY ff_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING(remote_id) +WHERE +remoteRun.stage = 'ff' AND remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING(stage_id) +WHERE R.remote_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_stack.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_stack.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_stack.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT stack_id AS stage_id + FROM stackRun +WHERE +stackRun.state = 'new' +-- where hook %s +GROUP BY stack_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'stack' and remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_staticsky.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_staticsky.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_staticsky.sql (revision 41217) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT sky_id AS stage_id + FROM staticskyRun +WHERE +staticskyRun.state = 'new' +-- where hook %s +GROUP BY sky_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING(remote_id) +WHERE +remoteRun.stage = 'staticsky' AND remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING(stage_id) +WHERE R.remote_id IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_warp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_warp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_definebyquery_warp.sql (revision 41217) @@ -0,0 +1,24 @@ +SELECT stage_id FROM + +( SELECT warp_id AS stage_id, count(warpSkyCellMap.skycell_id) AS N + FROM warpRun + LEFT JOIN warpSkyCellMap USING(warp_id) +WHERE +warpRun.state = 'new' +-- where hook %s +GROUP BY warp_id +HAVING N != 0 +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'warp' and remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_doneprep.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_doneprep.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_doneprep.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT * FROM + (SELECT + remote_id, + remoteRun.path_base, + stage, + count(stage_id) as Ndone +FROM remoteRun +JOIN remoteComponent +USING (remote_id) +WHERE +((remoteComponent.state = 'prep_done') OR (remoteComponent.state = 'prep_fail') OR (remoteComponent.state = 'fail')) +-- and hook %s +GROUP BY remote_id) +AS DONE + +JOIN + +(SELECT +remote_id, +count(stage_id) as Njobs +FROM remoteRun +JOIN remoteComponent +USING (remote_id) +-- where hook %s +GROUP BY remote_id) +AS JOBS + +using (remote_id) + +where Ndone = Njobs Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_dropcomponent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_dropcomponent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_dropcomponent.sql (revision 41217) @@ -0,0 +1,4 @@ +DELETE FROM remoteComponent +WHERE +remote_id = %ld AND +stage_id = %ld Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_listcomponent.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_listcomponent.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_listcomponent.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT DISTINCT + remoteComponent.remote_id, + remoteComponent.stage_id, + remoteComponent.jobs, + remoteComponent.state, + remoteComponent.path_base, + remoteRun.stage, + remoteRun.path_base as run_path_base +FROM remoteRun +JOIN remoteComponent + USING(remote_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/remotetool_updatepoll.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/remotetool_updatepoll.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/remotetool_updatepoll.sql (revision 41217) @@ -0,0 +1,3 @@ +UPDATE remoteRun SET last_poll = CURRENT_TIMESTAMP +WHERE +%s Index: /branches/ipp-148_http_503_filesset_revert_fix/sctool_list.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/sctool_list.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/sctool_list.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + skycell.tess_id, + skycell.skycell_id, + TRUNCATE(skycell.radeg/15., 4) AS rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat, + skycell.width, + skycell.height +FROM skycell Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_addsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_addsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_addsummary.sql (revision 41217) @@ -0,0 +1,5 @@ +INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES ( + %lld, -- sass_id + '%s', -- projection_cell + '%s' -- path_base +) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_associationdefine_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_associationdefine_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_associationdefine_select.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + sass_id, + data_group, + projection_cell, + tess_id, + filter + FROM stackAssociation +RIGHT JOIN ( + SELECT + data_group, + tess_id, + filter, + CASE WHEN LOCATE('.',skycell_id,9) > 0 THEN + SUBSTRING_INDEX(skycell_id,'.',2) ELSE + SUBSTRING_INDEX(skycell_id,'.',1) END + AS projection_cell + FROM stackRun + WHERE stackRun.stack_id = @STACK_ID@ + ) AS RUN USING (data_group,tess_id,filter,projection_cell) LIMIT 1 Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert.sql (revision 41217) @@ -0,0 +1,25 @@ +-- Define the inputs to a defined stack +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warpRun.warp_id +FROM warpSkyfile +JOIN warpRun + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +WHERE + warpSkyfile.skycell_id = '@SKYCELL_ID@' + AND warpRun.state = 'full' + AND rawExp.filter = '@FILTER@' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part1.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part1.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part1.sql (revision 41217) @@ -0,0 +1,34 @@ +-- This is the part 1 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part2.sql should be appended. +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warp_id +FROM ( + -- Sub-select to get the random set of warps + SELECT + warpSkyfile.*, + rand() AS rnd_num + FROM warpSkyfile + JOIN warpRun + USING(warp_id) + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN camProcessedExp + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + skycell_id = '@SKYCELL_ID@' + AND warpRun.state = 'full' + AND rawExp.filter = '@FILTER@' -- the result of the query is grouped by filter and inserted for one at a time + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 +-- Put additional constraints here +-- stacktool_definebyquery_insert_random_part2.sql goes here Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part2.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part2.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_insert_random_part2.sql (revision 41217) @@ -0,0 +1,8 @@ +-- This is the part 2 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part1.sql should be prepended. + -- Sort by the random number, and take the first N + -- to get a random set of N. + ORDER BY rnd_num + LIMIT @RANDOM_LIMIT@ +) AS randomWarps Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_select.sql (revision 41217) @@ -0,0 +1,54 @@ +-- This is a query to get a list of skycells with warps to be stacked, +-- along with the number of warps already in a stack. + +SELECT + tess_id, + skycell_id, + filter, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + warpSkyfile.tess_id as tess_id, + skycell_id, + rawExp.filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + -- WHERE hook %s + GROUP BY + warpSkyfile.tess_id, + skycell_id, + filter + ) AS warpsToStack +LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + tess_id, + skycell_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + -- WHERE hook %s + GROUP BY + stack_id + ) AS stackSizes +-- JOINing the warpsToStack and stackSizes tables + USING(tess_id, skycell_id, filter) + ) +GROUP BY + tess_id, + skycell_id, + filter + Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_test.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_test.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_definebyquery_test.sql (revision 41217) @@ -0,0 +1,59 @@ +-- This is the combination of the two parts of the query to get a list +-- of skycells with warps to be stacked, along with the number of +-- warps already in a stack. + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO stacktool_definebyquery_part1.sql +-- AND stacktool_definebyquery_part2.sql +-------------------------------------------------------------------------------- + +-- stacktool_definebyquery_part1.sql +SELECT + skycell_id, + tess_id, + filter, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.quality = 0 + AND warpSkyfile.fault = 0 + -- Any additional selection on warps/exposures goes here +-- stacktool_definebyquery_part2.sql + GROUP BY + skycell_id, + filter + ) AS warpsToStack + LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + GROUP BY + stack_id + ) AS stackSizes + -- JOINing the warpsToStack and stackSizes tables + USING(skycell_id, filter) + ) + GROUP BY + skycell_id, + filter +; Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + stackRun.* +FROM stackRun +WHERE + stackRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_input_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_input_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_input_skyfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT stackInputSkyfile.* FROM stackInputSkyfile Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_run.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT stackRun.* from stackRun Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_sum_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_sum_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_export_sum_skyfile.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT stackSumSkyfile.* FROM stackSumSkyfile Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_find_complete_warps.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_find_complete_warps.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_find_complete_warps.sql (revision 41217) @@ -0,0 +1,46 @@ +SELECT + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + stackRun.stack_id, + COUNT(warpSkyfile.skycell_id) AS num_avail, + COUNT(stackRun.stack_id) AS num_extant +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN stackInputSkyfile + ON warpSkyfile.warp_id = stackInputSkyfile.warp_id +LEFT JOIN stackRun + ON stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.stack_id = stackInputSkyfile.stack_id +WHERE + warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 +GROUP BY + warpSkyfile.skycell_id, stackRun.stack_id +HAVING + num_avail > num_extant + +-- It seems like we should be grouping the results here but in fact we don't +-- want to do that as each warp_id may contain multiple skycells. So a warp_id +-- may end up in more than one stackRun (for different skycells) +-- GROUP BY +-- warp_id + + +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + 13, + warp_id +FROM warpSkyfile +WHERE + skycell_id = 'new'; Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_inputskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_inputskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_inputskyfile.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + warpSkyfile.*, + rawExp.camera, + rawExp.exp_id, + rawExp.exp_name, + rawExp.exp_time, + rawExp.object, + rawExp.dateobs, + rawExp.comment +FROM stackRun +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN warpSkyfile + ON warpSkyfile.warp_id = warpRun.warp_id + AND warpSkyfile.tess_id = stackRun.tess_id + AND warpSkyfile.skycell_id = stackRun.skycell_id +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,31 @@ +-- results in too many runs adding DISTINCT fixes this +-- is the camera needed for this? YES I think so that update can define the camera +-- is the camera unique for this? YES. Well, sort of. +-- stack_skycell.pl insures that the files +-- we're here to clean come from the same camera so +-- if the camera is not unique we have nothing to do +SELECT DISTINCT + stackRun.stack_id, + rawExp.camera, + stackRun.state, + stackSumSkyfile.path_base +FROM stackRun +JOIN stackSumSkyfile using(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING (exp_id) +WHERE + (stackRun.state = 'goto_cleaned' OR stackRun.state = 'goto_scrubbed' OR stackRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanupskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanupskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_pendingcleanupskyfile.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + stackRun.workdir, + stackRun.dvodb, + stackRun.tess_id, + stackRun.skycell_id +FROM stackRun +JOIN stackSumSkyfile + USING(stack_id) +WHERE + (stackRun.state = 'goto_cleaned' OR stackRun.state = 'goto_scrubbed' OR stackRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsummary.sql (revision 41217) @@ -0,0 +1,3 @@ +DELETE FROM stackSummary + WHERE + sass_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsumskyfile_delete.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsumskyfile_delete.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_revertsumskyfile_delete.sql (revision 41217) @@ -0,0 +1,5 @@ +DELETE FROM stackSumSkyfile +USING stackSumSkyfile, stackRun +WHERE stackSumSkyfile.stack_id = stackRun.stack_id + AND stackRun.state = 'new' + AND fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sassskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sassskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sassskyfile.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + stackAssociation.sass_id, + stackAssociation.projection_cell, + stackSumSkyfile.*, + stackRun.state, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + stackRun.workdir, + stackRun.label, + (SELECT rawExp.camera FROM + stackInputSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun ON warpRun.fake_id = fakeRun.fake_id + JOIN camRun ON camRun.cam_id = fakeRun.cam_id + JOIN chipRun ON camRun.chip_id = chipRun.chip_id + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id + where stack_id = stackRun.stack_id limit 1 + ) as camera +FROM stackAssociation +JOIN stackAssociationMap USING(sass_id) +JOIN stackRun USING(stack_id) +LEFT JOIN stackSumSkyfile USING(stack_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_summary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_summary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_summary.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT DISTINCT sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + rawExp.camera +FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + JOIN stackInputSkyfile using(stack_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sumskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sumskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_sumskyfile.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + stackRun.workdir, + stackRun.label, + (SELECT rawExp.camera FROM + stackInputSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun ON warpRun.fake_id = fakeRun.fake_id + JOIN camRun ON camRun.cam_id = fakeRun.cam_id + JOIN chipRun ON camRun.chip_id = chipRun.chip_id + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id + where stack_id = stackRun.stack_id limit 1 + ) as camera +FROM stackRun +JOIN stackSumSkyfile +USING(stack_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tobkg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tobkg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tobkg.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT + stackRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.workdir, + stackRun.reduction, + stackRun.label, + stackRun.state, + stackSumSkyfile.path_base, + rawExp.camera, + IFNULL(Label.priority, 10000) AS priority +FROM stackRun +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN warpSkyfile USING(warp_id,skycell_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN Label ON Label.label = stackRun.label +WHERE + ((stackRun.state = 'full' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 AND stackSumSkyfile.background_model != 1)) + AND (Label.active OR Label.active IS NULL) + -- WHERE hook %s +GROUP BY stack_id +HAVING ((SUM(IF(warpRun.state = 'cleaned', 1, 0)) = COUNT(stackInputSkyfile.warp_id) OR + SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id)) AND + SUM(IF(warpSkyfile.background_model = 1, 1, 0)) = COUNT(stackInputSkyfile.warp_id)) + Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosum.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosum.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosum.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + stackRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.workdir, + stackRun.reduction, + stackRun.label, + stackRun.state, + stackSumSkyfile.path_base, + IFNULL(Label.priority, 10000) AS priority +FROM stackRun +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +LEFT JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN Label ON Label.label = stackRun.label +WHERE + ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL) + OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0)) + AND (Label.active OR Label.active IS NULL) + -- WHERE hook %s +GROUP BY stack_id +HAVING SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/stacktool_tosummary.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT sass_id,camera,workdir,tess_id,state FROM +(SELECT sass_id,rawExp.camera,stackRun.workdir,stackRun.tess_id,stackRun.state, + sum((stackRun.state = 'full')) AS is_done, sum(1) AS is_defined + FROM stackRun + JOIN stackInputSkyfile ON stackRun.stack_id = stackInputSkyfile.stack_id + JOIN stackAssociationMap ON stackRun.stack_id = stackAssociationMap.stack_id + JOIN stackAssociation USING(sass_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN stackSummary USING(sass_id) +WHERE stackRun.state = 'full' AND + stackSummary.projection_cell IS NULL +-- WHERE HOOK %s +GROUP BY sass_id +) AS T +WHERE is_done = is_defined Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY + tess_id, + skycell_id + ) AS TMP +WHERE num_filter == %d + Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_inputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_inputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_inputs.sql (revision 41217) @@ -0,0 +1,16 @@ + SELECT + MAX(stack_id) as stack_id, + tess_id, + skycell_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND stackRun.tess_id = '%s' + AND stackRun.skycell_id = '%s' + -- WHERE hook 1 %s + -- WHERE hook 2 %s + GROUP BY stackRun.filter Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select.sql (revision 41217) @@ -0,0 +1,41 @@ +SELECT DISTINCT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + JOIN skycell using(tess_id, skycell_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by selected filters, stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY + tess_id, + skycell_id + ) AS Stacks +LEFT JOIN ( + SELECT DISTINCT sky_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.data_group, + COUNT(stack_id) AS num_filter + FROM staticskyRun + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + WHERE 1 + -- restrict by selected filters, stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + -- and unless -rerun is supplied that no existing staticskyRun exists + -- WHERE hook 2 %s + GROUP BY sky_id + ) AS ExistingRunsInputs +USING(tess_id, skycell_id, num_filter) +WHERE num_filter = %d +-- WHERE hook 3 %s Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select_by_dg.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select_by_dg.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_definebyquery_select_by_dg.sql (revision 41217) @@ -0,0 +1,43 @@ +SELECT DISTINCT + tess_id, + skycell_id, + Stacks.data_group, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + label, + data_group, + COUNT(DISTINCT filter) AS num_filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by selected filters, stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY + tess_id, + skycell_id, + data_group + ) AS Stacks +LEFT JOIN ( + SELECT DISTINCT sky_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.data_group, + COUNT(stack_id) AS num_filter + FROM staticskyRun + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + WHERE 1 + -- WHERE hook 2 %s + -- restrict by selected filters, stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY sky_id + ) AS ExistingRunsInputs +USING(tess_id, skycell_id, data_group, num_filter) +WHERE num_filter = %d + AND Stacks.data_group = ExistingRunsInputs.data_group +-- WHERE hook 3 %s Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_defineskycalrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_defineskycalrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_defineskycalrun.sql (revision 41217) @@ -0,0 +1,19 @@ +SELECT + staticskyRun.sky_id, + stackRun.stack_id, + skycell_id, + filter, + staticskyRun.workdir, + staticskyRun.label, + staticskyRun.data_group, + staticskyRun.dist_group +FROM staticskyRun + JOIN staticskyResult USING(sky_id) + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + LEFT JOIN skycalRun ON staticskyRun.sky_id = skycalRun.sky_id AND stackRun.stack_id = skycalRun.stack_id -- join hook %s +WHERE staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_input.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_input.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_input.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT staticskyInput.* FROM staticskyInput Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_result.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_result.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_result.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT staticskyResult.* FROM staticskyResult Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_run.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT staticskyRun.* from staticskyRun Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalresult.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalresult.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalresult.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * from skycalResult Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_export_skycalrun.sql (revision 41217) @@ -0,0 +1,1 @@ +SELECT * from skycalRun Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_inputs.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_inputs.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_inputs.sql (revision 41217) @@ -0,0 +1,9 @@ +SELECT + staticskyRun.*, + staticskyInput.stack_id, + stackSumSkyfile.path_base, + stackRun.filter +FROM staticskyRun +JOIN staticskyInput USING(sky_id) +JOIN stackSumSkyfile USING (stack_id) +JOIN stackRun using(stack_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_pendingskycalrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_pendingskycalrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_pendingskycalrun.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT DISTINCT + skycalRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + staticskyResult.path_base, + rawExp.camera, + staticskyResult.num_inputs = 1 AS singlefilter +FROM skycalRun +JOIN staticskyRun USING (sky_id) +JOIN staticskyResult USING(sky_id) +JOIN stackRun USING (stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING (chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN skycalResult USING(skycal_id) +WHERE + ((skycalRun.state = 'new' AND skycalResult.skycal_id IS NULL) + OR (skycalRun.state ='update' and staticskyRun.state ='full' and skycalResult.fault = 0)) Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_result.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_result.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_result.sql (revision 41217) @@ -0,0 +1,18 @@ +SELECT + staticskyResult.*, + staticskyRun.state, + staticskyRun.workdir, + staticskyRun.label, + stackRun.tess_id, + stackRun.skycell_id, + count(stackRun.filter) AS num_filters, + TRUNCATE(skycell.radeg/15., 4) AS rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat +FROM staticskyRun +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun USING(stack_id) +LEFT JOIN skycell USING(tess_id, skycell_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert.sql (revision 41217) @@ -0,0 +1,9 @@ +DELETE FROM staticskyResult +USING staticskyResult, staticskyRun, staticskyInput, stackRun, skycell +WHERE staticskyRun.sky_id = staticskyResult.sky_id + AND staticskyRun.sky_id = staticskyInput.sky_id + AND staticskyInput.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id + AND stackRun.skycell_id = skycell.skycell_id + AND staticskyRun.state = 'new' + AND staticskyResult.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert_update.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert_update.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revert_update.sql (revision 41217) @@ -0,0 +1,11 @@ +UPDATE staticskyRun + JOIN staticskyResult USING(sky_id) JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id) +SET staticskyResult.fault = 0 +WHERE staticskyRun.sky_id = staticskyResult.sky_id + AND staticskyRun.sky_id = staticskyInput.sky_id + AND staticskyInput.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id + AND stackRun.skycell_id = skycell.skycell_id + AND staticskyRun.state = 'update' + AND staticskyResult.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revertskycal.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revertskycal.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_revertskycal.sql (revision 41217) @@ -0,0 +1,8 @@ +DELETE FROM skycalResult +USING skycalResult, skycalRun, stackRun, skycell +WHERE skycalRun.skycal_id = skycalResult.skycal_id + AND stackRun.stack_id = skycalRun.stack_id + AND skycalRun.state = 'new' + AND skycalResult.fault != 0 + AND stackRun.tess_id = skycell.tess_id + AND stackRun.skycell_id = skycell.skycell_id Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_skycalresult.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_skycalresult.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_skycalresult.sql (revision 41217) @@ -0,0 +1,15 @@ +SELECT + skycalResult.*, + skycalRun.stack_id, + skycalRun.workdir, + stackRun.filter, + skycalRun.state, + skycalRun.label, + skycalRun.data_group, + skycalRun.dist_group, + skycalRun.sky_id, + skycell.* +FROM skycalRun +JOIN skycalResult USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN skycell USING(tess_id, skycell_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_todo.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_todo.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/staticskytool_todo.sql (revision 41217) @@ -0,0 +1,31 @@ +SELECT + staticskyRun.sky_id, + staticskyRun.workdir, + staticskyRun.reduction, + staticskyRun.label, + staticskyRun.state, + stackRun.tess_id, + stackRun.skycell_id, + TRUNCATE(skycell.radeg/15., 4) as rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat, + staticskyResult.path_base, + IFNULL(Label.priority, 10000) AS priority +FROM staticskyRun +JOIN staticskyInput USING (sky_id) +JOIN stackRun USING (stack_id) +JOIN skycell USING(tess_id, skycell_id) +LEFT JOIN staticskyResult USING(sky_id) +LEFT JOIN Label ON staticskyRun.label = Label.label +WHERE + ((staticskyRun.state = 'new' AND staticskyResult.sky_id IS NULL) + OR (staticskyRun.state = 'update' AND staticskyResult.fault = 0)) + + AND (Label.active OR Label.active IS NULL) + + -- WHERE hook %s +GROUP BY sky_id +HAVING SUM(IF(stackRun.state = 'full', 1, 0)) = COUNT(staticskyInput.sky_id) +ORDER BY priority DESC, sky_id Index: /branches/ipp-148_http_503_filesset_revert_fix/vptool_find_rawexp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/vptool_find_rawexp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/vptool_find_rawexp.sql (revision 41217) @@ -0,0 +1,12 @@ +-- this query is used to find potental rawExps to be queued for vptool +-- processeing +SELECT DISTINCT + rawExp.*, + newExp.label +FROM rawExp +JOIN newExp using (exp_id) +JOIN rawImfile using(exp_id) +LEFT JOIN vpRun USING (exp_id) +WHERE + rawExp.fault = 0 + AND rawImfile.video_cells Index: /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingimfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingimfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingimfile.sql (revision 41217) @@ -0,0 +1,12 @@ +SELECT + vpRun.*, + rawImfile.class_id, + rawImfile.uri +FROM vpRun + JOIN rawExp USING(exp_id) + JOIN rawImfile USING(exp_id) + LEFT JOIN vpProcessedCell USING(vp_id, class_id) +WHERE rawImfile.video_cells + AND vpRun.state = 'new' + AND vpProcessedCell.vp_id IS NULL + Index: /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/vptool_pendingrun.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT vp_id, + rawExp.camera, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + vpRun.label, + vpRun.workdir, + vpRun.dest_id, + rcDestination.name AS dest_name, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost, + IFNULL(Label.priority, 10000) as priority +FROM vpRun + JOIN rawExp USING(exp_id) + LEFT JOIN rcDestination USING(dest_id) + LEFT JOIN Label ON vpRun.label = Label.label +WHERE vpRun.state = 'new' + AND vpRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/ipp-148_http_503_filesset_revert_fix/vptool_processedcell.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/vptool_processedcell.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/vptool_processedcell.sql (revision 41217) @@ -0,0 +1,22 @@ +SELECT + vpProcessedCell.*, + vpRun.state, + vpRun.label, + vpRun.data_group, + vpRun.outroot, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.obs_mode, + rawExp.comment, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM vpRun + JOIN vpProcessedCell USING(vp_id) + JOIN rawExp USING(exp_id) Index: /branches/ipp-148_http_503_filesset_revert_fix/vptool_revertrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/vptool_revertrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/vptool_revertrun.sql (revision 41217) @@ -0,0 +1,4 @@ +UPDATE vpRun + SET fault = 0 +WHERE vpRun.state = 'new' + AND fault > 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_addsummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_addsummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_addsummary.sql (revision 41217) @@ -0,0 +1,5 @@ +INSERT INTO warpSummary (warp_id,projection_cell,path_base) VALUES( + %lld, -- warp_id + '%s', -- projection_cell + '%s' -- path_base +) Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_run_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_run_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_run_state.sql (revision 41217) @@ -0,0 +1,20 @@ +-- change state of warpRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant skyfiles are in the end state +-- arguments are new state (cleaned or purged) warp_id and new state again for +-- the chipProcessedImfile sub query +UPDATE warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) + SET warpRun.state = '%s' + -- set magicked hook %s + WHERE + warpRun.warp_id = %lld + AND (SELECT + COUNT(warp_id) + FROM warpSkyfile + WHERE + warpSkyfile.warp_id = warpRun.warp_id + AND warpSkyfile.data_state != '%s' + AND warpSkyfile.quality = 0 + ) = 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_skyfile_data_state.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_skyfile_data_state.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_change_skyfile_data_state.sql (revision 41217) @@ -0,0 +1,14 @@ +-- handle changes in warpSkyfile.data_state. +-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile +-- arguments are are new data_state, string for magic hook, warp_id, skycell_id +UPDATE warpSkyfile +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) + SET + data_state = '%s' + -- set magicked hook %s +WHERE + warp_id = %lld + AND skycell_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_coalesce_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_coalesce_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_coalesce_run.sql (revision 41217) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from warpSkyfile where warp_id = %lld Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_definebyquery.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_definebyquery.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_definebyquery.sql (revision 41217) @@ -0,0 +1,33 @@ +SELECT DISTINCT + fakeRun.*, + chipRun.chip_id, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.sun_angle +FROM fakeRun +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE fakeRun.state = 'full' +AND camRun.state = 'full' +AND chipRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_donecleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_donecleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_donecleanup.sql (revision 41217) @@ -0,0 +1,5 @@ +SELECT + warpRun.* +FROM warpRun +WHERE + warpRun.state = 'cleaned' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_exp.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_exp.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_exp.sql (revision 41217) @@ -0,0 +1,8 @@ +SELECT + fakeRun.* +FROM warpRun +JOIN fakeRun + USING(fake_id) +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_imfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + warpImfile.* +FROM warpImfile Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_run.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + warpRun.* +FROM warpRun Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skycell_map.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skycell_map.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skycell_map.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + warpSkyCellMap.* +FROM warpSkyCellMap Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_export_skyfile.sql (revision 41217) @@ -0,0 +1,3 @@ +SELECT + warpSkyfile.* +FROM warpSkyfile Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_finish_run.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_finish_run.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_finish_run.sql (revision 41217) @@ -0,0 +1,5 @@ +UPDATE warpRun + SET state = 'full', + magicked = %lld +WHERE warp_id = %lld + Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_finished_run_select.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_finished_run_select.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_finished_run_select.sql (revision 41217) @@ -0,0 +1,23 @@ +SELECT + warp_id, + magicked +FROM + (SELECT DISTINCT + warpRun.warp_id, + warpRun.label as label, + warpSkyCellMap.warp_id as foo, + warpSkyfile.warp_id as bar, + warpSkyfile.magicked as magicked + FROM warpRun + JOIN warpSkyCellMap + USING(warp_id) + LEFT JOIN warpSkyfile + USING(warp_id, skycell_id) + WHERE + warpRun.state = 'new' + GROUP BY + warpRun.warp_id + HAVING + COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id) + AND SUM(warpSkyfile.fault > 0) = 0 + ) as Foo Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_imfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_imfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_imfile.sql (revision 41217) @@ -0,0 +1,30 @@ +SELECT DISTINCT + rawImfile.*, + warpRun.fake_id, + camRun.cam_id as cam_id, + chipProcessedImfile.uri as chip_uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id +JOIN rawImfile -- is there any reason not to refer back to rawimfiles? + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id + AND rawImfile.ignored = 0 +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND chipProcessedImfile.quality = 0 + and camProcessedExp.quality = 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_listrun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_listrun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_listrun.sql (revision 41217) @@ -0,0 +1,25 @@ +SELECT + warpRun.*, + camRun.cam_id, + chipRun.chip_id, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + magicDSRun.state AS dsRun_state, + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id +FROM warpRun +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSRun + ON stage = 'warp' AND stage_id = warp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanuprun.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanuprun.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanuprun.sql (revision 41217) @@ -0,0 +1,17 @@ +SELECT + warpRun.warp_id, + warpRun.workdir, + rawExp.camera, + rawExp.exp_tag, + warpRun.state +FROM warpRun +JOIN fakeRun +USING (fake_id) +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile.sql (revision 41217) @@ -0,0 +1,20 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE + ((warpRun.state = 'goto_cleaned' + AND (warpSkyfile.data_state = 'full' + OR warpSkyfile.data_state = 'error_cleaned' + OR warpSkyfile.data_state = 'update') + AND warpSkyfile.quality = 0) + OR + (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed') + OR + (warpRun.state = 'goto_purged' AND warpSkyfile.data_state != 'purged')) Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile_all.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile_all.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_pendingcleanupskyfile_all.sql (revision 41217) @@ -0,0 +1,11 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged') Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertcleanup.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertcleanup.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertcleanup.sql (revision 41217) @@ -0,0 +1,7 @@ +UPDATE warpSkyfile + JOIN warpRun using(warp_id) +SET warpSkyfile.data_state = 'full', + warpRun.state = '%s' +WHERE + warpRun.state = '%s' + AND warpSkyfile.data_state = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertoverlap.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertoverlap.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertoverlap.sql (revision 41217) @@ -0,0 +1,10 @@ +DELETE FROM warpSkyCellMap +USING warpSkyCellMap, warpRun, fakeRun, camRun, chipRun, rawExp +WHERE warpSkyCellMap.warp_id = warpRun.warp_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND warpRun.state = 'new' + AND warpSkyCellMap.fault != 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_delete.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_delete.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_delete.sql (revision 41217) @@ -0,0 +1,10 @@ +DELETE FROM warpSkyfile +USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +WHERE warpSkyfile.warp_id = warpRun.warp_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND warpRun.state = 'new' + AND warpSkyfile.fault != 0 + Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_updated.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_updated.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_revertwarped_updated.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +SET warpSkyfile.fault = 0 +WHERE warpRun.state = 'update' + AND warpSkyfile.data_state = 'update' + AND warpSkyfile.fault != 0 Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_runstate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_runstate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_runstate.sql (revision 41217) @@ -0,0 +1,16 @@ +SELECT + warp_id, + warpRun.label, + warpRun.state, + warpRun.magicked, + chip_id, + chipRun.state AS chip_state, + exp_id, + magic_ds_id, + magicDSRun.state AS magic_ds_state +FROM warpRun + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON stage = 'warp' AND stage_id = warp_id AND re_place Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_scmap.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_scmap.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_scmap.sql (revision 41217) @@ -0,0 +1,44 @@ +SELECT + warpSkyCellMap.*, + chipProcessedImfile.uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, + camProcessedExp.fault as cam_fault, + camProcessedExp.background_model AS cam_background_model, + chipProcessedImfile.chip_id, + chipRun.state, + chipRun.data_group as chip_data_group, + chipProcessedImfile.data_state, + chipProcessedImfile.fault AS chip_fault, + chipProcessedImfile.magicked, + camRun.state as camState, + rawImfile.magicked AS raw_magicked, + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id, + IFNULL(magicDSRun.state, 0) AS dsRun_state, + IFNULL(magicDSFile.data_state, 0) AS dsFile_data_state, + IFNULL(magicDSFile.fault, 0) as dsFile_fault +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND warpSkyCellMap.class_id = chipProcessedImfile.class_id + AND chipProcessedImfile.quality = 0 +JOIN rawImfile + ON chipRun.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id +LEFT JOIN magicDSRun + ON chipRun.chip_id = magicDSRun.stage_id AND magicDSRun.stage = 'chip' +LEFT JOIN magicDSFile + ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND chipProcessedImfile.class_id = magicDSFile.component +WHERE + 1 Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_setskyfiletoupdate.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_setskyfiletoupdate.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_setskyfiletoupdate.sql (revision 41217) @@ -0,0 +1,10 @@ +UPDATE warpRun + JOIN warpSkyfile USING(warp_id) +SET warpRun.state = 'update', + warpSkyfile.data_state = 'update', + warpSkyfile.fault = 0 + -- set hook %s +WHERE warp_id = %lld + AND (warpRun.state = 'cleaned' OR warpRun.state = 'update') + AND (warpSkyfile.data_state = 'cleaned' OR warpSkyfile.data_state = 'update') + AND (warpSkyfile.quality = 0) Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_tooverlap.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_tooverlap.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_tooverlap.sql (revision 41217) @@ -0,0 +1,29 @@ +SELECT + warpRun.warp_id, + warpRun.fake_id, + warpRun.workdir, + warpRun.tess_id, + warpRun.label, + rawExp.camera, + exp_id, + exp_tag +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN warpSkyCellMap + USING(warp_id) +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpSkyCellMap.warp_id IS NULL + AND warpMask.label IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_tosummary.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_tosummary.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_tosummary.sql (revision 41217) @@ -0,0 +1,7 @@ +SELECT DISTINCT warp_id,rawExp.camera,warpRun.workdir,warpRun.tess_id,rawExp.exp_tag,warpRun.state + FROM warpRun + JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) + LEFT JOIN warpSummary USING(warp_id) +WHERE warpRun.state = 'full' AND + warpSummary.projection_cell IS NULL Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped.sql (revision 41217) @@ -0,0 +1,111 @@ +SELECT + warp_id, + warp_skyfile_id, + skycell_id, + tess_id, + fake_id, + state, + reduction, + cam_id, + camera, + exp_tag, + workdir, + label, + magicked, + path_base, + priority +FROM ( + (SELECT DISTINCT + warpSkyCellMap.warp_id, + warpImfile.warp_skyfile_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + warpRun.state, + warpRun.reduction, + warpRun.label, + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + warpRun.workdir, + chipRun.magicked, + CAST(NULL AS CHAR(255)) AS path_base, + IFNULL(Label.priority, 10000) AS priority + FROM warpRun + JOIN warpSkyCellMap + USING(warp_id) + JOIN warpImfile + ON warpSkyCellMap.warp_id = warpImfile.warp_id + AND warpSkyCellMap.skycell_id = warpImfile.skycell_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN warpSkyfile + ON warpRun.warp_id = warpSkyfile.warp_id + AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id + AND warpSkyCellMap.tess_id = warpSkyfile.tess_id + LEFT JOIN warpMask + ON warpRun.label = warpMask.label + LEFT JOIN Label ON warpRun.label = Label.label + WHERE + warpRun.state = 'new' + AND warpSkyfile.warp_id IS NULL + AND warpSkyfile.skycell_id IS NULL + AND warpSkyfile.tess_id IS NULL + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpMask.label IS NULL + AND warpSkyCellMap.fault = 0 + AND (Label.active OR Label.active IS NULL) + -- where hook 1 %s + -- limit hook 1 %s + ) +UNION + (SELECT + warpSkyCellMap.warp_id, + warpImfile.warp_skyfile_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + warpRun.state, + warpRun.reduction, + warpRun.label, + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + warpRun.workdir, + MIN(chipProcessedImfile.magicked) AS magicked, + warpSkyfile.path_base, + IFNULL(Label.priority, 10000) AS priority + FROM warpRun + JOIN warpImfile USING(warp_id) + JOIN warpSkyCellMap USING(warp_id, skycell_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN chipProcessedImfile USING(chip_id, class_id) + LEFT JOIN Label ON warpRun.label = Label.label + WHERE warpRun.state = 'update' + AND warpSkyfile.data_state = 'update' + AND warpSkyfile.fault = 0 + AND camRun.state = 'full' + AND chipProcessedImfile.quality = 0 + -- Maybe add this to solve the if any chip is bad a whole warp can't be updated problem? + -- AND (chipProcessedImfile.fault = 0 OR chipProcessedImfile.fault = 26) + AND (Label.active OR Label.active IS NULL) + -- where hook 2 %s + GROUP BY warp_id, skycell_id + HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full', 1, 0)) + -- limit hook 2 %s + ) +) as towarped Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped_labels.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped_labels.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_towarped_labels.sql (revision 41217) @@ -0,0 +1,13 @@ +SELECT DISTINCT + warpRun.label, + Label.label AS label_label, + IFNULL(Label.priority, 10000) AS priority, + count(warpRun.warp_id) + FROM warpRun + LEFT JOIN Label ON warpRun.label = Label.label + WHERE + (warpRun.state = 'new' OR warpRun.state = 'update') + AND (Label.active OR Label.active IS NULL) + -- where hook %s + group by warpRun.label + ORDER BY priority DESC, warp_id Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_updateskyfile.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_updateskyfile.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_updateskyfile.sql (revision 41217) @@ -0,0 +1,1 @@ +UPDATE warpSkyfile SET fault = %d WHERE warp_id = %lld AND skycell_id = '%s' Index: /branches/ipp-148_http_503_filesset_revert_fix/warptool_warped.sql =================================================================== --- /branches/ipp-148_http_503_filesset_revert_fix/warptool_warped.sql (revision 41217) +++ /branches/ipp-148_http_503_filesset_revert_fix/warptool_warped.sql (revision 41217) @@ -0,0 +1,33 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.data_group, + warpImfile.warp_skyfile_id, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + camRun.cam_id, + magicDSRun.state AS dsRun_state, + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN warpImfile + USING(warp_id, skycell_id) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSRun + ON stage = 'warp' AND stage_id = warp_id