Index: /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42571) +++ /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42572) @@ -564,5 +564,20 @@ xcstacktool_pendingcleanuprun.sql \ xcstacktool_pendingcleanupskyfile.sql \ - xcstacktool_donecleanup.sql - - + xcstacktool_donecleanup.sql \ + xcskytool_definebyquery_select.sql \ + xcskytool_definebyquery_select_by_dg.sql \ + xcskytool_definebyquery_inputs.sql \ + xcskytool_definexccalrun.sql \ + xcskytool_pendingxccalrun.sql \ + xcskytool_revertxccal.sql \ + xcskytool_xccalresult.sql \ + xcskytool_export_run.sql \ + xcskytool_export_input.sql \ + xcskytool_export_result.sql \ + xcskytool_export_xccalrun.sql \ + xcskytool_export_xccalresult.sql \ + xcskytool_inputs.sql \ + xcskytool_todo.sql \ + xcskytool_result.sql \ + xcskytool_revert.sql \ + xcskytool_revert_update.sql Index: /branches/eam_branches/ipp-20230313/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/pxadmin_create_tables.sql (revision 42571) +++ /branches/eam_branches/ipp-20230313/ippTools/share/pxadmin_create_tables.sql (revision 42572) @@ -2176,4 +2176,100 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +-- xcsky and xccal tables +CREATE TABLE xcskyRun ( + xcsky_id BIGINT AUTO_INCREMENT, -- unique identifier + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(xcsky_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Inputs for xcsky analysis +CREATE TABLE xcskyInput ( + xcsky_id BIGINT, -- static sky identifier + xcstack_id BIGINT, -- stack identifier + PRIMARY KEY(xcsky_id,xcstack_id), + KEY(xcstack_id), + FOREIGN KEY(xcsky_id) REFERENCES xcskyRun(xcsky_id), + FOREIGN KEY(xcstack_id) REFERENCES xcstackSumSkyfile(xcstack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of static sky analysis +CREATE TABLE xcskyResult ( + xcsky_id BIGINT, -- static sky identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + dtime_phot FLOAT, -- elapsed time for photometry + dtime_script FLOAT, -- elapsed time for script + sources INT, -- number of sources + num_inputs INT, -- number of input images + hostname VARCHAR(64) NOT NULL, -- host that executed script + good_frac FLOAT, -- good fraction of skycell + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + PRIMARY KEY(xcsky_id), + KEY(good_frac), + KEY(fault), + KEY(quality), + FOREIGN KEY(xcsky_id) REFERENCES xcskyRun(xcsky_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE xccalRun ( + xccal_id BIGINT AUTO_INCREMENT, -- unique identifier + xcsky_id BIGINT, + xcstack_id BIGINT, + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(xccal_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(xcsky_id), + KEY(xcstack_id), + FOREIGN KEY(xcsky_id) REFERENCES xcskyRun(xcsky_id), + FOREIGN KEY(xcstack_id) REFERENCES xcstackRun(xcstack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of sky calibration analysis +CREATE TABLE xccalResult ( + xccal_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + dtime_astrom FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + n_astrom INT, + n_detections INT, + n_extended INT, + n_forced INT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(xccal_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(xccal_id) REFERENCES xccalRun(xccal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE skycell ( tess_id VARCHAR(64), Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery.sql (revision 42572) @@ -0,0 +1,22 @@ +SELECT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM xcstackRun + JOIN xcstackSumSkyfile USING(xcstack_id) + WHERE xcstackRun.state = 'full' + AND xcstackSumSkyfile.fault = 0 + AND xcstackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by xcstackSumSkyfile.good_frac, xcstackRun.skycell_id, xcstackRun.label + GROUP BY + tess_id, + skycell_id + ) AS TMP +WHERE num_filter == %d + Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_inputs.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_inputs.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_inputs.sql (revision 42572) @@ -0,0 +1,16 @@ + SELECT + MAX(xcstack_id) as xcstack_id, + tess_id, + skycell_id, + filter + FROM xcstackRun + JOIN xcstackSumSkyfile USING(xcstack_id) + JOIN skycell USING(tess_id, skycell_id) + WHERE xcstackRun.state = 'full' + AND xcstackSumSkyfile.fault = 0 + AND xcstackSumSkyfile.quality = 0 + AND xcstackRun.tess_id = '%s' + AND xcstackRun.skycell_id = '%s' + -- WHERE hook 1 %s + -- WHERE hook 2 %s + GROUP BY xcstackRun.filter Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql (revision 42572) @@ -0,0 +1,41 @@ +SELECT DISTINCT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM xcstackRun + JOIN xcstackSumSkyfile USING(xcstack_id) + JOIN skycell using(tess_id, skycell_id) + WHERE xcstackRun.state = 'full' + AND xcstackSumSkyfile.fault = 0 + AND xcstackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by selected filters, xcstackSumSkyfile.good_frac, xcstackRun.skycell_id, xcstackRun.label + GROUP BY + tess_id, + skycell_id + ) AS xcstacks +LEFT JOIN ( + SELECT DISTINCT xcsky_id, + xcstackRun.skycell_id, + xcstackRun.tess_id, + xcstackRun.data_group, + COUNT(xcstack_id) AS num_filter + FROM xcskyRun + JOIN xcskyInput USING(xcsky_id) + JOIN xcstackRun USING(xcstack_id) + JOIN xcstackSumSkyfile USING(xcstack_id) + JOIN skycell USING(tess_id, skycell_id) + WHERE 1 + -- restrict by selected filters, xcstackSumSkyfile.good_frac, xcstackRun.skycell_id, xcstackRun.label + -- and unless -rerun is supplied that no existing xcskyRun exists + -- WHERE hook 2 %s + GROUP BY xcsky_id + ) AS ExistingRunsInputs +USING(tess_id, skycell_id, num_filter) +WHERE num_filter = %d +-- WHERE hook 3 %s Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select_by_dg.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select_by_dg.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select_by_dg.sql (revision 42572) @@ -0,0 +1,43 @@ +SELECT DISTINCT + tess_id, + skycell_id, + Xcstacks.data_group, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + label, + data_group, + COUNT(DISTINCT filter) AS num_filter + FROM xcstackRun + JOIN xcstackSumSkyfile USING(xcstack_id) + WHERE xcstackRun.state = 'full' + AND xcstackSumSkyfile.fault = 0 + AND xcstackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by selected filters, xcstackSumSkyfile.good_frac, xcstackRun.skycell_id, xcstackRun.label + GROUP BY + tess_id, + skycell_id, + data_group + ) AS Xcstacks +LEFT JOIN ( + SELECT DISTINCT xcsky_id, + xcstackRun.skycell_id, + xcstackRun.tess_id, + xcstackRun.data_group, + COUNT(xcstack_id) AS num_filter + FROM xcskyRun + JOIN xcskyInput USING(xcsky_id) + JOIN xcstackRun USING(xcstack_id) + JOIN xcstackSumSkyfile USING(xcstack_id) + WHERE 1 + -- WHERE hook 2 %s + -- restrict by selected filters, xcstackSumSkyfile.good_frac, xcstackRun.skycell_id, xcstackRun.label + GROUP BY xcsky_id + ) AS ExistingRunsInputs +USING(tess_id, skycell_id, data_group, num_filter) +WHERE num_filter = %d + AND Xcstacks.data_group = ExistingRunsInputs.data_group +-- WHERE hook 3 %s Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definexccalrun.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definexccalrun.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definexccalrun.sql (revision 42572) @@ -0,0 +1,18 @@ +SELECT + xcskyRun.xcsky_id, + xcstackRun.xcstack_id, + skycell_id, + filter, + xcskyRun.workdir, + xcskyRun.label, + xcskyRun.data_group, + xcskyRun.dist_group +FROM xcskyRun + JOIN xcskyResult USING(xcsky_id) + JOIN xcskyInput USING(xcsky_id) + JOIN xcstackRun USING(xcstack_id) + JOIN xcstackSumSkyfile USING(xcstack_id) + JOIN skycell USING(tess_id, skycell_id) + LEFT JOIN xccalRun ON xcskyRun.xcsky_id = xccalRun.xcsky_id AND xcstackRun.xcstack_id = xccalRun.xcstack_id -- join hook %s +WHERE xcskyRun.state = 'full' + AND xcskyResult.quality = 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_input.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_input.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_input.sql (revision 42572) @@ -0,0 +1,1 @@ +SELECT xcskyInput.* FROM xcskyInput Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_result.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_result.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_result.sql (revision 42572) @@ -0,0 +1,1 @@ +SELECT xcskyResult.* FROM xcskyResult Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_run.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_run.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_run.sql (revision 42572) @@ -0,0 +1,1 @@ +SELECT xcskyRun.* from xcskyRun Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalresult.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalresult.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalresult.sql (revision 42572) @@ -0,0 +1,1 @@ +SELECT * from xccalResult Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalrun.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalrun.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_export_xccalrun.sql (revision 42572) @@ -0,0 +1,1 @@ +SELECT * from xccalRun Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_inputs.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_inputs.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_inputs.sql (revision 42572) @@ -0,0 +1,9 @@ +SELECT + xcskyRun.*, + xcskyInput.xcstack_id, + xcstackSumSkyfile.path_base, + xcstackRun.filter +FROM xcskyRun +JOIN xcskyInput USING(xcsky_id) +JOIN xcstackSumSkyfile USING (xcstack_id) +JOIN xcstackRun using(xcstack_id) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_pendingxccalrun.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_pendingxccalrun.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_pendingxccalrun.sql (revision 42572) @@ -0,0 +1,16 @@ +SELECT DISTINCT + xccalRun.*, + xcstackRun.tess_id, + xcstackRun.skycell_id, + xcstackRun.filter, + xcskyResult.path_base, + xcskyResult.num_inputs = 1 AS singlefilter +FROM xccalRun +JOIN xcskyRun USING (xcsky_id) +JOIN xcskyResult USING(xcsky_id) +JOIN xcstackRun USING (xcstack_id) +JOIN xcstackInputSkyfile USING(xcstack_id) +LEFT JOIN xccalResult USING(xccal_id) +WHERE + ((xccalRun.state = 'new' AND xccalResult.xccal_id IS NULL) + OR (xccalRun.state ='update' and xcskyRun.state ='full' and xccalResult.fault = 0)) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_result.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_result.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_result.sql (revision 42572) @@ -0,0 +1,18 @@ +SELECT + xcskyResult.*, + xcskyRun.state, + xcskyRun.workdir, + xcskyRun.label, + xcstackRun.tess_id, + xcstackRun.skycell_id, + count(xcstackRun.filter) AS num_filters, + TRUNCATE(skycell.radeg/15., 4) AS rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat +FROM xcskyRun +JOIN xcskyResult USING(xcsky_id) +JOIN xcskyInput USING(xcsky_id) +JOIN xcstackRun USING(xcstack_id) +LEFT JOIN skycell USING(tess_id, skycell_id) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert.sql (revision 42572) @@ -0,0 +1,9 @@ +DELETE FROM xcskyResult +USING xcskyResult, xcskyRun, xcskyInput, xcstackRun, skycell +WHERE xcskyRun.xcsky_id = xcskyResult.xcsky_id + AND xcskyRun.xcsky_id = xcskyInput.xcsky_id + AND xcskyInput.xcstack_id = stackRun.stack_id + AND xcstackRun.tess_id = skycell.tess_id + AND xcstackRun.skycell_id = skycell.skycell_id + AND xcskyRun.state = 'new' + AND xcskyResult.fault != 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert_update.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert_update.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revert_update.sql (revision 42572) @@ -0,0 +1,11 @@ +UPDATE xcskyRun + JOIN xcskyResult USING(xcsky_id) JOIN xcskyInput USING(xcsky_id) + JOIN xcstackRun USING(xcstack_id) JOIN skycell USING(tess_id, skycell_id) +SET xcskyResult.fault = 0 +WHERE xcskyRun.xcsky_id = xcskyResult.xcsky_id + AND xcskyRun.xcsky_id = xcskyInput.xcsky_id + AND xcskyInput.xcstack_id = stackRun.xcstack_id + AND xcstackRun.tess_id = skycell.tess_id + AND xcstackRun.skycell_id = skycell.skycell_id + AND xcskyRun.state = 'update' + AND xcskyResult.fault != 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revertxccal.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revertxccal.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_revertxccal.sql (revision 42572) @@ -0,0 +1,8 @@ +DELETE FROM xccalResult +USING xccalResult, xccalRun, xcstackRun, skycell +WHERE xccalRun.xccal_id = xccalResult.xccal_id + AND xcstackRun.stack_id = xccalRun.stack_id + AND xccalRun.state = 'new' + AND xccalResult.fault != 0 + AND xcstackRun.tess_id = skycell.tess_id + AND xcstackRun.skycell_id = skycell.skycell_id Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql (revision 42572) @@ -0,0 +1,29 @@ +SELECT + xcskyRun.xcsky_id, + xcskyRun.workdir, + xcskyRun.reduction, + xcskyRun.label, + xcskyRun.state, + xcstackRun.tess_id, + xcstackRun.skycell_id, + TRUNCATE(skycell.radeg/15., 4) as rahours, + skycell.radeg, + skycell.decdeg, + skycell.glong, + skycell.glat, + xcskyResult.path_base, + IFNULL(Label.priority, 10000) AS priority +FROM xcskyRun +JOIN xcskyInput USING (xcsky_id) +JOIN xcstackRun USING (xcstack_id) +JOIN skycell USING(tess_id, skycell_id) +LEFT JOIN xcskyResult USING(xcsky_id) +LEFT JOIN Label ON xcskyRun.label = Label.label +WHERE + ((xcskyRun.state = 'new' AND xcskyResult.xcsky_id IS NULL) + OR (xcskyRun.state = 'update' AND xcskyResult.fault = 0)) + AND (Label.active OR Label.active IS NULL) + -- WHERE hook %s +GROUP BY xcsky_id +HAVING SUM(IF(xcstackRun.state = 'full', 1, 0)) = COUNT(xcskyInput.xcsky_id) +ORDER BY priority DESC, xcsky_id Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_xccalresult.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_xccalresult.sql (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_xccalresult.sql (revision 42572) @@ -0,0 +1,15 @@ +SELECT + xccalResult.*, + xccalRun.xcstack_id, + xccalRun.workdir, + xcstackRun.filter, + xccalRun.state, + xccalRun.label, + xccalRun.data_group, + xccalRun.dist_group, + xccalRun.xcsky_id, + skycell.* +FROM xccalRun +JOIN xccalResult USING(xccal_id) +JOIN xcstackRun USING(xcstack_id) +JOIN skycell USING(tess_id, skycell_id) Index: /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am (revision 42571) +++ /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am (revision 42572) @@ -25,4 +25,5 @@ staticskytool \ xcstacktool \ + xcskytool \ warptool \ receivetool \ @@ -84,4 +85,5 @@ stacktool.h \ xcstacktool.h \ + xcskytool.h \ staticskytool.h \ warptool.h \ @@ -253,4 +255,10 @@ xcstacktoolConfig.c +xcskytool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +xcskytool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +xcskytool_SOURCES = \ + xcskytool.c \ + xcskytoolConfig.c + pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42572) @@ -0,0 +1,1539 @@ +/* + * xcskytool.c + * + * Copyright (C) 2023 Eugene Magnier (based on staticskytool.c) + * + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "xcskytool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool inputsMode(pxConfig *config); +static bool todoMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool resultMode(pxConfig *config); +static bool revertMode(pxConfig *config); +static bool updateresultMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +static bool definexccalrunMode(pxConfig *config); +static bool updatexccalrunMode(pxConfig *config); +static bool pendingxccalrunMode(pxConfig *config); +static bool addxccalresultMode(pxConfig *config); +static bool xccalresultMode(pxConfig *config); +static bool revertxccalresultMode(pxConfig *config); +static bool updatexccalresultMode(pxConfig *config); +static bool exportxccalrunMode(pxConfig *config); +static bool importxccalrunMode(pxConfig *config); + +static bool setxcskyRunState(pxConfig *config, psS64 xcsky_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = xcskytoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(XCSKYTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(XCSKYTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(XCSKYTOOL_MODE_INPUTS, inputsMode); + MODECASE(XCSKYTOOL_MODE_TODO, todoMode); + MODECASE(XCSKYTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(XCSKYTOOL_MODE_RESULT, resultMode); + MODECASE(XCSKYTOOL_MODE_REVERT, revertMode); + MODECASE(XCSKYTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(XCSKYTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(XCSKYTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(XCSKYTOOL_MODE_DEFINEXCCALRUN, definexccalrunMode); + MODECASE(XCSKYTOOL_MODE_UPDATEXCCALRUN, updatexccalrunMode); + MODECASE(XCSKYTOOL_MODE_PENDINGXCCALRUN, pendingxccalrunMode); + MODECASE(XCSKYTOOL_MODE_ADDXCCALRESULT, addxccalresultMode); + MODECASE(XCSKYTOOL_MODE_UPDATEXCCALRESULT, updatexccalresultMode); + MODECASE(XCSKYTOOL_MODE_REVERTXCCALRESULT, revertxccalresultMode); + MODECASE(XCSKYTOOL_MODE_XCCALRESULT, xccalresultMode); + MODECASE(XCSKYTOOL_MODE_EXPORTXCCALRUN, exportxccalrunMode); + MODECASE(XCSKYTOOL_MODE_IMPORTXCCALRUN, importxccalrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + // optional + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_xcstack_id", "xcstackRun.xcstack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "xcstackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "xcstackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "xcstackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "xcstackRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "xcstackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, whereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + // find the number of requested filters: + psMetadataItem *filters = psMetadataLookup(config->args, "-select_filter"); + psAssert (filters, "-select_filter must exist"); + psAssert (filters->type == PS_DATA_METADATA_MULTI, "-select_filter should be a multi container"); + psAssert (filters->data.list->n, "-select_filter should at least have a place-holder"); + int num_filter = filters->data.list->n; + + PXOPT_LOOKUP_BOOL(group_by_data_group, config->args, "-group_by_data_group", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(check_inputs, config->args, "-check_inputs", false); + + psString select; + if (group_by_data_group) { + select = pxDataGet("xcskytool_definebyquery_select_by_dg.sql"); + } else { + select = pxDataGet("xcskytool_definebyquery_select.sql"); + } + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + // this 'where' is used for both xcskytool_definebyquery_select.sql and xcskytool_definebyquery_inputs.sql + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + psString where2 = NULL; + psString make_unique = NULL; + if (!rerun) { + psStringAppend(&where2, "\n %s\nAND xcskyRun.label = '%s'", where, label); + psStringAppend(&make_unique, "\nAND xcsky_id IS NULL"); + } + + if (!p_psDBRunQueryF(config->dbh, select, where, where2 ? where2 : where, num_filter, make_unique ? make_unique : "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + return false; + } + psFree(select); + psFree(where2); + psFree(make_unique); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("xcskytool: no rows found"); + psFree(output); + psFree(where); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "xcskyInput", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString inputsSQL = pxDataGet("xcskytool_definebyquery_inputs.sql"); + if (!inputsSQL) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + // pull out the skycell_id, tess_id, filter + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(output); + psFree(inputsSQL); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(output); + psFree(inputsSQL); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psString by_data_group = NULL; + psString xcstack_data_group = NULL; + if (group_by_data_group) { + xcstack_data_group = psMetadataLookupStr(&status, row, "data_group"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup data_group"); + psFree(output); + psFree(inputsSQL); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psStringAppend(&by_data_group, "\nAND xcstackRun.data_group = '%s'", xcstack_data_group); + } + + // query for the inputs + if (!p_psDBRunQueryF(config->dbh, inputsSQL, tess_id, skycell_id, where, by_data_group ? by_data_group : "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(inputsSQL); + return false; + } + + psArray *inputs = p_psDBFetchResult(config->dbh); + if (!inputs) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(inputs); + psFree(output); + psFree(inputsSQL); + return false; + } + if (!psArrayLength(inputs)) { + psWarning("xcskytool ERROR: no rows found for known tess_id, skycell_id?"); + continue; + } + + if (check_inputs) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, inputs, "xcskyInput", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(inputs); + continue; + } + + // XXX if we are unable to guarantee that all selected inputs have all and only the + // requested filters, see the code at the bottom of this file (ifdef'ed out for now) + + // insert a new xcsky entry and find its new id + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); + psFree(output); + psFree(inputsSQL); + return false; + } + + // create a xcskyRun + if (!xcskyRunInsert(config->dbh, + 0x0, // xcsky_id + "new", // state + workdir, + label, + data_group ? data_group : (xcstack_data_group ? xcstack_data_group : label), + dist_group, + reduction, + registered, + note + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); + psFree(output); + psFree(inputsSQL); + return false; + } + + psS64 xcsky_id = psDBLastInsertID(config->dbh); + + // loop over the possible inputs and record only the valid ones + for (int j = 0; j < inputs->n; j++) { + psMetadata *inputRow = inputs->data[j]; // Row from select + + // pull out the skycell_id, tess_id, filter + psS64 xcstack_id = psMetadataLookupS64(&status, inputRow, "xcstack_id"); + psAssert(status, "failed to find xcstack_id?"); + + // add a xcskyInput entry + if (!xcskyInputInsert(config->dbh, xcsky_id, xcstack_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); + psFree(output); + psFree(inputsSQL); + return false; + } + } + psFree (inputs); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(inputsSQL); + return false; + } + } + psFree(inputsSQL); + psFree(output); + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcsky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "xcskyRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "xcskyRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "xcskyRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "xcstackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcstackRun.skycell_id", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE xcskyRun JOIN xcskyInput USING(xcsky_id) JOIN xcstackRun using(xcstack_id) JOIN skycell USING(tess_id, skycell_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "xcskyRun", "xcsky_id", "xcskyResult", true, false); + psFree(query); + psFree(where); + + return result; +} + +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // require at least an xcsky id (add better search options) + PXOPT_LOOKUP_S64(xcsky_id, config->args, "-xcsky_id", true, false); + + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64 (where, PS_LIST_TAIL, "xcsky_id", 0, "==", xcsky_id); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("xcskytool_inputs.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "xcskyInput"); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("xcskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "xcskyInput", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-xcsky_id", "xcsky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "xcskyRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("xcskytool_todo.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&where, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQueryF(config->dbh, query, where)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("xcskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "xcskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(xcsky_id, config->args, "-xcsky_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_S32(num_inputs, config->args, "-num_inputs", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // XXX not sure we need a transaction here... + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!xcskyResultInsert(config->dbh, + xcsky_id, + path_base, + dtime_phot, + dtime_script, + sources, + num_inputs, + hostname, + good_frac, + fault, + quality + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fault) { + if (!setxcskyRunState(config, xcsky_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change xcskyRun state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +// XXX what is this used by? what filters are needed? +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcskyRun.xcsky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcstack_id", "xcskyInput.xcstack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "xcskyRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "xcskyRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "xcstackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcstackRun.skycell_id", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "xcskyResult.fault", "=="); + pxskycellAddWhere(config, where); + PXOPT_LOOKUP_S32(num_filters, config->args, "-num_filters", false, false); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("xcskytool_result.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + psStringAppend(&query, "\nGROUP BY xcsky_id"); + if (num_filters) { + psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("xcskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "xcskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcskyResult.xcsky_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "xcskyRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "xcskyResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString delete = pxDataGet("xcskytool_revert.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString update = pxDataGet("xcskytool_revert_update.sql"); + if (!update) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psStringAppend(&update, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); + psLogMsg("xcskytool", PS_LOG_INFO, "Deleted %d rows", numRows); + + if (!p_psDBRunQuery(config->dbh, update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(update); + + numRows = psDBAffectedRows(config->dbh); + psLogMsg("xcskytool", PS_LOG_INFO, "Updated %d rows", numRows); + + psFree(where); + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcsky_id", "=="); + + if (!pxSetFaultCode(config->dbh, "xcskyResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 3; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // XXX unused PXOPT_LOOKUP_S64(det_id, config->args, "-xcsky_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcsky_id", "=="); + + ExportTable tables [] = { + {"xcskyRun", "xcskytool_export_run.sql"}, + {"xcskyInput", "xcskytool_export_input.sql"}, + {"xcskyResult", "xcskytool_export_result.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "xcskyRun")) { + if (!pxSetStateCleaned("xcskyRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + } + + fclose (f); + return true; +} + +bool importrunMode(pxConfig *config) +{ + return false; +} + +bool exportxccalrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // XXX unused PXOPT_LOOKUP_S64(det_id, config->args, "-xcsky_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xccal_id", "=="); + + ExportTable tables [] = { + {"xccalRun", "xcskytool_export_xccalrun.sql"}, + {"xccalResult", "xcskytool_export_xccalresult.sql"}, + }; + + for (int i = 0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "xccalRun")) { + if (!pxSetStateCleaned("xccalRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + } + + fclose (f); + return true; +} + +bool importxccalrunMode(pxConfig *config) +{ + return false; +} + +static bool setxcskyRunState(pxConfig *config, psS64 xcsky_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid xcskyRun state: %s", state); + return false; + } + + char *query = "UPDATE xcskyRun SET state = '%s' WHERE xcsky_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, xcsky_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for xcsky_id %"PRId64, xcsky_id); + return false; + } + return true; + +} + +static bool definexccalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + // none required. We get workdir, etc from xcskyRun if not provided + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_xcsky_id", "xcskyRun.xcsky_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_xcstack_id", "xcstackRun.xcstack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "xcstackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "xcstackRun.filter", "LIKE"); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "xcstackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "xcskyRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "xcskyRun.data_group", "LIKE"); + if (!pxskycellAddWhere(config, whereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("xcskytool_definexccalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + psString labelHook = psStringCopy(""); + if (!rerun) { + if (label) { + psStringAppend(&labelHook, "\nAND xccalRun.label = '%s'", label); + } + psStringAppend(&query, "\nAND xccal_id IS NULL"); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(labelHook); + psFree(query); + return false; + } + psFree(labelHook); + psFree(query); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(output)) { + psWarning("xcskytool: no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "xccalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 xcsky_id = psMetadataLookupS64(&status, row, "xcsky_id"); + psS64 xcstack_id = psMetadataLookupS64(&status, row, "xcstack_id"); + psString sky_workdir = psMetadataLookupStr(&status, row, "workdir"); + psString sky_label = psMetadataLookupStr(&status, row, "label"); + psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString sky_dist_group = psMetadataLookupStr(&status, row, "dist_group"); + + // create a xcskyRun + if (!xccalRunInsert(config->dbh, + 0x0, // xccal_id + xcsky_id, + xcstack_id, + "new", // state + workdir ? workdir : sky_workdir, + label ? label : sky_label, + data_group ? data_group : sky_data_group, + dist_group ? dist_group : sky_dist_group, + reduction, + registered, + note + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + } + psFree(output); + return true; +} + +static bool updatexccalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xccal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xcsky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcstack_id", "xcstack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcstackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "xccalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "xccalRun.state", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE xccalRun JOIN xcstackRun using(xcstack_id) join skycell using(tess_id, skycell_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "xccalRun", "xccal_id", "xccalResult", true, false); + psFree(query); + psFree(where); + + return result; +} + +static bool pendingxccalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-xccal_id", "xccal_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-xcsky_id", "xcsky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "xccalRun.label", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-filter", "xcstackRun.filter", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("xcskytool_pendingxccalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("xcskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingxccalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addxccalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(xccal_id, config->args, "-xccal_id", true, false); + + // optional + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + + + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + PXOPT_LOOKUP_S32(n_detections, config->args,"-n_detections", false, false); + PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_S32(n_forced, config->args, "-n_forced", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + xccalResultRow *row = xccalResultRowAlloc( + xccal_id, + path_base, + dtime_script, + dtime_astrom, + sigma_ra, + sigma_dec, + n_astrom, + n_detections, + n_extended, + n_forced, + zpt_obs, + zpt_stdev, + fwhm_major, + fwhm_minor, + quality, + software_ver, + hostname, + fault + ); + + if (!xccalResultInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + psFree(row); + + if (fault) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + psString query = "UPDATE xccalRun SET state = 'full' WHERE xccal_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, xccal_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool updatexccalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xccal_id", "=="); + + if (!pxSetFaultCode(config->dbh, "xccalResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool xccalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xccalRun.xccal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcsky_id", "xccalRun.xcsky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcstack_id", "xcstackRun.xcstack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "xcstackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcstackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "xcstackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "xccalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "xccalRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-quality", "xccalResult.quality", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "xccalResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("xcskytool_xccalresult.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("xcskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "xcskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool revertxccalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xccalResult.xccal_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "xccalRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-data_group", "xccalRun.data_group", "=="); + pxAddLabelSearchArgs(config, where, "-filter", "xcstackRun.filter", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "xccalResult.fault", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("xcskytool_revertxccal.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("xcskytool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.h =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.h (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.h (revision 42572) @@ -0,0 +1,39 @@ +/* + * xcskytool.h + * + * Copyright (C) 2023 Eugene Magnier (based on staticskytool.h) + * + */ + +#ifndef XCSKYTOOL_H +#define XCSKYTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + XCSKYTOOL_MODE_NONE = 0x0, + XCSKYTOOL_MODE_DEFINEBYQUERY, + XCSKYTOOL_MODE_UPDATERUN, + XCSKYTOOL_MODE_INPUTS, + XCSKYTOOL_MODE_TODO, + XCSKYTOOL_MODE_ADDRESULT, + XCSKYTOOL_MODE_RESULT, + XCSKYTOOL_MODE_REVERT, + XCSKYTOOL_MODE_UPDATERESULT, + XCSKYTOOL_MODE_EXPORTRUN, + XCSKYTOOL_MODE_IMPORTRUN, + + XCSKYTOOL_MODE_DEFINEXCCALRUN, + XCSKYTOOL_MODE_PENDINGXCCALRUN, + XCSKYTOOL_MODE_UPDATEXCCALRUN, + XCSKYTOOL_MODE_ADDXCCALRESULT, + XCSKYTOOL_MODE_UPDATEXCCALRESULT, + XCSKYTOOL_MODE_XCCALRESULT, + XCSKYTOOL_MODE_REVERTXCCALRESULT, + XCSKYTOOL_MODE_EXPORTXCCALRUN, + XCSKYTOOL_MODE_IMPORTXCCALRUN, +} xcskytoolMode; + +pxConfig *xcskytoolConfig(pxConfig *config, int argc, char **argv); + +#endif // XCSKYTOOL_H Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcskytoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcskytoolConfig.c (revision 42572) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcskytoolConfig.c (revision 42572) @@ -0,0 +1,298 @@ +/* + * xcskytoolConfig.c + * + * Copyright (C) 2023 Eugene Magnier (based on staticskytoolConfig.c) + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "xcskytool.h" + +pxConfig *xcskytoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { config = pxConfigAlloc(); } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_xcstack_id", 0, "search for xcstack_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by xcstackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by xcstackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + pxskycellAddArguments(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-group_by_data_group", 0, "queue unique run for each data_group", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess_id", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + pxskycellAddArguments(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(todoArgs); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-xcsky_id", 0, "define xcsky ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-sources", 0, "number of sources", 0); + psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-num_inputs", 0, "number of inputs", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-good_frac", 0, "define fraction of good pixels", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xcstack_id", 0, "search by input xcstack ID (if used num_filters will be wrong)", 0); + pxskycellAddArguments(resultArgs); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertArgs); + psMetadataAddBool(revertArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-xcsky_id", 0, "define xcsky ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-xcsky_id", 0, "export this xcsky ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -definexccalrun + psMetadata *definexccalrunArgs = psMetadataAlloc(); + psMetadataAddS64(definexccalrunArgs, PS_LIST_TAIL, "-select_xcsky_id", 0, "search for xcsky_id", 0); + psMetadataAddS64(definexccalrunArgs, PS_LIST_TAIL, "-select_xcstack_id", 0, "search for xcstack_id", 0); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definexccalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by xcstackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by xcstackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + pxskycellAddArguments(definexccalrunArgs); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definexccalrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(definexccalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definexccalrunArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(definexccalrunArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definexccalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(definexccalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definexccalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -updatexccalrun + psMetadata *updatexccalrunArgs = psMetadataAlloc(); + psMetadataAddS64(updatexccalrunArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0); + psMetadataAddS64(updatexccalrunArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddS64(updatexccalrunArgs, PS_LIST_TAIL, "-xcstack_id", 0, "search by xcstack ID", 0); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxskycellAddArguments(updatexccalrunArgs); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updatexccalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -pendingxccalrun + psMetadata *pendingxccalrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingxccalrunArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0); + psMetadataAddS64(pendingxccalrunArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddStr(pendingxccalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(pendingxccalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddU64(pendingxccalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingxccalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addxccalresult + psMetadata *addxccalresultArgs = psMetadataAlloc(); + psMetadataAddS64(addxccalresultArgs, PS_LIST_TAIL, "-xccal_id", 0, "define xccal_id (required)", 0); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define fwhm (major axis; pixels)", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define fwhm (minor axis; pixels)", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddF32(addxccalresultArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + + psMetadataAddStr(addxccalresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS32(addxccalresultArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + psMetadataAddS32(addxccalresultArgs, PS_LIST_TAIL, "-n_detections", 0, "define number of detections", 0); + psMetadataAddS32(addxccalresultArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended detections", 0); + psMetadataAddS32(addxccalresultArgs, PS_LIST_TAIL, "-n_forced", 0, "define number of forced detections", 0); + + psMetadataAddStr(addxccalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addxccalresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addxccalresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + // -revertxccal + psMetadata *revertxccalArgs= psMetadataAlloc(); + psMetadataAddS64(revertxccalArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0); + psMetadataAddStr(revertxccalArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(revertxccalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0); + psMetadataAddStr(revertxccalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0); + pxskycellAddArguments(revertxccalArgs); + psMetadataAddS16(revertxccalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertxccalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updatexccalresult + psMetadata *updatexccalresultArgs = psMetadataAlloc(); + psMetadataAddS64(updatexccalresultArgs, PS_LIST_TAIL, "-xccal_id", 0, "define xccal ID (required)", 0); + psMetadataAddS16(updatexccalresultArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatexccalresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality code", 0); + + // -xccalresult + psMetadata *xccalresultArgs= psMetadataAlloc(); + psMetadataAddS64(xccalresultArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0); + psMetadataAddS64(xccalresultArgs, PS_LIST_TAIL, "-xcsky_id", 0, "search by xcsky ID", 0); + psMetadataAddS64(xccalresultArgs, PS_LIST_TAIL, "-xcstack_id", 0, "search by xcstack ID", 0); + psMetadataAddStr(xccalresultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(xccalresultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(xccalresultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(xccalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(xccalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(xccalresultArgs, PS_LIST_TAIL, "-quality", 0, "search by quality", 0); + psMetadataAddS16(xccalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(xccalresultArgs); + psMetadataAddU64(xccalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit xccalresult set to N items", 0); + psMetadataAddBool(xccalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -exportxccalrun + psMetadata *exportxccalrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportxccalrunArgs, PS_LIST_TAIL, "-xccal_id", 0, "export this xccal ID (required)", 0); + psMetadataAddStr(exportxccalrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportxccalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportxccalrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + + // -importrun + psMetadata *importxccalrunArgs = psMetadataAlloc(); + psMetadataAddStr(importxccalrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new run", XCSKYTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", XCSKYTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputs", "Get inputs", XCSKYTOOL_MODE_INPUTS, inputsArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", XCSKYTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result of run", XCSKYTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-result", "Get result of run", XCSKYTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed run", XCSKYTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update fault for run", XCSKYTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-exportrun", "Export run", XCSKYTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "Import run", XCSKYTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-definexccalrun", "Define a new xccalrun", XCSKYTOOL_MODE_DEFINEXCCALRUN, definexccalrunArgs); + PXOPT_ADD_MODE("-updatexccalrun", "Update a xccalrun", XCSKYTOOL_MODE_UPDATEXCCALRUN, updatexccalrunArgs); + PXOPT_ADD_MODE("-pendingxccalrun", "Get xccal runs to do", XCSKYTOOL_MODE_PENDINGXCCALRUN, pendingxccalrunArgs); + PXOPT_ADD_MODE("-addxccalresult", "add xccal result", XCSKYTOOL_MODE_ADDXCCALRESULT, addxccalresultArgs); + PXOPT_ADD_MODE("-revertxccal", "revert faulted xccal run", XCSKYTOOL_MODE_REVERTXCCALRESULT, revertxccalArgs); + PXOPT_ADD_MODE("-updatexccal", "revert faulted xccal run", XCSKYTOOL_MODE_UPDATEXCCALRESULT, updatexccalresultArgs); + PXOPT_ADD_MODE("-xccalresult", "Get result of xccal run", XCSKYTOOL_MODE_XCCALRESULT, xccalresultArgs); + PXOPT_ADD_MODE("-exportxccalrun", "Export xccal run", XCSKYTOOL_MODE_EXPORTXCCALRUN, exportxccalrunArgs); + PXOPT_ADD_MODE("-importxccalrun", "Import xccal run", XCSKYTOOL_MODE_IMPORTXCCALRUN, importxccalrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +}