IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 4, 2008, 1:44:33 PM (18 years ago)
Author:
bills
Message:

on branch. Reorganizing the diff stage to consist of a collection of
skyfiles

Location:
branches/bills_081204/ippTools
Files:
14 edited

Legend:

Unmodified
Added
Removed

  • branches/bills_081204/ippTools/share/Makefile.am

    r20841 r20903  
    7070     dettool_toresidimfile.sql \
    7171     dettool_tostacked.sql \
     72     difftool_completed_runs.sql \
    7273     difftool_definebyquery.sql \
    7374     difftool_donecleanup.sql \

  • branches/bills_081204/ippTools/share/difftool_definebyquery.sql

    r20682 r20903  
    99    warpsToDiff.good_frac,
    1010    warpsToDiff.diff_id,
    11     warpsToDiff.kind,
    1211    current_stack_id,
    13     best_stack_id
     12    best_stack_id,
     13    exp_id
    1414FROM (
    1515    -- Get list of warps that can be diffed, with any associated diff
    … …  
    2323        warpRun.label as warp_label,
    2424        diffInputs.diff_id,
    25         diffInputs.kind,
    26         diffTemplates.stack_id AS current_stack_id
     25        diffInputs.stack2 AS current_stack_id,
     26        rawExp.exp_id
    2727    FROM warpSkyfile
    2828    JOIN warpRun USING(warp_id)
    … …  
    3333    -- Check if it has an associated diff
    3434    LEFT JOIN diffInputSkyfile AS diffInputs
    35         ON diffInputs.warp_id = warpSkyfile.warp_id
     35        ON diffInputs.warp1 = warpSkyfile.warp_id
    3636        AND diffInputs.skycell_id = warpSkyfile.skycell_id
    37         AND diffInputs.template = 0 -- only join input files
    38     -- Get the stack_id currently used as a template, if any
    39     LEFT JOIN diffInputSkyfile AS diffTemplates
    40         ON diffTemplates.diff_id = diffInputs.diff_id
    41         AND diffTemplates.template != 0 -- only join template files
     37        AND diffInputs.stack2 IS NOT NULL
    4238    WHERE
    4339        warpSkyfile.fault = 0

  • branches/bills_081204/ippTools/share/difftool_inputskyfile.sql

    r19582 r20903  
    11SELECT * FROM
    2     (SELECT
     2    (SELECT
     3        -- warp input
    34        diffRun.diff_id,
    4         diffRun.skycell_id,
    5         diffRun.tess_id,
     5        diffInputSkyfile.skycell_id,
     6        diffInputSkyfile.tess_id,
    67        0 as stack_id,
    78        warpSkyfile.warp_id,
    89        warpSkyfile.uri,
    910        warpSkyfile.path_base,
    10         diffInputSkyfile.template,
     11        0 as template,
    1112        rawExp.camera
    1213    FROM diffRun
    … …  
    1415        USING(diff_id)
    1516    JOIN warpSkyfile
    16         ON  diffInputSkyfile.warp_id    = warpSkyfile.warp_id
     17        ON  diffInputSkyfile.warp1      = warpSkyfile.warp_id
    1718        AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
    1819        AND diffInputSkyfile.tess_id    = warpSkyfile.tess_id
    1920    JOIN warpRun
    20         ON diffInputSkyfile.warp_id = warpRun.warp_id
     21        ON diffInputSkyfile.warp1 = warpRun.warp_id
     22    JOIN fakeRun
     23        USING(fake_id)
     24    JOIN camRun
     25        USING(cam_id)
     26    JOIN chipRun
     27        USING(chip_id)
     28    JOIN chipProcessedImfile
     29        USING(chip_id)
     30    JOIN rawExp
     31        ON chipRun.exp_id = rawExp.exp_id
     32    WHERE
     33        diffRun.state = 'new'
     34        AND warpRun.state = 'full'
     35        AND fakeRun.state = 'full'
     36        AND camRun.state = 'full'
     37        AND chipRun.state = 'full'
     38        -- where hook %s
     39    UNION
     40    SELECT
     41        -- warp template
     42        diffRun.diff_id,
     43        diffInputSkyfile.skycell_id,
     44        diffInputSkyfile.tess_id,
     45        0 as stack_id,
     46        warpSkyfile.warp_id,
     47        warpSkyfile.uri,
     48        warpSkyfile.path_base,
     49        0 as template,
     50        rawExp.camera
     51    FROM diffRun
     52    JOIN diffInputSkyfile
     53        USING(diff_id)
     54    JOIN warpSkyfile
     55        ON  diffInputSkyfile.warp2      = warpSkyfile.warp_id
     56        AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
     57        AND diffInputSkyfile.tess_id    = warpSkyfile.tess_id
     58    JOIN warpRun
     59        ON diffInputSkyfile.warp2 = warpRun.warp_id
    2160    JOIN fakeRun
    2261        USING(fake_id)
    … …  
    3877    UNION
    3978    SELECT
     79        -- stack input
    4080        diffRun.diff_id,
    41         diffRun.skycell_id,
    42         diffRun.tess_id,
     81        diffInputSkyfile.skycell_id,
     82        diffInputSkyfile.tess_id,
    4383        stackSumSkyfile.stack_id,
    4484        0 as warp_id,
    4585        stackSumSkyfile.uri,
    4686        stackSumSkyfile.path_base,
    47         diffInputSkyfile.template,
     87        1 as template,
    4888        rawExp.camera
    4989    FROM diffRun
    … …  
    5191        USING(diff_id)
    5292    JOIN stackSumSkyfile
    53         ON  diffInputSkyfile.stack_id = stackSumSkyfile.stack_id
     93        ON  diffInputSkyfile.stack1 = stackSumSkyfile.stack_id
    5494    JOIN stackInputSkyfile
    55         ON diffInputSkyfile.stack_id = stackInputSkyfile.stack_id
    56     JOIN warpRun
    57         ON stackInputSkyfile.warp_id = warpRun.warp_id
    58     JOIN fakeRun
    59         USING(fake_id)
    60     JOIN camRun
    61         USING(cam_id)
    62     JOIN chipRun
    63         USING(chip_id)
    64     JOIN chipProcessedImfile
    65         USING(chip_id)
     95        ON diffInputSkyfile.stack1 = stackInputSkyfile.stack_id
    6696    JOIN rawExp
    67         ON chipRun.exp_id = rawExp.exp_id
     97        USING(exp_id)
    6898    WHERE
    6999        diffRun.state = 'new'
    70         AND warpRun.state = 'full'
    71         AND fakeRun.state = 'full'
    72         AND camRun.state = 'full'
    73         AND chipRun.state = 'full'
     100        -- where hook %s
     101    UNION
     102    SELECT
     103        -- stack template
     104        diffRun.diff_id,
     105        diffInputSkyfile.skycell_id,
     106        diffInputSkyfile.tess_id,
     107        stackSumSkyfile.stack_id,
     108        0 as warp_id,
     109        stackSumSkyfile.uri,
     110        stackSumSkyfile.path_base,
     111        1 as template,
     112        rawExp.camera
     113    FROM diffRun
     114    JOIN diffInputSkyfile
     115        USING(diff_id)
     116    JOIN stackSumSkyfile
     117        ON  diffInputSkyfile.stack2 = stackSumSkyfile.stack_id
     118    JOIN stackInputSkyfile
     119        ON diffInputSkyfile.stack2 = stackInputSkyfile.stack_id
     120    JOIN rawExp
     121        USING(exp_id)
     122    WHERE
     123        diffRun.state = 'new'
    74124        -- where hook %s
    75125    ) as Foo

  • branches/bills_081204/ippTools/share/difftool_skyfile.sql

    r20694 r20903  
    11SELECT
    2     diffRun.skycell_id,
     2    diffSkyfile.*,
    33    diffRun.tess_id,
    44    diffRun.state,
    5     diffSkyfile.*,
    6     (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    7         AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0,
    8     (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    9         AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0,
    10     (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    11         AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1,
    12     (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    13         AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1
     5    warp1,
     6    stack1,
     7    warp2,
     8    stack2
     9--    (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     10--        AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0,
     11--    (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     12--        AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0,
     13--    (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     14--        AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1,
     15--    (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     16--        AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1
    1417FROM diffRun
    1518JOIN diffSkyfile
    … …  
    1720JOIN diffInputSkyfile
    1821    ON diffInputSkyfile.diff_id = diffRun.diff_id
    19     AND diffInputSkyfile.template = 0
    2022JOIN warpRun
    21     USING(warp_id)
    22 JOIN fakeRun
    23     USING(fake_id)
    24 JOIN camRun
    25     USING(cam_id)
    26 JOIN chipRun
    27     USING(chip_id)
     23    ON warpRun.warp_id = diffInputSkyfile.warp1
    2824JOIN rawExp
    2925    USING(exp_id)

  • branches/bills_081204/ippTools/share/difftool_todiffskyfile.sql

    r19678 r20903  
    55    diffRun.diff_id,
    66    diffRun.workdir,
    7     diffRun.skycell_id,
     7    diffInputSkyfile.skycell_id,
    88    diffRun.tess_id,
    99    diffRun.label,
    1010    diffRun.state
    1111FROM diffRun
     12JOIN diffInputSkyfile USING(diff_id)
     13
    1214-- Get list of templates for each diffRun
    13 JOIN diffInputSkyfile AS diffTemplateSkyfile
    14     ON diffRun.diff_id = diffTemplateSkyfile.diff_id
    15     AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id
    16     AND diffTemplateSkyfile.template = 1
     15-- JOIN diffInputSkyfile AS diffTemplateSkyfile
     16--     ON diffRun.diff_id = diffTemplateSkyfile.diff_id
     17--     AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id
     18--     AND diffTemplateSkyfile.template = 1
    1719-- Get list of inputs for each diffRun
    18 JOIN diffInputSkyfile
    19     ON diffRun.diff_id = diffInputSkyfile.diff_id
    20     AND diffRun.skycell_id = diffInputSkyfile.skycell_id
    21     AND diffInputSkyfile.template = 0
     20-- JOIN diffInputSkyfile
     21--     ON diffRun.diff_id = diffInputSkyfile.diff_id
     22--     AND diffRun.skycell_id = diffInputSkyfile.skycell_id
     23--     AND diffInputSkyfile.template = 0
     24
    2225-- Get warp templates
    2326LEFT JOIN warpRun AS warpTemplateRun
    24     ON warpTemplateRun.warp_id = diffTemplateSkyfile.warp_id
    25     AND diffTemplateSkyfile.warp_id IS NOT NULL
     27    ON warpTemplateRun.warp_id = diffInputSkyfile.warp2
     28    AND diffInputSkyfile.warp2 IS NOT NULL
    2629LEFT JOIN warpSkyfile AS warpTemplateSkyfile
    2730    ON warpTemplateSkyfile.warp_id = warpTemplateRun.warp_id
    28     AND warpTemplateSkyfile.skycell_id = diffTemplateSkyfile.skycell_id
     31    AND warpTemplateSkyfile.skycell_id = diffInputSkyfile.skycell_id
     32
    2933-- Get warp inputs
    3034LEFT JOIN warpRun
    31     ON warpRun.warp_id = diffInputSkyfile.warp_id
    32     AND diffInputSkyfile.warp_id IS NOT NULL
     35    ON warpRun.warp_id = diffInputSkyfile.warp1
     36    AND diffInputSkyfile.warp1 IS NOT NULL
    3337LEFT JOIN warpSkyfile
    3438    ON warpSkyfile.warp_id = warpRun.warp_id
    3539    AND warpSkyfile.skycell_id = diffInputSkyfile.skycell_id
     40
    3641-- Get stack templates
    3742LEFT JOIN stackRun AS stackTemplateRun
    38     ON stackTemplateRun.stack_id = diffTemplateSkyfile.stack_id
     43    ON stackTemplateRun.stack_id = diffInputSkyfile.stack2
     44    AND diffInputSkyfile.stack2 IS NOT NULL
    3945LEFT JOIN stackSumSkyfile AS stackTemplateSkyfile
    4046    ON stackTemplateSkyfile.stack_id = stackTemplateRun.stack_id
     47
    4148-- Get stack inputs
    4249LEFT JOIN stackRun
    43     ON stackRun.stack_id = diffInputSkyfile.stack_id
    44     AND diffInputSkyfile.warp_id IS NULL
     50    ON stackRun.stack_id = diffInputSkyfile.stack1
     51    AND diffInputSkyfile.stack1 IS NOT NULL
    4552LEFT JOIN stackSumSkyfile
    4653    ON stackSumSkyfile.stack_id = stackRun.stack_id
    47     AND diffInputSkyfile.warp_id IS NULL
     54
    4855-- Get what's already been processed
    4956LEFT JOIN diffSkyfile
    … …  
    5764    )
    5865-- Ensure input warps are available
    59     AND (diffInputSkyfile.warp_id IS NULL
     66    AND (diffInputSkyfile.warp1 IS NULL
    6067    OR (warpRun.state = 'full'
    6168    AND warpSkyfile.fault = 0
    6269    AND warpSkyfile.ignored = 0))
    6370-- Ensure input stacks are available
    64     AND (diffInputSkyfile.stack_id IS NULL
     71    AND (diffInputSkyfile.stack1 IS NULL
    6572    OR (stackRun.state = 'full'
    6673    AND stackSumSkyfile.fault = 0))
    6774-- Ensure template warps are available
    68     AND (diffTemplateSkyfile.warp_id IS NULL
     75    AND (diffInputSkyfile.warp2 IS NULL
    6976    OR (warpTemplateRun.state = 'full'
    7077    AND warpTemplateSkyfile.fault = 0
    7178    AND warpTemplateSkyfile.ignored = 0))
    7279-- Ensure template stacks are available
    73     AND (diffTemplateSkyfile.stack_id IS NULL
     80    AND (diffInputSkyfile.stack2 IS NULL
    7481    OR (stackTemplateRun.state = 'full'
    7582    AND stackTemplateSkyfile.fault = 0))

  • branches/bills_081204/ippTools/share/magictool_definebyquery_insert.sql

    r20890 r20903  
    33SELECT
    44    @MAGIC_ID@, -- Update this with the appropriate magic_id
    5     diff_id,
    65    skycell_id
    7 FROM magicBestDiffs
     6FROM diffSkyfile
    87WHERE
    9     exp_id = @EXP_ID@ -- Update this with the appropriate exp_id
     8    diff_id = @DIFF_ID@ -- Update this with the appropriate diff_id
     9    AND fault = 0

  • branches/bills_081204/ippTools/share/magictool_definebyquery_select.sql

    r20742 r20903  
    11-- Get a list of exposures on which magic may be performed
    2 SELECT DISTINCT
     2SELECT
    33    exp_id,
    4     filter,
    5     num_todo,
    6     num_done,
    7     MAX(magic_id)
    8 FROM (
    9     -- Number of skycells as a function of exposure
    10     SELECT
    11         exp_id,
    12         filter,
    13         COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo
    14     FROM rawExp
    15     JOIN chipRun USING(exp_id)
    16     JOIN camRun USING(chip_id)
    17     JOIN fakeRun USING(cam_id)
    18     JOIN warpRun USING(fake_id)
    19     JOIN warpSkyCellMap USING(warp_id)
    20     JOIN warpSkyfile USING(warp_id, skycell_id)
    21     JOIN diffInputSkyfile USING(warp_id,skycell_id)
    22     JOIN diffRun USING(diff_id)
    23     WHERE
    24         warpSkyfile.ignored = 0
    25         -- magicSkycellNums WHERE hook %s
    26     GROUP BY
    27         exp_id
    28     ) AS magicSkycellNums
    29 JOIN (
    30     -- Number of completed diffs for an exposure
    31     SELECT
    32         exp_id,
    33         COUNT(diff_id) AS num_done
    34     FROM magicBestDiffs
    35     GROUP BY
    36         exp_id
    37     ) AS magicDiffNums USING(exp_id)
     4    MAX(diff_id) AS diff_id
     5FROM diffRun
     6JOIN rawExp USING(exp_id)
    387LEFT JOIN magicRun USING(exp_id)
     8-- WHERE hook %s
     9GROUP BY exp_id

  • branches/bills_081204/ippTools/share/magictool_definebyquery_temp_insert.sql

    r20738 r20903  
    1414JOIN warpSkyfile USING(warp_id, skycell_id)
    1515JOIN diffInputSkyfile
    16     ON diffInputSkyfile.warp_id = warpSkyfile.warp_id
     16    ON diffInputSkyfile.warp1 = warpSkyfile.warp_id
    1717    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
    18     AND diffInputSkyfile.template = 0 -- selecting inputs only
    1918JOIN diffRun USING(diff_id)
    2019JOIN diffSkyfile USING(diff_id)

  • branches/bills_081204/ippTools/share/magictool_inputs.sql

    r20695 r20903  
    1111    diffSkyfile.fault
    1212FROM magicInputSkyfile
     13JOIN magicRun USING(magic_id)
    1314JOIN diffSkyfile
    1415    USING(diff_id)
    15 JOIN magicRun
    16     USING(magic_id)
    1716UNION
    1817-- Merged skycells
    … …  
    2120    magicRun.state,
    2221    magicTree.node,
    23     0,
     22    0,   -- no diff_id
    2423    magicNodeResult.uri,
    2524    NULL, -- magicNodeResult doesn't have a path_base

  • branches/bills_081204/ippTools/share/pxadmin_create_tables.sql

    r20841 r20903  
    941941        dvodb VARCHAR(255),
    942942        registered DATETIME,
    943         skycell_id VARCHAR(64),
     943        exp_id  BIGINT,
    944944        tess_id VARCHAR(64),
    945945        PRIMARY KEY(diff_id),
    946946        KEY(diff_id),
    947947        KEY(state),
    948         KEY(skycell_id),
    949948        KEY(tess_id)
    950949) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    952951CREATE TABLE diffInputSkyfile (
    953952        diff_id BIGINT,
    954         template TINYINT,
    955         stack_id BIGINT,
    956         warp_id BIGINT,
     953        warp1 BIGINT,
     954        warp2 BIGINT,
     955        stack1 BIGINT,
     956        stack2 BIGINT,
    957957        skycell_id VARCHAR(64),
    958958        tess_id VARCHAR(64),
    959         kind VARCHAR(64),
    960         PRIMARY KEY(diff_id, template),
    961         KEY(stack_id),
    962         KEY(warp_id),
     959        PRIMARY KEY(diff_id),
     960        KEY(warp1),
     961        KEY(warp2),
     962        KEY(stack1),
     963        KEY(stack2),
    963964        KEY(skycell_id),
    964965        KEY(tess_id),
    965         KEY(kind),
    966966        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id),
    967         FOREIGN KEY (stack_id)  REFERENCES  stackSumSkyfile(stack_id),
    968         FOREIGN KEY (warp_id, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id)
     967        FOREIGN KEY (warp1, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
     968        FOREIGN KEY (warp2, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
     969        FOREIGN KEY (stack1)  REFERENCES  stackSumSkyfile(stack_id),
     970        FOREIGN KEY (stack2)  REFERENCES  stackSumSkyfile(stack_id)
    969971) ENGINE=innodb DEFAULT CHARSET=latin1;
    970972
    971973CREATE TABLE diffSkyfile (
    972974        diff_id BIGINT,
     975        skycell_id VARCHAR(64),
    973976        uri VARCHAR(255),
    974977        path_base VARCHAR(255),
    … …  
    996999        KEY(good_frac),
    9971000        KEY(fault),
     1001        KEY(skycell_id),
    9981002        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)
    9991003) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    10021006        magic_id BIGINT AUTO_INCREMENT,
    10031007        exp_id BIGINT,
     1008        diff_id BIGINT,
    10041009        state VARCHAR(64),
    10051010        workdir VARCHAR(255),
    … …  
    10151020        KEY(label),
    10161021        KEY(fault),
    1017         FOREIGN KEY (exp_id)  REFERENCES  rawExp(exp_id)
     1022        FOREIGN KEY (exp_id)  REFERENCES rawExp(exp_id),
     1023        FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)
    10181024) ENGINE=innodb DEFAULT CHARSET=latin1;
    10191025
    10201026CREATE TABLE magicInputSkyfile (
    10211027        magic_id BIGINT,
    1022         diff_id BIGINT,
    10231028        node VARCHAR(64),
    1024         PRIMARY KEY(magic_id, diff_id),
    1025         FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
    1026         FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)
     1029        PRIMARY KEY(magic_id),
     1030        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
    10271031) ENGINE=innodb DEFAULT CHARSET=latin1;
    10281032

  • branches/bills_081204/ippTools/src/difftool.c

    r20719 r20903  
    4747
    4848static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
     49static bool diffRunComplete(pxConfig *config);
    4950
    5051# define MODECASE(caseName, func) \
    … …  
    108109    // required options
    109110    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    110     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    111111    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
    112112    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    113113    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
     114    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
    114115
    115116    // default
    … …  
    125126            NULL,       // dvodb
    126127            registered,
    127             skycell_id,
     128            exp_id,
    128129            tess_id
    129130    );
    … …  
    178179
    179180    // optional
    180     PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
    181     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
    182     PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false);
    183 
    184     // defaults to false
    185     PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
    186 
    187     // must provide either stack_id or warp_id but not BOTH
    188     if (!(stack_id || warp_id)) {
    189         psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
    190         return false;
    191     }
    192     if (stack_id && warp_id) {
    193         psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
    194         return false;
    195     }
    196 
    197     // if a warp_id was provided we need to lookup the skycell_id and tess_id
    198     // from the warpRun
    199     psString skycell_id = NULL;
     181    PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false);
     182    PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false);
     183    PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false);
     184    PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false);
     185    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
     186
     187    // must provide either stack1 or warp1 but not BOTH
     188    if (!(stack1 || warp1)) {
     189        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
     190        return false;
     191    }
     192    if (stack1 && warp1) {
     193        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
     194        return false;
     195    }
     196    // must provide either stack2 or warp2 but not BOTH
     197    if (!(stack2 || warp2)) {
     198        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
     199        return false;
     200    }
     201    if (stack2 && warp2) {
     202        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
     203        return false;
     204    }
     205
     206    // if a warp1 was provided we need to lookup the and tess_id from the diffRun
    200207    psString tess_id = NULL;
    201     if (warp_id) {
     208    if (warp1) {
    202209        if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) {
    203210            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    217224
    218225        diffRunRow *run = diffRunObjectFromMetadata(output->data[0]);
    219         skycell_id = run->skycell_id;
    220226        tess_id = run->tess_id;
    221227    }
    … …  
    228234    if (!diffInputSkyfileInsert(config->dbh,
    229235            diff_id,
    230             template,
    231             stack_id ? stack_id : PS_MAX_S64, // defined or NULL
    232             warp_id ? warp_id : PS_MAX_S64, // defined or NULL
     236            warp1 ? warp1 : PS_MAX_S64, // defined or NULL
     237            warp2 ? warp2 : PS_MAX_S64, // defined or NULL
     238            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
     239            stack2 ? stack2 : PS_MAX_S64, // defined or NULL
    233240            skycell_id,
    234             tess_id,
    235             kind
     241            tess_id
    236242        )) {
    237243        if (!psDBRollback(config->dbh)) {
    … …  
    300306    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
    301307    PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");
    302     PXOPT_COPY_STR(config->args, where,  "-kind", "kind", "==");
    303308
    304309    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    … …  
    349354    }
    350355
    351     if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) {
     356    if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) {
    352357        psError(PS_ERR_UNKNOWN, false, "database error");
    353358        psFree(whereClause);
    … …  
    471476
    472477    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
     478    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    473479    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
    474480    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
    … …  
    503509    if (!diffSkyfileInsert(config->dbh,
    504510                           diff_id,
     511                           skycell_id,
    505512                           uri,
    506513                           path_base,
    … …  
    533540    }
    534541
    535     if (!setdiffRunState(config, diff_id, "full")) {
     542    if (!diffRunComplete(config)) {
    536543        if (!psDBRollback(config->dbh)) {
    537544            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    546553        return false;
    547554    }
     555
    548556
    549557    return true;
    … …  
    762770                         psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
    763771                         psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
     772                         psS64 exp_id, // exposure id for input_warp_id (if defined)
    764773                         pxConfig *config // Configuration
    765774                         )
    … …  
    791800            NULL,       // dvodb
    792801            registered,
    793             skycell_id,
     802            exp_id,
    794803            tess_id
    795804    );
    … …  
    817826    if (!diffInputSkyfileInsert(config->dbh,
    818827            run->diff_id,
    819             true,
     828            input_warp_id,
     829            template_warp_id,
     830            input_stack_id,
    820831            template_stack_id,
    821             template_warp_id,
    822832            skycell_id,
    823             tess_id,
    824             NULL    // kind
    825         )) {
    826         if (!psDBRollback(config->dbh)) {
    827             psError(PS_ERR_UNKNOWN, false, "database error");
    828         }
    829         psError(PS_ERR_UNKNOWN, false, "database error");
    830         return false;
    831     }
    832 
    833     // Input
    834     if (!diffInputSkyfileInsert(config->dbh,
    835             run->diff_id,
    836             false,
    837             input_stack_id,
    838             input_warp_id,
    839             skycell_id,
    840             tess_id,
    841             NULL    // kind
     833            tess_id
    842834        )) {
    843835        if (!psDBRollback(config->dbh)) {
    … …  
    887879    PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
    888880    PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
     881    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
    889882    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    890883
    … …  
    906899        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
    907900                "No input has been defined (-input_stack_id or -input_warp_id)");
     901        return false;
     902    }
     903    if (input_warp_id && !exp_id) {
     904        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
     905                "-exp_id is required with -input_warp_id.");
    908906        return false;
    909907    }
    … …  
    916914                      template_warp_id ? template_warp_id : PS_MAX_S64,
    917915                      template_stack_id ? template_stack_id : PS_MAX_S64,
     916                      exp_id,
    918917                      config)) {
    919918        psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
    … …  
    945944    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
    946945    PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
    947     PXOPT_COPY_STR(config->args, warpWhere,  "-kind", "warpsToDiff.kind", "==");
    948946    PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
    949947    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
    … …  
    10481046            continue;
    10491047        }
     1048        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
     1049        if (!mdok) {
     1050            psWarning("exp_id not found --- ignoring row %ld", i);
     1051            continue;
     1052        }
    10501053
    10511054        if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id,
    1052                           PS_MAX_S64, PS_MAX_S64, stack_id, config)) {
     1055                          PS_MAX_S64, PS_MAX_S64, stack_id, exp_id, config)) {
    10531056            psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
    10541057                      warp_id, stack_id);
    … …  
    12741277}
    12751278
     1279static bool diffRunComplete(pxConfig *config)
     1280{
     1281    PS_ASSERT_PTR_NON_NULL(config, false);
     1282
     1283    // look for completed diffRuns
     1284    psString query = pxDataGet("difftool_completed_runs.sql");
     1285    if (!query) {
     1286        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     1287        return false;
     1288    }
     1289
     1290    if (!p_psDBRunQuery(config->dbh, query)) {
     1291        psError(PS_ERR_UNKNOWN, false, "database error");
     1292        psFree(query);
     1293        return false;
     1294    }
     1295    psFree(query);
     1296
     1297    psArray *output = p_psDBFetchResult(config->dbh);
     1298    if (!output) {
     1299        psError(PS_ERR_UNKNOWN, false, "database error");
     1300        return false;
     1301    }
     1302    if (!psArrayLength(output)) {
     1303        psTrace("difftool", PS_LOG_INFO, "no rows found");
     1304        psFree(output);
     1305        return true;
     1306    }
     1307    for (long i = 0; i < psArrayLength(output); i++) {
     1308        psMetadata *row = output->data[i];
     1309
     1310        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
     1311
     1312        // set diffRun.state to 'stop'
     1313        if (!setdiffRunState(config, diff_id, "full")) {
     1314            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
     1315                diff_id);
     1316            psFree(output);
     1317            return false;
     1318        }
     1319    }
     1320
     1321    return true;
     1322}
     1323

  • branches/bills_081204/ippTools/src/difftoolConfig.c

    r20693 r20903  
    4848    psMetadata *definerunArgs = psMetadataAlloc();
    4949    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
    50     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id",  0,            "define skycell ID (required)", NULL);
    5150    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID (required)", NULL);
    5251    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label",  0,            "define label", NULL);
    … …  
    6564    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "define stack ID", 0);
    6665    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "define warp ID", 0);
    67     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0,            "define kind", NULL);
    6866    psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template",  0,            "this sky cell file is the subtrahend", false);
    6967
    … …  
    7472    psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,            "search by skycell ID", NULL);
    7573    psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", NULL);
    76     psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0,            "search by kind", NULL);
    7774    psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template",  0,            "find only subtrahend", false);
    7875    psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-input", 0, "find only minuend", false);
    … …  
    9087    psMetadata *adddiffskyfileArgs = psMetadataAlloc();
    9188    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "define warp ID (required)", 0);
     89    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,       "define skycell of file (required)", 0);
    9290    psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
    9391    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
    … …  
    155153    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
    156154    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
    157     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL);
    158155    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
    159156    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);

  • branches/bills_081204/ippTools/src/magictool.c

    r20783 r20903  
    128128    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    129129
    130     // Create temporary table of the best diffs
    131     {
    132         psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
    133         if (!query) {
    134             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    135             return false;
    136         }
    137 
    138         if (!p_psDBRunQuery(config->dbh, query)) {
    139             psError(PS_ERR_UNKNOWN, false, "database error");
    140             return false;
    141         }
    142         psFree(query);
    143     }
    144 
    145     // Insert list of best diffs into temporary table
    146     {
    147         psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
    148         if (!query) {
    149             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    150             return false;
    151         }
    152 
    153         psMetadata *where = psMetadataAlloc();
    154         PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
    155         PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
    156         PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
    157 
    158         psString whereClause = NULL;    // WHERE conditions
    159         if (psListLength(where->list)) {
    160             whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    161             psStringPrepend(&whereClause, "\n AND ");
    162         }
    163         psFree(where);
    164 
    165         if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
    166             psError(PS_ERR_UNKNOWN, false, "database error");
    167             psFree(whereClause);
    168             psFree(query);
    169             return false;
    170         }
    171         psFree(whereClause);
    172         psFree(query);
    173     }
     130    psMetadata *where = psMetadataAlloc();
     131    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     132    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
     133
    174134
    175135    // Get list of exposures ready to magic
    … …  
    181141        }
    182142
    183         psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
    184         {
    185             psMetadata *where = psMetadataAlloc();
    186             PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
    187             PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
    188             PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
    189 
    190             if (psListLength(where->list)) {
    191                 magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
    192                 psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
    193             }
    194             psFree(where);
    195         }
    196 
    197         // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
    198         // told to only take what's available.
    199         // "new" means we want a new run even if there's already a magic run defined
     143        // "available" means queue magic run even though the diffRun has skycells that did not complete
     144        // "rerun" means we want a new run even if there's already a magic run defined for the exposure
    200145        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
    201         PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
     146        PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
    202147
    203148        psString queryWhere = NULL;     // WHERE conditions for entire query
    204         if (available) {
    205             psStringAppend(&queryWhere, " WHERE num_done = num_todo");
    206         }
    207         if (new) {
     149        if (!available) {
     150            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
     151        } else {
     152            // what if no skycells for the diff run completed?
     153        }
     154        if (!rerun) {
    208155            const char *newWhere = " magic_id IS NULL"; // String to add
    209156            if (queryWhere) {
    210157                psStringAppend(&queryWhere, " AND %s", newWhere);
    211158            } else {
    212                 psStringAppend(&queryWhere, " WHERE %s", newWhere);
     159                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
    213160            }
    214161        }
    215         if (queryWhere) {
    216             psStringAppend(&query, " %s", queryWhere);
     162        if (!queryWhere) {
     163            psStringAppend(&queryWhere, " ");
     164        }
     165
     166        if (!p_psDBRunQuery(config->dbh, query, queryWhere)) {
     167            psError(PS_ERR_UNKNOWN, false, "database error");
    217168            psFree(queryWhere);
    218         }
    219 
    220 
    221         if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
    222             psError(PS_ERR_UNKNOWN, false, "database error");
    223             psFree(magicSkyCellNumsWhere);
    224169            psFree(query);
    225170            return false;
    226171        }
    227         psFree(magicSkyCellNumsWhere);
     172        psFree(queryWhere);
    228173        psFree(query);
    229174    }
    … …  
    262207        psMetadata *row = output->data[i]; // Row of interest
    263208        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
     209        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // Exposure identifier
    264210
    265211        // create a new magicRun for this group
    266         magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
     212        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "run", workdir, "dirty", label, dvodb, registered, 0);
    267213        if (!run) {
    268214            psAbort("failed to alloc magicRun object");
    … …  
    297243        {
    298244            psString idString = NULL;
    299             psStringAppend(&idString, "%" PRId64, exp_id);
    300             psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
     245            psStringAppend(&idString, "%" PRId64, diff_id);
     246            psStringSubstitute(&thisInsert, idString, "@DIFF_ID@");
    301247            psFree(idString);
    302248        }
    … …  
    340286    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    341287    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
     288    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
    342289
    343290    // optional
    … …  
    350297            0,          // ID
    351298            exp_id,
     299            diff_id,
    352300            "reg",      // state
    353301            workdir,
    … …  
    414362            config->dbh,
    415363            magic_id,
    416             diff_id,
    417364            node
    418365    );

  • branches/bills_081204/ippTools/src/magictoolConfig.c

    r20744 r20903  
    5555    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
    5656    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
    57     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
    5857    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
    59     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false);
     58    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
    6059    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
    6160
Note: See TracChangeset for help on using the changeset viewer.