Index: /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25792) +++ /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25793) @@ -128,5 +128,6 @@ } -sub lookup { +sub lookup +{ my $ipprc = shift; my $image_db = shift; @@ -296,5 +297,5 @@ # if uri is nil this will get overridded below - # (this is for raw stage) + # (we do this here for raw stage) $out->{image} = $image->{uri}; if ($set_class_id) { @@ -302,8 +303,23 @@ $out->{component} = $class_id; } + my $stage_id; + if ($img_type eq "raw") { + $stage_id = $image->{exp_id}; + } elsif ($img_type eq "chip") { + $stage_id = $image->{chip}; + } elsif ($img_type eq "warp") { + $stage_id = $image->{warp_id}; + } elsif ($img_type eq "diff") { + $stage_id = $image->{diff_id}; + } elsif ($img_type eq "stack") { + $stage_id = $image->{stack_id}; + } + $image->{stage_id} = $stage_id; + $image->{stage} = $img_type; + $image->{image_db} = $image_db; # find the mask and weight images if ($base) { - $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; + $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; $out->{mask} = $ipprc->filename($mask_name, $base, $class_id) if $mask_name; $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name; Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 25792) +++ /trunk/dbconfig/changes.txt (revision 25793) @@ -1300,2 +1300,13 @@ ALTER TABLE pstampJob ADD COLUMN options BIGINT; +ALTER TABLE pstampJob ADD COLUMN dep_id BIGINT; +CREATE TABLE pstampDependent ( + dep_id BIGINT AUTO_INCREMENT, + stage VARCHAR(64), + stage_id BIGINT, + need_magic TINYINT, + image_db VARCHAR(64), + rlabel VARCHAR(64), + PRIMARY KEY(dep_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + Index: /trunk/dbconfig/pstamp.md =================================================================== --- /trunk/dbconfig/pstamp.md (revision 25792) +++ /trunk/dbconfig/pstamp.md (revision 25793) @@ -40,3 +40,14 @@ outputBase STR 255 options S64 64 + dep_id S64 0 END + +pstampDependent METADATA + dep_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 + stage STR 64 + stage_id S64 0 + imagedb STR 64 + rlabel STR 64 + no_magic BOOL f +END Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 25792) +++ /trunk/ippTools/share/Makefile.am (revision 25793) @@ -201,4 +201,5 @@ pstamptool_datastore.sql \ pstamptool_listjob.sql \ + pstamptool_pendingdependent.sql \ pstamptool_pendingjob.sql \ pstamptool_pendingreq.sql \ Index: /trunk/ippTools/share/pstamptool_pendingdependent.sql =================================================================== --- /trunk/ippTools/share/pstamptool_pendingdependent.sql (revision 25793) +++ /trunk/ippTools/share/pstamptool_pendingdependent.sql (revision 25793) @@ -0,0 +1,3 @@ +SELECT * +FROM pstampDependent +WHERE state = 'new' Index: /trunk/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /trunk/ippTools/share/pstamptool_pendingjob.sql (revision 25792) +++ /trunk/ippTools/share/pstamptool_pendingjob.sql (revision 25793) @@ -1,7 +1,9 @@ SELECT pstampJob.* FROM pstampJob - JOIN pstampRequest using(req_id) + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) WHERE pstampRequest.state = 'run' AND pstampRequest.fault = 0 AND pstampJob.state = 'run' AND pstampJob.fault = 0 + AND (dep_id = 0 OR pstampDependent.state = 'full') Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 25792) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 25793) @@ -1250,7 +1250,19 @@ outputBase VARCHAR(255), options BIGINT, + dep_id BIGINT, PRIMARY KEY(job_id, req_id), KEY(job_id), FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDependent ( + dep_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + stage VARCHAR(64), + stage_id BIGINT, + image_db VARCHAR(64), + rlabel VARCHAR(64), + need_magic TINYINT, + PRIMARY KEY(dep_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /trunk/ippTools/src/pstamptool.c =================================================================== --- /trunk/ippTools/src/pstamptool.c (revision 25792) +++ /trunk/ippTools/src/pstamptool.c (revision 25793) @@ -48,4 +48,7 @@ static bool projectMode(pxConfig *config); static bool modprojectMode(pxConfig *config); +static bool getdependentMode(pxConfig *config); +static bool pendingdependentMode(pxConfig *config); +static bool updatedependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -85,4 +88,7 @@ MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); + MODECASE(PSTAMPTOOL_MODE_GETDEPENDENT, getdependentMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); default: psAbort("invalid option (this should not happen)"); @@ -545,4 +551,5 @@ PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); // unless the job is being inserted with stop state require outputBase @@ -570,5 +577,6 @@ exp_id, outputBase, - options + options, + dep_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -896,2 +904,150 @@ } +static bool getdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (psArrayLength(output)) { + psMetadata *dep = output->data[0]; + psS64 dep_id = psMetadataLookupS64(NULL, dep, "dep_id"); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + printf("%" PRId64 "\n", dep_id); + return true; + } + if (no_create) { + return true; + } + // no existing dependent that matches, insert one + + if (!pstampDependentInsert( + config->dbh, + 0, // dep_id + "new", // state + stage, + stage_id, + imagedb, + rlabel, + no_magic + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); + return false; + } + + psS64 dep_id = psDBLastInsertID(config->dbh); + + printf("%" PRId64 "\n", dep_id); + + return true; +} + +static bool pendingdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampDependent"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDependent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatedependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + char *query ="UPDATE pstampDependent" + " SET state = '%s'" + " WHERE dep_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, dep_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} Index: /trunk/ippTools/src/pstamptool.h =================================================================== --- /trunk/ippTools/src/pstamptool.h (revision 25792) +++ /trunk/ippTools/src/pstamptool.h (revision 25793) @@ -43,4 +43,7 @@ PSTAMPTOOL_MODE_MODPROJECT, PSTAMPTOOL_MODE_PROJECT, + PSTAMPTOOL_MODE_GETDEPENDENT, + PSTAMPTOOL_MODE_PENDINGDEPENDENT, + PSTAMPTOOL_MODE_UPDATEDEPENDENT, } pstamptoolMode; Index: /trunk/ippTools/src/pstamptoolConfig.c =================================================================== --- /trunk/ippTools/src/pstamptoolConfig.c (revision 25792) +++ /trunk/ippTools/src/pstamptoolConfig.c (revision 25793) @@ -112,4 +112,5 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -143,4 +144,29 @@ psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -getdependent + psMetadata *getdependentArgs = psMetadataAlloc(); + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + + // -updatedependent + psMetadata *updatedependentArgs = psMetadataAlloc(); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + + // -pendingdependent + psMetadata *pendingdependentArgs = psMetadataAlloc(); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addproject @@ -189,4 +215,8 @@ PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + PXOPT_ADD_MODE("-getdependent", "", PSTAMPTOOL_MODE_GETDEPENDENT, getdependentArgs); + PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); + PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); PXOPT_ADD_MODE("-modproject", "", PSTAMPTOOL_MODE_MODPROJECT, modprojectArgs); Index: /trunk/pstamp/scripts/pstampparse.pl =================================================================== --- /trunk/pstamp/scripts/pstampparse.pl (revision 25792) +++ /trunk/pstamp/scripts/pstampparse.pl (revision 25793) @@ -72,4 +72,8 @@ } +# just deal with these arguments once and for all +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $dbserver" if $dbserver; + # list_job is a deugging mode $no_update = 1 if $mode eq "list_job"; @@ -105,6 +109,4 @@ if ($req_id and !$no_update) { my $command = "$pstamptool -listreq -name $req_name -not_req_id $req_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; # no verbose so that error message about request not found doesn't appear in parse_error.txt my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -127,6 +129,4 @@ my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -398,14 +398,26 @@ my $newState = "run"; my $fault = 0; - - if (($stage ne 'stack') and ($stage ne 'raw')) { - if (($image->{state} eq 'goto_purged') or ($image->{data_state} eq 'purged')) { + my $dep_id; + + if ($stage ne 'raw') { + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back $newState = 'stop'; $fault = $PSTAMP_GONE; - } elsif (($image->{data_state} ne 'full') or ($image->{state} ne 'full' )) { - # XXX here is where we need to queue an update job - # for now just say that the image is not available - $newState = 'stop'; - $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { + my $PSTAMP_NO_UPDATE = 2048; + # for now just fault while I build pstampDependent + if ($option_mask & $PSTAMP_NO_UPDATE) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, + $run_state, $stage, $image->{stage_id}, $need_magic, $image->{label}); + } } } @@ -416,6 +428,5 @@ $command .= " -fault $fault" if $fault; $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; + $command .= " -dep_id $dep_id" if $dep_id; if ($mode eq "list_job") { @@ -594,6 +605,4 @@ my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" . " -rownum $rownum -state stop -fault $fault"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if ($mode eq "list_job") { @@ -718,4 +727,36 @@ } +sub queue_update_run +{ + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic, $label) = @_; + + if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { + my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR); + } + + my $dep_id; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb"; + $command .= " -rlabel $label" if $label; + $command .= " -no_magic" if !$need_magic; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); + } + my $output = join "", @$stdout_buf; + chomp $output; + $dep_id = $output; + my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; + } else { + print STDERR "skipping $command\n"; + $dep_id = 42; + } + + $$r_dep_id = $dep_id; + $r_fault = 0; + $r_jobState = 'blocked'; +} + sub my_die {