Changeset 32795

Timestamp:

Nov 23, 2011, 2:34:18 PM (15 years ago)

Author:

bills

Message:

updates to postage stamp server required to put distribution bundles created
for get_image requests into nebulous

Location:

tags/ipp-20111110

Files:

• dbconfig/pstamp.md (modified) (1 diff, 1 prop)
• ippTools/share/Makefile.am (modified) (1 diff, 1 prop)
• ippTools/share/pstamptool_listfile.sql (added)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff, 1 prop)
• ippTools/src/pstamptool.c (modified) (3 diffs, 1 prop)
• ippTools/src/pstamptool.h (modified) (1 diff, 1 prop)
• ippTools/src/pstamptoolConfig.c (modified) (2 diffs, 1 prop)
• pstamp (modified) (1 prop)
• pstamp/scripts/pstamp_cleanup.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (7 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (2 diffs)

tags/ipp-20111110/dbconfig/pstamp.md

@@ -67 +67 @@
     num         S64         0    # Primary Key AUTO_INCREMENT
 END
+
+pstampFile METADATA
+    file_id     S64         0   # Primary Key AUTO_INCREMENT
+    job_id      S64         0
+    path        STR         255
+end

tags/ipp-20111110/ippTools/share/Makefile.am

@@ -308 +308 @@
         pstamptool_getdependent.sql \
         pstamptool_listjob.sql \
+        pstamptool_listfile.sql \
         pstamptool_pendingcleanup.sql \
         pstamptool_pendingdependent.sql \

tags/ipp-20111110/ippTools/share/pxadmin_create_tables.sql

@@ -1479 +1479 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE pstampFile (
+    file_id BIGINT AUTO_INCREMENT,
+    job_id BIGINT NOT NULL,
+    path VARCHAR(255),
+    PRIMARY KEY(file_id),
+    KEY(job_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE pstampWebRequest (
         num BIGINT AUTO_INCREMENT,

tags/ipp-20111110/ippTools/src/pstamptool.c

@@ -56 +56 @@
 static bool revertdependentMode(pxConfig *config);
 static bool getwebrequestnumMode(pxConfig *config);
+static bool addfileMode(pxConfig *config);
+static bool listfileMode(pxConfig *config);
+static bool deletefileMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -103 +106 @@
         MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode);
         MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode);
+        MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode);
+        MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode);
+        MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -1476 +1482 @@
     return true;
 }
+
+static bool addfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+    PXOPT_LOOKUP_STR(path,   config->args, "-path",   true, false);
+
+    if (!pstampFileInsert(config->dbh,
+            0, // file_id
+            job_id,
+            path
+            )) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    if (affected != 1) {
+        psError(PS_ERR_UNKNOWN, false,
+            "should have affected one row but %" PRIu64 " rows were modified",
+            affected);
+        return false;
+    }
+
+    psS64 file_id = psDBLastInsertID(config->dbh);
+    printf("%" PRId64 "\n", file_id);
+
+    return true;
+}
+
+static bool listfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    if (!psListLength(where->list)) {
+        fprintf(stderr, "search arguments are required\n");
+        exit (1);
+    }
+
+    psString query = pxDataGet("pstamptool_listfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool deletefileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+
+    psString query = NULL; 
+    psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected);
+
+    return true;
+}

tags/ipp-20111110/ippTools/src/pstamptool.h

@@ -50 +50 @@
     PSTAMPTOOL_MODE_REVERTDEPENDENT,
     PSTAMPTOOL_MODE_GETWEBREQUESTNUM,
+    PSTAMPTOOL_MODE_ADDFILE,
+    PSTAMPTOOL_MODE_LISTFILE,
+    PSTAMPTOOL_MODE_DELETEFILE,
 } pstamptoolMode;
 

tags/ipp-20111110/ippTools/src/pstamptoolConfig.c

@@ -274 +274 @@
     psMetadata *modes = psMetadataAlloc();
 
+    // -addfile
+    psMetadata *addfileArgs = psMetadataAlloc();
+    psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id",       0, "define job ID for file (required)", 0);
+    psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path",         0, "define path for file (required)", NULL);
+
+    // -listfile
+    psMetadata *listfileArgs = psMetadataAlloc();
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id",     0, "select by file ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id",      0, "select by request ID", 0);
+    psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple",     0, "use the simple output format", false);
+
+    // -deletefile
+    psMetadata *deletefileArgs = psMetadataAlloc();
+    psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID (required)", 0);
+
     PXOPT_ADD_MODE("-addreq",          "", PSTAMPTOOL_MODE_ADDREQ,       addreqArgs);
     PXOPT_ADD_MODE("-pendingreq",      "", PSTAMPTOOL_MODE_PENDINGREQ,   pendingreqArgs);
@@ -302 +319 @@
     PXOPT_ADD_MODE("-project",         "", PSTAMPTOOL_MODE_PROJECT,    projectArgs);
     PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM,   getwebrequestnumArgs);
+    PXOPT_ADD_MODE("-addfile",         "", PSTAMPTOOL_MODE_ADDFILE,   addfileArgs);
+    PXOPT_ADD_MODE("-listfile",        "", PSTAMPTOOL_MODE_LISTFILE,   listfileArgs);
+    PXOPT_ADD_MODE("-deletefile",      "", PSTAMPTOOL_MODE_DELETEFILE,   deletefileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

tags/ipp-20111110/pstamp/scripts/pstamp_cleanup.pl

@@ -23 +23 @@
 
 use PS::IPP::Config qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 
 my $req_id;
@@ -95 +96 @@
 }
 
+{
+    my $command = "$pstamptool -listfile -req_id $req_id";
+    $command   .= " -dbname $dbname" if $dbname;
+    $command   .= " -dbserver $dbserver" if $dbserver;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $files = parse_md_fast($mdcParser, $output);
+        foreach my $file (@$files) {
+            $ipprc->file_delete($file->{path}); 
+        }
+    }
+}
+
 # now go find the workdir for this request 
 # XXX: we finally *have* to store this in the database

tags/ipp-20111110/pstamp/scripts/pstamp_get_image_job.pl

@@ -25 +25 @@
 
 my $output_base;
+my $bundleroot;
 
 my $verbose;
-my $ipprc;
+my $imagedbname;
 my $dbname;
 my $dbserver;
@@ -41 +42 @@
         'rownum=s'        =>      \$rownum,
         'output_base=s'   =>      \$output_base,
+        'bundleroot=s'    =>      \$bundleroot,
+        'imagedbname=s'   =>      \$imagedbname,
         'dbname=s'        =>      \$dbname,
         'dbserver=s'      =>      \$dbserver,
@@ -52 +55 @@
 
 my_die( $err, $PS_EXIT_PROG_ERROR) if $err;
+
+my $ipprc = PS::IPP::Config->new();
 
 my $params_file = $output_base . ".mdc";
@@ -91 +96 @@
 my $missing_tools;
 my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+my $pstamptool   = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -96 +102 @@
 }
 
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+
 my $outdir = dirname($output_base);
 my $basename = basename($path_base);
-my $outroot = $output_base ."_" . $basename;
 my $results_file = $output_base . ".bundle_results";
+my $outroot;
+if ($bundleroot) {
+    my (undef, undef, undef, $mday, $month, $year) = gmtime(time());
+    $month += 1;
+    $year += 1900;
+
+    # This will generate an outroot like:
+    # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025
+
+    $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename;
+} else {
+    $outroot = $output_base ."_" . $basename;
+}
 
 {
@@ -105 +126 @@
     $command .= " --results_file $results_file";
     $command .= " --component $component --path_base $path_base --outroot $outroot";
-#    XXX: we need to do some work if we want to support muggle bundles
-#    $command .= " --no_magic if $no_magic";
+    #    XXX: we need to do some work if we want to support muggle bundles
+    #    $command .= " --no_magic if $no_magic";
     $command .= " --magicked" if $magicked;
-    $command .= " --dbname $dbname" if $dbname;
+    $command .= " --dbname $imagedbname" if $imagedbname;
     $command .= " --verbose" if $verbose;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -147 +168 @@
 }
 
+if ($bundleroot) {
+    {
+        # delete any existing pstampFile in case this job has faulted and
+        # been reverted
+        my $command = "$pstamptool -deletefile -job_id $job_id";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("Unable to perform $command: $error_code", $error_code >> 8);
+        }
+    }
+    my $linkname = "$outdir/$file_name";
+    if (-l $linkname or -e $linkname) {
+        unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $bundle_name = dirname($outroot) . "/$file_name";
+    my $resolved = $ipprc->file_resolve($bundle_name);
+    if (!$resolved) {
+        my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir", 
+        $PS_EXIT_UNKNOWN_ERROR);
+
+    my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("Unable to perform $command: $error_code", $error_code >> 8);
+    }
+}
+
+
 print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
 

tags/ipp-20111110/pstamp/scripts/pstamp_job_run.pl

@@ -88 +88 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $params = read_params_file($outputBase);
+
 my $jobStatus;
 if ($jobType eq "stamp") {
-    my $params = read_params_file($outputBase);
 
     my $argString;
@@ -272 +273 @@
 } elsif ($jobType eq "get_image") {
 
-    my $uri = "";
+    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+    my $imagedb = $params->{imagedb};
+
     my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
+    $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root;
+    $command .= " --imagedb $imagedb" if $imagedb;
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;