Changeset 41597

Timestamp:

May 18, 2021, 2:20:37 PM (5 years ago)

Author:

eugene

Message:

merge changes from trunk: neb-host in ~ippps1; add protective quotes in nightly_science.pro; add updatenewexp and updatesummitexp modes to pztool; tools/croot points at newer ps2 datastore

Location:

tags/ipp-ps1-20210510

Files:

• . (modified) (1 prop)
• Nebulous-Server/bin/neb-host (modified) (1 diff)
• ippTasks/nightly_science.pro (modified) (1 diff)
• ippTasks/summit.copy.pro (modified) (2 diffs)
• ippTools (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/pztool.c (modified) (4 diffs)
• ippTools/src/pztool.h (modified) (1 diff)
• ippTools/src/pztoolConfig.c (modified) (1 diff)
• ppTranslate/src/ppMops.h (modified) (1 diff)
• psastro (modified) (1 prop)
• psastro/src/psastroLoadCrosstalk.c (modified) (1 diff)
• psconfig (modified) (1 prop)
• psconfig/pscheckmods (modified) (3 diffs)
• psconfig/tagsets/ipp-3.0.perl (modified) (1 diff)
• pstamp/scripts/dquery_job_run.pl (modified) (2 diffs)
• pstamp/scripts/dqueryparse.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (1 diff)
• pstamp/scripts/pstamp_server_status (modified) (1 diff)
• tools (modified) (1 prop)
• tools/croot (modified) (1 diff)

tags/ipp-ps1-20210510/Nebulous-Server/bin/neb-host

@@ -25 +25 @@
 # $dbpass = $ENV{'NEB_PASS'} unless $dbpass;
 
-my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config";
+# my $conffile = "/data/ippc64.1/ippitc/ippconfig/nebulous.config";
+# my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config";
+my $conffile = "/home/panstarrs/ippps1/ippconfig/nebulous.config";
+
 if (-e $conffile) {
 #    printf STDERR "found config\n";

tags/ipp-ps1-20210510/ippTasks/nightly_science.pro

@@ -111 +111 @@
 
    if ($?camera_map:$2) 
-      if ($camera_map:$2 != "$3")
+      if ("$camera_map:$2" != "$3")
          echo "camera $3 does not match expected camera $camera_map:$2"
          break

tags/ipp-ps1-20210510/ippTasks/summit.copy.pro

@@ -24 +24 @@
 
 # list of DataStores to pull data from
-book init pzDataStore
+# book init pzDataStore
 # list of summit exps that need to be queried
 book init pzPendingExp
@@ -155 +155 @@
     task.exit 0
         # flush pzDataStore book
-#        book init pzDataStore
+        book init pzDataStore
         # convert 'stdout' to book format
         ipptool2book stdout pzDataStore -key camera:telescope -uniq -setword dbname $options:0

tags/ipp-ps1-20210510/ippTools/src/pztool.c

@@ -49 +49 @@
 static bool updatepzexpMode(pxConfig *config);
 static bool updatenewexpMode(pxConfig *config);
+static bool updatesummitExpMode(pxConfig *config);
 
 // static bool copydoneCompleteExp(pxConfig *config);
@@ -54 +55 @@
 static psArray *pzArrayZip(psArray *arraySet, psS64 limit);
 static bool pzDownloadExpSetState(pxConfig *config, const psS64 summit_id, const char *state);
+static bool summitExpSetFault(pxConfig *config, const psS64 summit_id, const int fault);
 
 # define MODECASE(caseName, func) \
@@ -88 +90 @@
         MODECASE(PZTOOL_MODE_UPDATEPZEXP, updatepzexpMode);
         MODECASE(PZTOOL_MODE_UPDATENEWEXP, updatenewexpMode);
+        MODECASE(PZTOOL_MODE_UPDATESUMMITEXP, updatesummitExpMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -933 +936 @@
 }
 
+static bool summitExpSetFault(pxConfig *config, const psS64 summit_id, const int fault)
+{
+    // check that state is a valid string value
+  if ((fault < 0) || (fault > 2048)) {
+    psError(PS_ERR_UNKNOWN, false,
+            "invalid summitExp fault: %d", fault);
+    return false;
+  }
+
+    char *query = "UPDATE summitExp SET fault = %d WHERE summit_id = %ld";
+    if (!p_psDBRunQueryF(config->dbh, query, fault, summit_id)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to change state for %lld", (long long) summit_id);
+        return false;
+    }
+
+    return true;
+}
+
+static bool updatesummitExpMode(pxConfig *config)
+{
+  int fault;
+
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  PXOPT_LOOKUP_S64(summit_id, config->args, "-summit_id", true, false);
+  PXOPT_LOOKUP_S32(state,    config->args, "-set_fault",true, false);
+
+  if (!summitExpSetFault(config,summit_id, fault)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to change state for %lld", (long long) summit_id);
+    return false;
+  }
+  return true;
+}
+  
+  
 static bool updatepzexpMode(pxConfig *config)
 {

tags/ipp-ps1-20210510/ippTools/src/pztool.h

@@ -39 +39 @@
     PZTOOL_MODE_UPDATEPZEXP,
     PZTOOL_MODE_UPDATENEWEXP,
+    PZTOOL_MODE_UPDATESUMMITEXP,
 } pztoolMode;
 

tags/ipp-ps1-20210510/ippTools/src/pztoolConfig.c

@@ -171 +171 @@
     psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-set_state",  0,            "define new state (required)", NULL);
     
+    // -updatesummitexp
+    psMetadata *updatesummitexpArgs = psMetadataAlloc();
+    psMetadataAddS64(updatesummitexpArgs, PS_LIST_TAIL, "-summit_id", 0,     "define summit_id", 0);
+    psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-exp_name",   0,            "search by exposure name (required)", NULL);
+    psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-inst",       0,            "search by camera (required)", NULL);
+    psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope (required)", NULL);
+    psMetadataAddS32(updatesummitexpArgs, PS_LIST_TAIL, "-set_fault",  0,            "define new fault (required)", 0);
+    
     // -updatenewexp
     psMetadata *updatenewexpArgs = psMetadataAlloc();

tags/ipp-ps1-20210510/ppTranslate/src/ppMops.h

@@ -12 +12 @@
 //#define SOURCE_MASK2 (PM_SOURCE_MODE2_DIFF_WITH_DOUBLE | PM_SOURCE_MODE2_ON_SPIKE | PM_SOURCE_MODE2_ON_STARCORE | PM_SOURCE_MODE2_ON_BURNTOOL | PM_SOURCE_MODE2_ON_CONVPOOR) // Flags2 to exclude
 #define SOURCE_MASK2 (PM_SOURCE_MODE2_DEFAULT) //Flags2 to exclude
+//#define SOURCE_MASK2 (PM_SOURCE_MODE2_SATSTAR_PROFILE) //Flags2 to exclude
 
 #define PS1_DV_FORMAT "PS1_DV%d"

tags/ipp-ps1-20210510/psastro/src/psastroLoadCrosstalk.c

@@ -409 +409 @@
               if (!(X == crossChip)) {
                 psTrace ("psastro.crosstalk",2,"Chip (%d%d) not a known crosstalk source.",X,Y); 
-                psLogMsg ("psastro", PS_LOG_INFO, "Chip (%d%d) not a known crosstalk source.",X,Y);
                 continue;
               }

tags/ipp-ps1-20210510/psconfig/pscheckmods

@@ -7 +7 @@
     shift @ARGV;
 }
-if (@ARGV > 2 && $ARGV[0] eq "-debug") {
-    $DEBUG = 1;
-    shift @ARGV;
-}
-if (@ARGV != 2) { die "USAGE: pscheckmods [-v] [-debug] (module) (version)\n"; }
+if (@ARGV != 2) { die "USAGE: pscheckmods (module) (version)\n"; }
 
 if ($VERBOSE) { print STDERR "checking in @INC\n"; }
@@ -28 +24 @@
 print "$ARGV[0]: $version\n";
 
-if ($ARGV[1] > $version) {
+my $requireExact = 0;
+my $myVersion = $ARGV[1];
+if ($myVersion =~ m|^=.|) {
+    $requireExact = 1;
+    $myVersion =~ s|^=||;
+    print "$ARGV[1], $myVersion\n";
+}
+
+if ($myVersion > $version) {
     print "$ARGV[0] is too old: have $version : need $ARGV[1]\n";
     exit 1;
 }
+
+if ($requireExact && ($myVersion != $version)) {
+    print "$ARGV[0] is not the right version: have $version : need $myVersion\n";
+    exit 1;
+}
+
 
 exit 0;
@@ -49 +59 @@
           print "real: $realfilename\n" if $DEBUG;
           if (-f $realfilename) {
-              print "my filename: $filename\n" if $DEBUG;
               $INC{$filename} = $realfilename;
               print "calling 'do' on $realfilename\n" if $DEBUG;

tags/ipp-ps1-20210510/psconfig/tagsets/ipp-3.0.perl

@@ -68 +68 @@
   55    IPC::Run                       IPC-Run-0.80.tar.gz                      0              NONE      NONE
   56    Cache                          Cache-2.04.tar.gz                        0              NONE      NONE
-  57    IPC::Cmd                       IPC-Cmd-0.36.tar.gz                      0.36           NONE      NONE
+  57    IPC::Cmd                       IPC-Cmd-0.36.tar.gz                      =0.36           NONE      NONE
   58    SOAP::Lite                     SOAP-Lite-0.69.v1.tar.gz                 0              NONE      yes,yes,no
   59    Log::Log4perl                  Log-Log4perl-1.10.v1.tar.gz              0              NONE      NONE

tags/ipp-ps1-20210510/pstamp/scripts/dquery_job_run.pl

@@ -121 +121 @@
 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run(command => $command, verbose => $verbose);
+## MEH need to adjust error_code to rc
 unless ($success) {
     my $rc = $error_code >> 8;
@@ -388 +389 @@
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
-    carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE");
+    #carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE");
+    carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE $exit_code");
+
     exit($exit_code);
 

tags/ipp-ps1-20210510/pstamp/scripts/dqueryparse.pl

@@ -385 +385 @@
                     ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 
                     ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 
-                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'error_cleaned') or
+                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'error_cleaned') or
                     ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 
                     ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
@@ -391 +391 @@
                     # image is gone and it's not coming back
                     $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
-                    ## MEH error_cleaned isn't necessarily gone though
-                    ## if BAD_COMPONENT not set then doesn't get added to faulted_rows list (others need this too?)
-                    $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 1;
+                    ## MEH why was this missing? should be added to other FAULT cases...
+                    ## messes with row order, skipping the faulted rows on output, unclear if problem since happens in other cases too 
+                    $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 1;
                 }
                 elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {

tags/ipp-ps1-20210510/pstamp/scripts/pstamp_job_run.pl

@@ -611 +611 @@
                     print STDERR "file $f is not available\n";
                     $return_code = 0;
-                    ## MEH doesn't look like return_code is even used from check_files...
-                    ## if file is not available, why is my_die not called with PSTAMP_NOT_AVAILABLE?
-                    ## is this only the case when neb-host down and fails to wait for fix?
-                    ## with multiple data nodes down, this needs to be a fault case
-                    my_die( "file $f is not available:", $job_id, $error_code, 'stop');
+                    ## MEH 20210127 doesn't look like return_code is even used from check_files...
+                    ## if file is not available, why is my_die not called with PSTAMP_NOT_AVAILABLE?
+                    ## is this only the case when neb-host down and fails to wait for fix?
+                    ## with multiple data nodes down, this needs to be a fault case
+                    my_die( "file $f is not available:", $job_id, $error_code, 'stop'); 
                 }
             } else {

tags/ipp-ps1-20210510/pstamp/scripts/pstamp_server_status

@@ -124 +124 @@
         if ($controller_state) {
             print $br . "Pantasks Controller $controller_state.\n";
-            # loading active labels
-            my $runlabels = `echo \"show.labels; quit\" \| pantasks_client -c ~ippitc/pstamp/ptolemy.rc \| tail -n +7`;
-            print $br."current labels: \n".$runlabels."\n";
         } else {
             print STDERR "Controller state not found";

tags/ipp-ps1-20210510/tools/croot

@@ -28 +28 @@
 } elsif ($camera eq 'gpc2')  {
     $out = `dsrootls --uri http://dsmaster.ps2/ds/index.txt`;
+## old tunnel 201809xx MEH    
+#    $out = `dsrootls --uri http://conductor.ps2.ifa.hawaii.edu:8080/ds/index.txt`;
 } else {
     die "don't know data store address for camera $camera\n";