Changeset 39581

Timestamp:

May 27, 2016, 10:21:16 AM (10 years ago)

Author:

bills

Message:

Added options to specify the age of filesets that should be preserved by cleanup.
The first is --retention-days. Filesets registered that many days ago or older will be
cleaned up.
A second option --use-configured-publish-retention is a flag tells the script to go find
the number of days in the siteConfig if it wasn't supplied in the arguments.
If it is not found in the siteConfig the script dies.

I also removed the hard coded value for data store url from the code and require
it to be found in the siteConfig.

File:

• trunk/tools/cleandsproduct.pl (modified) (6 diffs)

trunk/tools/cleandsproduct.pl

@@ -19 +19 @@
 use PS::IPP::Metadata::List qw( parse_md_list );
 
+use Date::Parse;
+
 use PS::IPP::Config 1.01 qw( :standard );
 
@@ -37 +39 @@
 my ($product, $datastore, $last_fileset, $keep_files, $verbose);
 
+# If supplied, retention_days is the number of days that filesets in a data store will be preserved.
+# The use_configured_publish_retention_days option tells us to get retention_days from the site config.
+# The later is only used if retention-days is not supplied in the options.
+# NOTE: if last_fileset is also given, clean up will stop after that fileset irrespective of whether
+# subsequent filesets are older than the retention time.
+
+my ($retention_days, $use_publish_retention_days);
+
 # Parse the command-line arguments
 GetOptions(
-           'product=s'      => \$product,      #product name
-           'datastore=s'    => \$datastore,     #datastore uri
+           'product=s'      => \$product,      # product name
+           'datastore=s'    => \$datastore,    # datastore uri
            'last_fileset=s' => \$last_fileset, # last fileset to delete
-           'no_rm'          => \$keep_files, # last fileset to delete
-           'verbose'        => \$verbose,    # Print stuff?
+           'last-fileset=s' => \$last_fileset, # more modern spelling for last option
+           'retention-days=s'   => \$retention_days, # see above
+           'use-configured-publish-retention' => \$use_publish_retention_days, #see above
+           'no_rm'          => \$keep_files,   # remove fileset but leave files in place
+           'no-rm'          => \$keep_files,   # more modern spelling for last option
+           'verbose'        => \$verbose,      # Print stuff?
            ) or pod2usage( 2 );
 
@@ -52 +66 @@
 
 
-
-# since this needs to run on the MHPCC cluster there is no need to go to the
-# proxy to get the datastore listing
-$datastore = "http://ippc17/ds" if !$datastore;
-# $datastore = "http://datastore.ipp.ifa.hawaii.edu" if !$datastore;
+if (!$datastore) {
+    $datastore = metadataLookupStr($ipprc->{_siteConfig}, "DATA_STORE_ROOT_URL");
+    die ("Unable to find DATA_STORE_ROOT_URL in siteConfig\n") unless $datastore;
+}
 
 my $uri = "$datastore/$product/index.txt";
 
+# XXX: I'm not sure why this bracket is here
 {
     my $command = "$dsproductls --uri $uri --extra --timeout 150";
@@ -77 +91 @@
     }
 
+    # if retention_days is specified, use that. Otherwise if requested use the publish config value from the siteConfig
+    if (!$retention_days and $use_publish_retention_days) {
+        $retention_days = metadataLookupS32($ipprc->{_siteConfig}, "DATA_STORE_PUBLISH_RETENTION_DAYS");
+        die ("required value DATA_STORE_PUBLISH_RETENTION_DAYS not found in siteConfig\n") unless defined $retention_days;
+    }
+
+    # if $retention_days is set we are to clean only filesets registered before
+    # that date/time. Convert the limit to seconds.
+    my $clean_before;
+    if ($retention_days) {
+        $clean_before = time() - $retention_days * 86400;
+    }
+
     foreach my $line (@lines) {
         chomp $line;
@@ -82 +109 @@
 
         my ($fsuri, $fs_name, $registered, $fs_type, $target_id, $stage, $stage_id) = split " ", $line;
+
+        if ($clean_before) {
+            # registered is in the format YYYY-MM-DDTHH:MM:SSZ. Date::Parse::str2time handles this.
+            my $ticks = str2time($registered);
+            if ($ticks > $clean_before) {
+                # This fileset and therefore subsequent ones in the list was registered after our end
+                # date. We're done.
+                last;
+            }
+        }
 
         $command = "$dsreg --del $fs_name --product $product";
@@ -96 +133 @@
         last if $last_fileset and ($fs_name eq $last_fileset);
     }
-
-
 }