Changeset 23345

Timestamp:

Mar 17, 2009, 11:08:44 AM (17 years ago)

Author:

eugene

Message:

add so_id_start and so_id_range to neb-admin pendingreplicate and replicate task

Location:

trunk

Files:

• Nebulous-Server/bin/neb-admin (modified) (5 diffs)
• ippTasks/replicate.pro (modified) (5 diffs)

trunk/Nebulous-Server/bin/neb-admin

@@ -23 +23 @@
     $dbpass,
     $dbuser,
+    $so_id_start,
+    $so_id_range,
     $limit,
     $pending,
@@ -41 +43 @@
     'pendingreplicate|r'    => \$pending,
     'pendingremoval'        => \$removal,
+    'so_id_start=i'         => \$so_id_start,
+    'so_id_range=i'         => \$so_id_range,
     'limit|l=i'             => \$limit,
     'verbose|v'             => \$verbose,
@@ -58 +62 @@
 
 # check to make sure that only one instance of neb-admin is running
+# XXX this implies we should move pending replicate elsewhere
 my $pidfile = '/var/tmp/neb-admin';
 # abort if an instance is already running
@@ -91 +96 @@
     $dbh->do("INSERT INTO myvolume SELECT * FROM volume");
 
+    if (not defined $so_id_start) { $so_id_start = 0; }
+    if (not defined $so_id_range) { $so_id_range = 100000; }
+    my $so_id_end = $so_id_start + $so_id_range;
+
+    # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10
+    { 
+        my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object");
+        $query->execute;
+        my $answer = $query->fetchrow_arrayref;
+        my $max_so_id = $$answer[0];
+        $query->finish;
+
+        if ($so_id_start > $max_so_id) { 
+            print STDERR "at end of so_id range, reset please\n";
+            exit 10;
+        }
+    }
+        
     my $query = $dbh->prepare(
     "        SELECT
@@ -111 +134 @@
                 USING(vol_id)
             WHERE mymountedvol.available = 1
-    --        WHERE storage_object_xattr.name = 'user.copies'
+            AND storage_object_xattr.name = 'user.copies'
+            AND storage_object.so_id >= $so_id_start
+            AND storage_object.so_id <  $so_id_end
             GROUP BY so_id
             HAVING available_instances < instances OR instances < copies

trunk/ippTasks/replicate.pro

@@ -62 +62 @@
 # the replicate process interacts with only the single Nebulous server
 
+# Each 'pendingreplicate' query is limited to a finite number of so_id values.  
+# Each time we call replicate.load, we increment SO_ID_START by the range value.  
+# If the pendingreplicate query exits with exit status 10, we start over at 0
+
+$SO_ID_START = 0
+$SO_ID_RANGE = 100000
+
 # select Nebulous objects which desire additional copies
 task           replicate.load
@@ -68 +75 @@
   # modify these after the tasks are tested
   periods      -poll 10
-  periods      -exec 1
+  periods      -exec 10
   periods      -timeout 1500
   npending     1
@@ -79 +86 @@
       # command does not need to be dynamic, but having it so allows us to adjust the periods
       # so that we dont have to wait 10 minutes for things to start up
-      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 7500
+      # XXX smaller limited?  7500 will be a huge book of things to do...
+      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 7500 --so_id_start $SO_ID_START --so_id_range $SO_ID_RANGE
       periods      -exec 1800
 
@@ -86 +94 @@
   # success
   task.exit $EXIT_SUCCESS
+    # advance the so_id counter
+    $SO_ID_START += $SO_ID_RANGE
+
     # convert 'stdout' to book format
     ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT
@@ -95 +106 @@
     # delete existing entries in the appropriate pantaskStates
     process_cleanup replicatePending
+  end
+
+  # out of so_id range, reset
+  task.exit 10
+    # advance the so_id counter
+    $SO_ID_START = 0
   end