Changeset 17552 for trunk/DataStore

Timestamp:

May 6, 2008, 5:05:03 PM (18 years ago)

Author:

jhoblitt

Message:

add support for checksumming compressed fits files

Location:

trunk/DataStore

Files:

• Build.PL (modified) (1 diff)
• Changes (modified) (1 diff)
• lib/DataStore/File.pm (modified) (7 diffs)
• scripts/dsget (modified) (2 diffs)

trunk/DataStore/Build.PL

@@ -12 +12 @@
         'Data::Validate::URI'   => '0.01',
         'Digest::MD5::File'     => '0.03',
+        'IPC::Cmd'              => '0.36',
         'File::Basename'        => 0,
         'File::Temp'            => '0.16',

trunk/DataStore/Changes

@@ -2 +2 @@
 
 0.07
+    - add support for checksumming compressed fits files
     - Change DateStore::File to handle passing args to LWP::UserAgent
     - add dsget --timeout option

trunk/DataStore/lib/DataStore/File.pm

@@ -1 +1 @@
 # Copyright (C) 2006  Joshua Hoblitt
 #
-# $Id: File.pm,v 1.17 2008-05-06 02:35:22 jhoblitt Exp $
+# $Id: File.pm,v 1.18 2008-05-07 03:05:03 jhoblitt Exp $
 
 package DataStore::File;
@@ -17 +17 @@
 use DataStore::Utils qw( $STD_FIELD $BYTE_FIELD $MD5_FIELD %KNOWN_FILE_TYPES );
 use Digest::MD5::File qw( file_md5_hex );
+use File::Basename qw( basename );
+use File::Temp ();
 use File::stat;
-use Params::Validate qw( validate SCALAR ARRAYREF );
+use IPC::Cmd 0.36 qw( can_run run );
+use Params::Validate qw( validate SCALAR ARRAYREF UNDEF );
 
 use vars qw( @BASE_FIELDS );
 
-@BASE_FIELDS = qw( fileid bytes md5sum type extra );
+@BASE_FIELDS = qw( fileid bytes md5sum type extra compressed );
 
 __PACKAGE__->mk_accessors(@BASE_FIELDS);
@@ -158 +161 @@
                 optional    => 1,
             },
+            compressed => {
+                type        => SCALAR | UNDEF,
+                optional    => 1,
+            },
         },
     );
@@ -228 +235 @@
         },
     );
+
+    my $verbose = 0;
 
     # make request
@@ -241 +250 @@
 
     if ($response->is_success) {
+        my $funpack;
+        if ($self->compressed) {
+            $funpack = can_run('funpack')
+                or warn "can't find funpack -- unable to compute checksum";
+        }
+
+        my $chk_file = $filename;
+        # so that the tmp file stays in scope until we're done processing
+        my $tmp;
+        if ($funpack) {
+            $tmp = File::Temp->new(
+                DIR         => '/tmp',
+                TEMPLATE    => basename($filename) . '.XXXXXXXX',
+                SUFFIX      => '.tmp',
+                UNLINK      => 1,
+            );
+
+            my $unpacked_path = $tmp->filename;
+
+            my $command = "$funpack -S -C $filename > $unpacked_path";
+            my ($success, $status, $full_buf, $stdout_buf, $stderr_buf)
+                = run(command => $command, verbose => $verbose);
+
+            unless ($success) {
+                die "funpack returned exit status $status\n";
+            }
+
+            $chk_file = $unpacked_path;
+        }
+
         # check size
         if (defined $self->bytes) {
-            my $size = stat($filename)->size;
+            my $size = stat($chk_file)->size;
             if (! ($self->bytes == $size)) {
                 unlink $filename;
@@ -251 +290 @@
                 # set the filename to undef to indicate an error
                 $filename = undef;
+            } else {
+                carp "uri: ", $self->uri,
+                     " - expected size: ", $self->bytes,
+                     " got: ", $size
+                if $verbose;
             }
         }
 
         if (defined $filename and defined $self->md5sum) {
-            my $md5 = file_md5_hex($filename);
+            my $md5 = file_md5_hex($chk_file);
             if (! ($self->md5sum eq $md5)) {
                 unlink $filename;
@@ -263 +307 @@
                 # set the filename to undef to indicate an error
                 $filename = undef;
+            } else {
+                carp "uri: ", $self->uri,
+                     " - expected md5: ", $self->md5sum,
+                     " got: ", $md5
+                if $verbose;
             }
         }

trunk/DataStore/scripts/dsget

@@ -3 +3 @@
 # Copyright (C) 2006-2008  Joshua Hoblitt
 #
-# $Id: dsget,v 1.22 2008-05-07 01:54:20 jhoblitt Exp $
+# $Id: dsget,v 1.23 2008-05-07 03:05:03 jhoblitt Exp $
 
 use strict;
@@ -76 +76 @@
 $p{bytes} = $bytes if defined $bytes; 
 $p{md5sum} = $md5 if defined $md5;
+$p{compressed} = 1 if defined $compress;
 
 my $tmp;