#!/bin/env perl
###
### pstampparse.pl
###
###     Run the postage stamp request parser for a given request id
###

use warnings;
use strict;

use Sys::Hostname;
use Getopt::Long qw( GetOptions );
use PS::IPP::PStamp::RequestFile qw( :standard );
use PS::IPP::PStamp::Job qw( :standard );
use File::Temp qw(tempfile);

my $verbose;
my $dbname;
my $dbserver;
my $req_id;
my $request_file_name;
my $mode = "list_uri";
my $out_dir;
my $product;
my $save_temps;
my $no_update;

GetOptions(
    'file=s'    =>  \$request_file_name,
    'req_id=s'  =>  \$req_id,
    'out_dir=s' =>  \$out_dir,
    'product=s' =>  \$product,
    'mode=s'    =>  \$mode,
    'dbname=s'  =>  \$dbname,
    'dbserver=s'=>  \$dbserver,
    'verbose'   =>  \$verbose,
    'save-temps'=>  \$save_temps,
    'no-update' =>  \$no_update,
);

die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
die "--file is required"     if !defined($request_file_name);

if ($mode ne "list_uri") {
    die "req_id is required"   if !$req_id;
    die "out_dir is required"  if !$out_dir;
    die "product is required"  if !$product;
}

use IPC::Cmd 0.36 qw( can_run run );

use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::Stats;
use PS::IPP::Metadata::List qw( parse_md_list );

use PS::IPP::Config qw( :standard );
my $ipprc = PS::IPP::Config->new(); # IPP Configuration

my $missing_tools;

my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
my $dvoImagesAtCoords  = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);

if ($missing_tools) {
    warn("Can't find required tools.");
    exit ($PS_EXIT_CONFIG_ERROR);
}

$no_update = 1 if $mode eq "list_job";

my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

#
# get the data from the extension header
#
my $fields_output;
{
    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);
    # fields doesn't return zero when it succeeds
    #unless ($success) {
    #    print STDERR @$stderr_buf;
    #}
    $fields_output = join "", @$stdout_buf;
}
my (undef, $extname, $extver, $req_name) = split " ", $fields_output;

# make sure the file contains what we are expecting

die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
die "REQ_NAME not found in $request_file_name"  if (!$req_name);
die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");

if ($req_id and !$no_update) {
    my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
    $command .= " -outProduct $product";
    $command .= " -dbname $dbname" if $dbname;
    $command .= " -dbserver $dbserver" if $dbserver;
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);
    unless ($success) {
        die "$command failed";
    }
}

#
# now convert the request table to an array of metadata config docs
#

my $rows;
{
    my $command = "$pstampdump $request_file_name";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);
    unless ($success) {
        print STDERR @$stderr_buf;
    }
    my $table =  $mdcParser->parse(join "", @$stdout_buf) or
        die("Unable to parse metdata config doc");

    $rows = parse_md_list($table);
}

my @rowList;
my $num_jobs = 0;
my $imageList;
my $stage;
my $need_magic;
foreach my $row (@$rows) {
    # XXX: TODO: sanity check all parameters

    # XXX: TODO: We shouldn't really just die in this loop.
    # If we encounter an error for a particular row
    # add a job with the proper fault code. If there is a db or config error we should probably just
    # trash the request.

    my $rownum   = $row->{ROWNUM};
    my $job_type = $row->{JOB_TYPE};
    
    # parameters that select the images of interest
    my $project  = $row->{PROJECT};
    my $req_type = $row->{REQ_TYPE};
    $stage = $row->{IMG_TYPE};
    my $id       = $row->{ID};
    my $component = $row->{COMPONENT};

    my $filter   = $row->{REQFILT};
    my $mjd_min = $row->{MJD_MIN};
    my $mjd_max = $row->{MJD_MAX};

    my $option_mask= $row->{OPTION_MASK};

    die "valid region of interest is required to make postage stamps"
        if (($job_type eq "stamp") and ! validROI($row));

    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
    
    die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");


    # note: resolve_project avoids running pstamptool every time by remembering the
    # last project resolved
    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    die "project $project not found\n" unless $proj_hash;

    my $image_db   = $proj_hash->{dbname};
    my $camera     = $proj_hash->{camera};
    $need_magic = $proj_hash->{need_magic};

    # collect rows with the same images of interest in a list so that they
    # can be processed optimially
    if (@rowList) {
        my $firstRow = $rowList[0];
        # XXX: the collecting might work with !$skycenter but I need to think about it
        if ($skycenter and same_images_of_interest($row, $firstRow)) {
            push @rowList, $row;
            # On to the next row
            next;
        } else {
            # queue the jobs for this set of rows
            $num_jobs += queueJobs($mode, \@rowList, $imageList);
            @rowList = ();
        }
    }

    if ($req_type eq "bycoord") {
        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
        die "lookup bycoord is not yet implemented";
    } else {

        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
        # request specification. An array reference is returned
        my ($x, $y);
        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
    }

    if (!$imageList or !@$imageList) {
        print STDERR "no matching images found for row $rownum\n";
        next;
    }
    $row->{need_magic} = $need_magic;
    push @rowList, $row;
}

if (@rowList) {
    $num_jobs += queueJobs($mode, \@rowList, $imageList);
}

if ($mode eq "queue_jobs") {
    if ($num_jobs) {
        exit 0;
    } else {
        # no jobs queued return error to get the request stopped
        # TODO: need to define meaningful error codes
        exit $PS_EXIT_UNKNOWN_ERROR;
    }
} else {
    exit 0;
}


sub queueJobsForRow
{
    my $row = shift;
    my $stage = shift;
    my $imageList = shift;
    my $have_skycells = shift;
    my $need_magic = shift;

    # loop over images
    my $job_num = 0;
    foreach my $image (@$imageList) {
        my $component;
        if ($have_skycells) {
            $component = $image->{skycell_id};
        } else {
            $component = $image->{class_id};
        }

        # skip this component if it is not in the list for this row
        next if !$row->{components}->{$component};

        my $rownum = $row->{ROWNUM};

        my $roi_string;

        # note values were insured to be numbers in validROI()
        my $x = $row->{CENTER_X};
        my $y = $row->{CENTER_Y};
        my $w = $row->{WIDTH};
        my $h = $row->{HEIGHT};
        my $coord_mask = $row->{COORD_MASK};
        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
                $roi_string = "-pixcenter $x $y";
            } else {
                $roi_string = "-skycenter $x $y";
            }
            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
                $roi_string .= " -pixrange $w $h";
            } else {
                $roi_string .= " -arcrange $w $h";
            }
        }

        my $imagefile = $image->{image};
        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
            # XXX: should we add a faulted job so the client can know what happened?
            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
            next;
        }
        my $exp_id = $image->{exp_id};
            
        my $args = $roi_string ? $roi_string : "";
        if ($stage eq "raw" or $stage eq "chip") {
            $args .= " -class_id $component" if $component;
        }

        $job_num++;

        # add astrometry file for raw and chip images if one is available
        if (($stage eq "chip") || ($stage eq "raw")) {
            $args .= " -astrom $image->{astrom}" if $image->{astrom};
        }

        $args .= " -file $imagefile";

        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
            $args .= " -mask $image->{mask}";
        }
        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
            $args .= " -variance $image->{weight}";
        }

        my $output_base = "$out_dir/${rownum}_${job_num}";
        my $argslist = "${output_base}.args";

        # copy the argument list to a file
        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
        print ARGSLIST "$args\n";
        close ARGSLIST or die "failed to close $argslist";

        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
        # and if not, queue an update job and set the job state appropriately.

        my $newState = "run";

        $num_jobs++;
        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
                        . " -outputBase $output_base -rownum $rownum -state $newState";
        $command .= " -exp_id $exp_id" if $exp_id;
        $command .= " -dbname $dbname" if $dbname;
        $command .= " -dbserver $dbserver" if $dbserver;

        if ($mode eq "list_job") { 
            # this is a debugging mode, just print the pstamptool that would have run
            # this is sort of like the mode -noupdate that some other tools support
            print "$command\n";
        } elsif (!$no_update) {
            # mode eq "queue_job"
            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                run(command => $command, verbose => $verbose);
            unless ($success) {
                print STDERR @$stderr_buf;
                # XXX TODO: now what? Should we mark the error state for the request?
                # should we keep going for other uris? If so how do we report that some
                # of the work that the request wanted isn't going to get done
                die "failed to queue job for request $req_id";
            }
        } else {
            print "skipping command: $command\n";
        }
    }
    return $num_jobs;
}

sub queueJobs
{
    my $mode = shift;
    my $rowList = shift;
    my $imageList = shift;

    my $firstRow = $rowList[0];
    my $stage    = $firstRow->{IMG_TYPE};
    my $job_type = $firstRow->{JOB_TYPE};
    my $need_magic = $firstRow->{need_magic};

    my $num_jobs = 0;

    if ($mode eq "list_uri") {
        foreach my $image (@$imageList) {
            print "$image->{image}\n";
        }
    } elsif ($job_type eq "get_image") {
        die "get_image jobs not implemented yet";
    } else {
        my $thisRun;

        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
        foreach my $row (@$rowList) {
            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
            $row->{components} = {};
        }
        close $pointsList;

        my $have_skycells;
        if (($stage eq "raw") or ($stage eq "chip")) {
            $have_skycells = 0;
        } else {
            $have_skycells = 1;
        }
        
        my $tess_dir_abs;
        my $last_tess_id = "";
        while ($thisRun = getOneRun($stage, $imageList)) {
            {
                my $command = "$dvoImagesAtCoords $pointsListName";
                if ($have_skycells) {
                    my $tess_id = $thisRun->[0]->{tess_id};
                    if ($tess_id ne $last_tess_id) {
                        $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
                        $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
                        $last_tess_id = $tess_id;
                    }
                    $command .= " -D CATDIR $tess_dir_abs";
                } else {
                    my $astrom = $thisRun->[0]->{astrom};
                    die "no astrometry file found" if !$astrom;
                    my $astrom_resolved = $ipprc->file_resolve($astrom);
                    $command .= " -astrom $astrom_resolved";
                }
                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                    run(command => $command, verbose => $verbose);
                unless ($success) {
                    print STDERR @$stderr_buf;
                    my $rc = $error_code >> 8;
                    die "dvoImagesAtCoords failed: $rc";
                }
                # now we have a list of row numbers and components
                # eventually we might want to multiple stamp requests for the same image
                # into the same ppstamp job but not yet. For now we will queue a new
                my @lines = split "\n", join "", @$stdout_buf;
                foreach my $line (@lines) {
                    # parse the line, ignoring the ra and dec
                    my ($rownum, undef, undef, $component) = split " ", $line;

                    # I guess since we need this function we should be useing a hash for rowList 
                    my $row = findRow($rownum, $rowList);
                    $row->{components}->{$component} = 1;
                }
            }
            
            foreach my $row (@$rowList) {
                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic);
            }
        }
    }
    return $num_jobs;
}

sub get_run_id
{
    my $stage = shift;
    my $image = shift;

    if ($stage eq "raw") {
        return $image->{exp_id};
    } elsif ($stage eq "chip") {
        return $image->{chip_id};
    } elsif ($stage eq "warp") {
        return $image->{warp_id};
    } elsif ($stage eq "stack") {
        return $image->{stack_id};
    } elsif ($stage eq "diff") {
        return $image->{diff_id};
    } else {
        die "unenexpected stage: $stage found";
    }
}

# extract components from the imageList that have the same run id and return the list
sub getOneRun {
    my $stage = shift;
    my $imageList = shift;

    # return if array is empty
    return undef if ! @$imageList;

    my $last_run_id = 0;
    my @runList;
    while ($imageList->[0]) {
        my $run_id = get_run_id($stage, $imageList->[0]);

        last if ($last_run_id and ($run_id ne $last_run_id));

        my $image = shift @$imageList;
        $image->{stage} = $stage;
        push @runList, $image;
        $last_run_id = $run_id;
    }
    return \@runList;
}

sub same_images_of_interest {
    my $r1 = shift;
    my $r2 = shift;

    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
    return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE});
    return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
    return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
    return 0 if ($r1->{ID} ne $r2->{ID});

    if (defined($r1->{COMPONENT})) {
        return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
    } elsif (defined($r2->{COMPONENT})) {
        return 0;
    }

    return 1;
}

sub validNumber
{
    my $val = shift;

    return $val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/;
}

sub validROI
{
    my $row = shift;
    return 0 if !validNumber($row->{CENTER_X});
    return 0 if !validNumber($row->{CENTER_Y});
    return 0 if !validNumber($row->{WIDTH});
    return 0 if !validNumber($row->{HEIGHT});

    return 1;
}

sub findRow
{
    my $rownum = shift;
    my $rowList = shift;

    foreach my $row (@$rowList) {
        return $row if $row->{ROWNUM} eq $rownum;
    }
    return undef;
}
