#!/usr/bin/perl

=head1 NAME
 
validate - Validate the file is readable by pSConfig agents
 
=head1 SYNOPSIS
 
validate [options] FILE
 
 Options:
   -h | --help              help message
   -B | --bind ADDR         local address to bind to when sending http/https requests
   -q | --quiet             suppress output to stdout and stderr
   --pscheduler-server ADDR address of pscheduler server to use for validation.
   --deep                   perform 'deep' pscheduler validation 
   --skip-expand            skip expanding any include directives
   --skip-pscheduler        skip validating against pScheduler
   --skip-refs              skip validating references to objects
   --timeout N              number of seconds to wait to retrieve JSON

=head1 ARGUMENTS

=over 8

=item B<FILE>
 
A path to a local file on the filesystem or an http(s) URL pointing at the JSON 
to validate.  

=back

=head1 OPTIONS
 
=over 8
 
=item B<-h | --help>
 
Prints a help message and exits

=item B<-B | --bind ADDR>
 
Local address to bind to when sending http/https requests

=item B<-q | --quiet>
 
Suppresses output to stdout and stderr. Useful when scripting and only concerned
about exit code.

=item B<--pscheduler-server ADDR>
 
Address of pScheduler server to contact to validate test and archiver specs in 
use by tasks. If not specified, the script will check if pScheduler is running
on 127.0.0.1. If it is not, then it will silently skip the pScheduler validation.

=item B<--deep>
 
Perform 'deep' validation where every task has its template variables expanded 
and is validated against pscheduler. This is much more  thorough but can take a 
significant amount of time for large templates. By default, if a pscheduler 
server can be detected only one instance of each task is validated (this is 
called `Quick` mode). While this will still catch many common issues, this can 
miss address and host-specific validation errors in archives and contexts since 
not every potential task is tested. 

=item B<--skip-expand>
 
Skip expanding include directives and just validate schema prior to processing includes.

=item B<--skip-pscheduler>
 
Skip validating test, archives and contexts against pscheduler.

=item B<--skip-refs>
 
Skip validating that fields referencing other objects map to something that exists

=item B<--timeout N>
 
The integer number of seconds to wait to retrieve JSON. Default is 0, which is no timeout. 
The timeout is applied to each individual request separately. For example, if you have two 
includes and a 30 second timeout, your program may spend up to 90 seconds retrieving 
files: 30s for the original file and 30s for both includes

=back
 
=head1 DESCRIPTION
 
B<This program> will read a JSON file from local filesystem or an http(s) URL 
and validate the JSON. An exit code of 0 indicates valid, 1 indicates it is 
invalid, and 2 indicates an error occurred.

Specifically the program validates the following:

=over 4
 
=item * The JSON can be retrieved and is valid JSON

=item * The JSON validates against the pSConfig JSON schema

=item * Include files can be expanded (if applicable)

=item * Fields that reference other JSON objects only reference items that exist

=item * It asks a pScheduler server if the test, archiver and context specifications are valid.

=back

The first two bullets always happen but the remaining items can be skipped with 
command-line options. In addition, validation against a pScheduler server only
occurs if there is a local pscheduler server running (default) or the 
--pscheduler-server command-line option is given. pScheduler validation can either
run the the default `quick` mode that only expands template variables for the
first instance of task (e.g. the first source/destination pair). You can also
run 'deep' mode using the --deep option that expands every possible task but can
take a siginificant amount of time for large templates. The advantage is you get
a more complete validation of all potential generated tasks and associated objects.

=cut
use strict;
use warnings;

use FindBin qw($Bin);
use lib "$Bin/../../lib";
use Getopt::Long qw(GetOptions);
use Pod::Usage;
use Term::ANSIColor qw(:constants);
use Term::ProgressBar;
use URI;

use perfSONAR_PS::Client::PSConfig::ApiConnect;
use perfSONAR_PS::Client::PSConfig::ApiFilters;
use perfSONAR_PS::Client::PSConfig::Parsers::TaskGenerator;
use perfSONAR_PS::Client::PScheduler::ApiConnect;
use perfSONAR_PS::Client::Utils qw(send_http_request);

#parse options
my $bind;
my $quiet;
my $help;
my $deep;
my $skip_expand;
my $skip_pscheduler;
my $skip_refs;
my $pscheduler_server;
my $timeout = 0;
GetOptions(
    'help|h' => \$help,
    'bind|B=s' => \$bind,
    'quiet|q' => \$quiet,
    'pscheduler-server=s' => \$pscheduler_server,
    'deep' => \$deep,
    'skip-expand' => \$skip_expand,
    'skip-pscheduler' => \$skip_pscheduler,
    'skip-refs' => \$skip_refs,
    'timeout=i' => \$timeout
) or pod2usage({ -verbose => 0, -exitval => 2 });

#parse arguments
if($help){
    pod2usage({ -exitval => 0, -verbose => 2, -noperldoc => 1});
}elsif(@ARGV == 0){
    my $exitval = 2;
    pod2usage({
        -message => "Must specify a URL or file path to JSON file.",
        -exitval => $exitval,
        -verbose => 0   
    }) unless($quiet);
    exit $exitval;
}elsif(@ARGV > 1){
    my $exitval = 2;
    pod2usage({
        -message => "Only one file or URL can be specified on the command-line",
        -exitval => $exitval,
        -verbose => 0
    }) unless($quiet);
    exit $exitval;
}
#TODO: Validate URL and only add prefix if not a url but is existing file?
#unless otherwise specified assume a file
my $json_url = shift @ARGV;
chomp $json_url;

######################
#read file  
_stage_start("Loading template");
my $psconfig_filters = new perfSONAR_PS::Client::PSConfig::ApiFilters(timeout => $timeout);
my $psconfig_client = new perfSONAR_PS::Client::PSConfig::ApiConnect(url => $json_url, filters => $psconfig_filters);
if($bind){
    $psconfig_client->bind_address($bind);
}
my $psconfig = $psconfig_client->get_config();
if($psconfig_client->error()){
    my $err = "Error retrieving JSON. Encountered the following error:\n\n";
    $err .= "   " . $psconfig_client->error();
    chomp $err;
    _stage_fail($err);
    exit 2;
} 
_stage_success();

######################
#validate schema
_stage_start("Validating JSON schema");
my @errors = $psconfig->validate();
if(@errors){
    my $err = "pSConfig JSON is not valid. Encountered the following validation errors:\n";
    foreach my $error(@errors){
        $err .=  "\n   Node: " . $error->path . "\n";
        $err .=  "   Error: " . $error->message . "\n";
    }
    _stage_fail($err);
    exit 1;
}
_stage_success();

######################
#handle includes
if(!$skip_expand && $psconfig->includes() && @{$psconfig->includes()} > 0){
    #expand includes
    _stage_start("Expanding includes");
    $psconfig_client->expand_config($psconfig);
    if($psconfig_client->error()){
        my @errors = split "\n", $psconfig_client->error();
        my $err = "Error(s) encountered expanding includes:\n\n";
        foreach my $error(@errors){
            chomp $error;
            $err .= "   $error\n" if($error);
        }
        _stage_fail($err);
        exit 1;
    }
    _stage_success();
    
    #validate again after expansion
    _stage_start("Validating JSON schema (post include expansion)");
    @errors = $psconfig->validate();
    if(@errors){
        my $err = "pSConfig JSON is not valid after expanding includes. Encountered the following validation errors:\n\n";
        foreach my $error(@errors){
            $err .= "\n   Node: " . $error->path . "\n";
            $err .= "   Error: " . $error->message . "\n";
        }
        _stage_fail($err);
        exit 1;
    }
    _stage_success();
}

######################
#validate references
if(!$skip_refs){
    _stage_start("Verifying object references");
    my $ref_errors = [];
    #check addresses
    foreach my $addr_name(@{$psconfig->address_names()}){
        my $address = $psconfig->address($addr_name);
        my $host_ref = $address->host_ref();
        my $context_refs = $address->context_refs();
        #check host ref
        if($host_ref && !$psconfig->host($host_ref)){
            push @{$ref_errors}, "Address $addr_name references a host object $host_ref that does not exist.";
        }
        #check context refs
        if($context_refs){
            foreach my $context_ref(@{$context_refs}){
                if(!$psconfig->context($context_ref)){
                    push @{$ref_errors}, "Address $addr_name references a context object $context_ref that does not exist.";
                }
            }
        }
        #check remote addresses
        foreach my $remote_name(@{$address->remote_address_names()}){
            #check remote context refs
            my $remote = $address->remote_address($remote_name);
            if($remote->context_refs()){
                foreach my $context_ref(@{$remote->context_refs()}){
                    if(!$psconfig->context($context_ref)){
                        push @{$ref_errors}, "Address $addr_name has a remote definition for $remote_name using a context object $context_ref that does not exist.";
                    }
                }
            }
            #check remote labels
            foreach my $label_name(@{$remote->label_names()}){
                my $label = $address->label($label_name);
                if($label->context_refs()){
                    foreach my $context_ref(@{$label->context_refs()}){
                        if(!$psconfig->context($context_ref)){
                            push @{$ref_errors}, "Address $addr_name has a label $label_name using a context object $context_ref that does not exist.";
                        }
                    }
                }
            }
        }
        #check labels
        foreach my $label_name(@{$address->label_names()}){
            my $label = $address->label($label_name);
            #check label context refs
            if($label->context_refs()){
                foreach my $context_ref(@{$label->context_refs()}){
                    if(!$psconfig->context($context_ref)){
                        push @{$ref_errors}, "Address $addr_name has a label $label_name using a context object $context_ref that does not exist.";
                    }
                }
            }
        }
    }
    #check groups
    foreach my $group_name(@{$psconfig->group_names()}){
        my $group = $psconfig->group($group_name);
        if($group->type() eq 'disjoint'){
            foreach my $a_addr_sel(@{$group->a_addresses()}){
                ref_check_addr_select($a_addr_sel, $group_name, $psconfig, $ref_errors);
            }
            foreach my $b_addr_sel(@{$group->b_addresses()}){
                ref_check_addr_select($b_addr_sel, $group_name, $psconfig, $ref_errors);
            }
        }elsif($group->can('addresses')){
            foreach my $addr_sel(@{$group->addresses()}){
                ref_check_addr_select($addr_sel, $group_name, $psconfig, $ref_errors);
            }
        }
    }
    #check hosts
    foreach my $host_name(@{$psconfig->host_names()}){
        my $host = $psconfig->host($host_name);
        if($host->archive_refs()){
            foreach my $archive_ref(@{$host->archive_refs()}){
                if($archive_ref && !$psconfig->archive($archive_ref)){
                    push @{$ref_errors}, "Host $host_name references an archive $archive_ref that does not exist.";
                }
            }
        }
    }
    #check tasks
    foreach my $task_name(@{$psconfig->task_names()}){
        my $task = $psconfig->task($task_name);
        my $group_ref = $task->group_ref();
        my $test_ref = $task->test_ref();
        my $schedule_ref = $task->schedule_ref();
        
        #check group ref
        if($group_ref && !$psconfig->group($group_ref)){
            push @{$ref_errors}, "Task $task_name references a group $group_ref that does not exist.";
        }
        #check test ref
        if($test_ref && !$psconfig->test($test_ref)){
            push @{$ref_errors}, "Task $task_name references a test $test_ref that does not exist.";
        }
        #check schedule ref
        if($schedule_ref && !$psconfig->schedule($schedule_ref)){
            push @{$ref_errors}, "Task $task_name references a schedule $schedule_ref that does not exist.";
        }
        #check archive refs
        if($task->archive_refs()){
            foreach my $archive_ref(@{$task->archive_refs()}){
                if($archive_ref && !$psconfig->archive($archive_ref)){
                    push @{$ref_errors}, "Task $task_name references an archive $archive_ref that does not exist.";
                }
            }
        }
    }
    #print any errors
    if(@{$ref_errors}){
        my $ref_err_str = join "\n", @{$ref_errors};
        _stage_fail($ref_err_str);
        exit 1;
    }
    _stage_success();
}

######################
#validate specs against pscheduler
my $pscheduler;
if(!$skip_pscheduler && !$pscheduler_server){
    #autodetect pscheduler
    $pscheduler = new perfSONAR_PS::Client::PScheduler::ApiConnect(
        url => _build_pscheduler_url("127.0.0.1")
    );
    #test if server works at all
    $pscheduler->get_hostname();
    if($pscheduler->error()){
        #client didn't work so skip
        $pscheduler = "";
    }
}elsif(!$skip_pscheduler){
    $pscheduler = new perfSONAR_PS::Client::PScheduler::ApiConnect(
        url => _build_pscheduler_url($pscheduler_server)
    );
}
if($pscheduler){
    my $task_max = @{$psconfig->task_names()};
    my $psched_progress = _stage_start("pScheduler Validation (" . ($deep ? 'Deep' : 'Quick') . ")", $task_max);
    my $task_count = 0;
    foreach my $task_name(@{$psconfig->task_names()}){
        my $tg = new perfSONAR_PS::Client::PSConfig::Parsers::TaskGenerator(
                psconfig => $psconfig,
                task_name => $task_name,
                use_psconfig_archives => 1
            );
        
        my $task = $psconfig->task($task_name);
        my $test_ref = $task->test_ref();
        if($tg->start()){
            while($tg->next()){
                #test spec validation
                my $validation = $pscheduler->get_test_spec_is_valid(
                    $tg->expanded_test()->{type},
                    $tg->expanded_test()->{spec}
                );
                if($pscheduler->error()){
                    my $err = "Problem communicating with pscheduler while validating test spec $test_ref when used in task $task_name: \n\n";
                    $err .= "    " . $pscheduler->error();
                    _stage_fail($err, $psched_progress, $task_max);
                    exit 1;
                }elsif(!$validation->{'valid'}){
                    my $err = "Test spec $test_ref is invalid when used in task $task_name: \n\n";
                    $err .= "    " . ($validation->{'error'} ? $validation->{'error'} : 'No error specified');
                    $err .= "";
                    _stage_fail($err, $psched_progress, $task_max);
                    exit 1;
                }
                #archiver validation
                if($tg->expanded_archives()){
                    foreach my $expanded_archive(@{$tg->expanded_archives()}){
                        #can't be sure of exact reference since may come from host or defaults
                        my $archiver_type = $expanded_archive->{archiver};
                        my $validation = $pscheduler->get_archiver_is_valid(
                            $archiver_type,
                            $expanded_archive->{data}
                        );
                        if($pscheduler->error()){
                            my $err = "Problem communicating with pscheduler while validating archiver spec of type $archiver_type used in task $task_name: \n\n";
                            $err .= "    " . $pscheduler->error() . "\n";
                            _stage_fail($err, $psched_progress, $task_max);
                            exit 1;
                        }elsif(!$validation->{'valid'}){
                            my $err = "Archiver of type $archiver_type is invalid when used in task $task_name: \n\n";
                            $err .= "    " . ($validation->{'error'} ? $validation->{'error'} : 'No error specified');
                            _stage_fail($err, $psched_progress, $task_max);
                            exit 1;
                        }
                    }
                }
                #context validation
                if($tg->expanded_contexts()){
                    foreach my $expanded_context_list(@{$tg->expanded_contexts()}){
                        foreach my $expanded_context(@{$expanded_context_list}){
                            my $context_type = $expanded_context->{context};
                            my $validation = $pscheduler->get_context_is_valid(
                                $context_type,
                                $expanded_context->{data}
                            );
                            if($pscheduler->error()){
                                my $err = "Problem communicating with pscheduler while validating context spec of type $context_type used in task $task_name: \n\n";
                                $err .= "    " . $pscheduler->error() . "\n";
                                _stage_fail($err, $psched_progress, $task_max);
                                exit 1;
                            }elsif(!$validation->{'valid'}){
                                my $err = "Context of type $context_type is invalid when used in task $task_name: \n\n";
                                $err .= "    " . ($validation->{'error'} ? $validation->{'error'} : 'No error specified');
                                _stage_fail($err, $psched_progress, $task_max);
                                exit 1;
                            }
                        }
                    }
                }
                last unless($deep);
            }
            #break loop unless doing a deep validation
            $task_count++;
            $psched_progress->update($task_count) if($psched_progress);
        }else{
            _stage_fail("Problem building specs to send to pscheduler: " . $tg->error(), $psched_progress, $task_max);
            exit 1;
        }
    }
    _stage_success($psched_progress, $task_max);
}

_success();

#Subroutines
sub _success{
    print BOLD, GREEN "\npSConfig JSON is valid\n", RESET unless($quiet);
    exit 0;
}

sub _stage_start{
    my ($msg, $pb_max) = @_;
    
    my $progress;
    if($pb_max && !$quiet){
        $progress = Term::ProgressBar->new({
            name => "$msg",
            count => $pb_max, 
            remove => 1, 
            silent => !(-t STDIN), 
            ETA => 'linear'}
        );
    }elsif(!$quiet){
        print "$msg ...... ";
    }
    
    return $progress;
}

sub _stage_fail{
    my ($msg, $pb, $pb_max) = @_;
    if($pb){
        $pb->update($pb_max);
        _stage_start($pb->name);
    }
    print BOLD, RED, "FAIL\n", RESET unless($quiet);
    print STDERR RESET, "\n$msg\n\n" unless($quiet);
}

sub _stage_success{
    my ($pb, $pb_max) = @_;
    if($pb){
        $pb->update($pb_max);
        _stage_start($pb->name);
    }
    print BOLD, GREEN, "OK\n", RESET unless($quiet);
}

sub ref_check_addr_select{
    my ($addr_sel, $group_name, $psconfig, $errors) = @_;
    
    if($addr_sel->can('name')){
        my $addr_name = $addr_sel->name();
        my $addr_obj = $psconfig->address($addr_name);
        unless($addr_obj){
            push @{$errors}, "Group $group_name references an address object $addr_name that does not exist.";
            return;
        }
        my $addr_label_name = $addr_sel->label();
        if($addr_label_name && !$addr_obj->label($addr_label_name)){
            push @{$errors}, "Group $group_name references a label $addr_label_name for address object $addr_name that does not exist.";
        }
    }elsif($addr_sel->can('class')){
        my $class_name = $addr_sel->class();
        unless($psconfig->address_class()){
            push @{$errors}, "Group $group_name references a class object $class_name that does not exist.";
            return;
        }
    }
}

sub _build_pscheduler_url {
    my ($hostport) = @_;
    
    my $uri = new URI();
    $uri->scheme('https');
    $uri->host_port($hostport);
    $uri->path('pscheduler');
    
    return $uri->as_string;
}
