#=============================================================================
# Doloforge::Command::Report : doloforge reporting and log analysis command-line tool
#
# This module executes log analysers and creates activity reports for each
# client/project. It is usually run by the cron command, on the proper hosts.
# Running this as a standalone command is only helpful for testing analysers.
#
# Full help :
#   doloforge report --man
#
# Debug run :
#   sudo perl ./src/doloforge --config ./t/doloforge.yaml report --run --date '20/08/2008'
#   sudo perl ./src/doloforge --config ./t/doloforge.yaml --verbose --verbose report --run --date '16/09/2010' --client jim --project clicodrome
#=============================================================================


package Doloforge::Command::Report;

use strict;
use warnings;

# This base class implements the constructor and help methods in a generic way.
use base qw(Doloforge::Command);

use Date::Calc qw(:all);
use Cwd;
use File::Path;


#=============================================================================
=encoding utf8

=head1 NAME

Doloforge::Command::Report - doloforge reporting and log analysis command-line tool

=head1 SYNOPSIS

    # Run the reports for a specific project, at a specific date
    doloforge report --run --client internal --project default --date '20/08/2008'

    # Simplest run (as launched by cron run)
    doloforge report --run

=head1 DESCRIPTION

Todo

This module executes log analysers and creates activity reports for each
client/project. It is usually run by the cron command, on the proper hosts.
Running this as a standalone command is only helpful for testing analysers.

=cut


#=============================================================================
# Method getOptions
#
# Accepted arguments from the command-line, in the form accepted by Getopt::Long.
# Options with no value that match method names of that package will be called as
# methods, with the decoded options as parameters.

sub getOptions {
    return (
        # actions :
        'run',                  # run de reporting jobs configured
        # parameters :
        'client|c=s',           # the client we want to run reports for, or all if not given
        'project|p=s',          # the project we want to run reports for, or all if not given
                                # client and project params just define which of project.*.client.*.yaml files will be used, thus may lead to none or many if not used correctly...
        'date|d=s',             # override the current date for interval matching evaluation (formated as DD/MM/YYYY as in `date +'%d/%m/%Y'`)
        'dry-run',              # test the reports, without doing anything (don't create dirs, dont call analysers)
    );
}


#=============================================================================
# Method run
#
# Run the defined cron jobs, following these rules:
#   - the command is meant to be called everyday in the morning by cron, and in the /etc/cron.hourly system pool
#   - the specified date, if provided, overrides the current date
#   - a lock is taken on a directory shared by application servers, allowing only one server to run the cron jobs
#   - the hourly and daily intervals always execute the corresponding jobs
#   - all other intervals are checked with the date
#       + weekly jobs are only run when day of week is sunday (morning)
#       + monthly jobs are only run when this is the first day of the month (1st in the morning)
#   - running a jobs is done the following way :
#       + setup a proper environment, after reading the project configuration
#       + environment variables are cleanly defined
#       + redirect stdout/err to a temporary file, close stdin
#       + set working directory to the project's main directory
#       + run the script and mail the output to the project contact email

sub run {
    my $self = shift;

    my ($option) = @_;

    # Get the passed date, or the current one if not defined. Send an error if the date is not parsable.
    # @date == ($year,$month,$day)
    my @date = Today();
    if (defined($option->{date})) {
        (@date = Decode_Date_EU($option->{date}))
        or (@date = Parse_Date($option->{date}))
        or $self->error("Cannot parse date parameter '$option->{date}' (see Date::Calc Decode_Date_EU and Parse_Date)");
    }
    # Format the date as a proper partial ISO-8601 date (YYYY-MM-DD)
    my $date = sprintf('%04d-%02d-%02d', @date);

    # Get the wanted clients and/or projects, or undef is none were given
    my $client = (exists($option->{client}) ? $option->{client} : undef);
    my $project = (exists($option->{project}) ? $option->{project} : undef);
    $self->info("Starting reports for ".($client ? "client '$client'" : "all clients").", ".($project ? "project '$project'" : "all projects").", date $date");

    # From now on, the messages will be prefixed by a proper timestamp...
    $self->show_timestamps();

    # TODO: process the following logs too
    #   /srv/doloforge/shared/.doloforge/apache/_logs/error.log.20091125.gz
    #   /srv/doloforge/shared/.doloforge/mason/_logs/error.log.20091125.gz
    #   /srv/doloforge/shared/.doloforge/php5/_logs/error.log.20091125.gz
    #       -> in the _pre or _post runs
    #   /srv/doloforge/shared/tickettruck/lib/_logs/php5/error.log.20091125.gz
    #   /srv/doloforge/shared/travelonomy/lib/_logs/php5/error.log.20091125.gz
    #       -> foreach client, sub _run_client (stripped-down copy of _run_project)

    # TODO : run the config.run_default._pre reports here... unless $client or $project

    # List all active projects (beware: an active project can be owned by an inactive client, and will still be listed)
    my $clients = (&Doloforge::Command::Factory('Client', $self->{global}))->_list('active', $client);
    my $projects = (&Doloforge::Command::Factory('Project', $self->{global}))->_list('active', $client, $project);

    # The keys used for the sort is "client/project" so projects are in fact executed in client order first...
    # TODO : have a configurable sorting algo, to make sure tracker analysers will run before project analysers, for instance...
    foreach (sort(keys(%{$projects}))) {
        my $project = $projects->{$_};
        my $client = $clients->{$project->{client}};

        # If the client is inactive, or the project, just don't run anything
        next unless ($client->{active} and $project->{active});

        # We run report generation in a forked process, in order to be sure to :
        #   - run all code and file creation with the correct ID
        #   - merge the project config with the global config, knowing that the caller will not be impacted
        $self->run_sub(
            "$self->{name} calling analysers for $project->{client}/$project->{name}, run date $date",
            (getpwnam($client->{name}))[2,3], # convert the string uid of the client into it's numeric uid and gid
            sub {
                $self->_run_project($client, $project, $date, $option->{'dry-run'});
            }
        );
    }

    # TODO : run the config.run_default._post reports here... unless $client or $project
}


#=============================================================================
# Method _run_project
#
# Run the report scripts for the given project. This method is called for each
# project in turn. It is run with the credential of the client the project
# belongs to. This leaves this method free to mess with configugation hashes,
# adding project-specific config to the platform-wide config, knowing that
# everything will be forgotten at the end of the method.

sub _run_project {
    my $self = shift;

    # Client and project hashes, date in ISO format (YYYY-MM-DD)
    my ($client, $project, $date, $dry_run) = @_;
    my @date = ($date =~ m/^(\d{4})-(\d{2})-(\d{2})$/);
    #$dry_run = 1;

    $self->info("Running analysers for $client->{name}/$project->{name}, run date $date");

    # Change current directory to [client]/[project]/ as a specific base for all code that will run
    my $working_dir = "$self->{global}->{dir}->{shared}/$project->{client}/$project->{name}";
    $working_dir = cwd . "/$working_dir" unless ($working_dir =~ m/^\//);
    # If the project working directory was not created (the configuration file is there, but "config --update" was not run), then just don't run the reports, as there is no logs yet...
    unless (chdir($working_dir)) {
        $self->debug("Can't chdir to '$working_dir', aborting analysis: $!");
    } else {
        $ENV{PWD} = $working_dir;

        # Read project config file, and merge it with the main command config
        #   - if the YAML file exists, read it
        #   - all subkeys parts of authorized keys, taken precence over existing keys,
        #     ie. run:apache key from the project would overwrite the global run:apache if it existed
        #         run_default:mason key will overwrite the global run_default:mason (but not run_default:apache)
        #         definitions:acces_ip will be added to the definitions: global hash (because there is no global value for it)
        #         definitions:analog will overwrite the global definitions:analog
        #   - run_default: is not meant to be normally replaced by the project configuration, but rather suffixed by run: keys
        my $project_conffile = "$working_dir/conf/report.yaml";
        if (-f $project_conffile) {
            # Read the file in a new hash object
            $self->debug("Reading $project_conffile project configuration file");
            my $include = Config::YAML->new(config => $project_conffile);

            # The included configuration sub-keys take precedence over all previously defined configuration keys.
            foreach my $key (qw(run_default run definitions)) {
                next unless (ref($include->{$key}) eq 'HASH');
                map {$self->{config}->{$key}->{$_} = $include->{$key}->{$_}} keys(%{$include->{$key}});
            }
        }

        # For each service of the project...
        foreach my $service (sort(keys(%{$project->{services}}))) {

            # ...matching services which have a configuration entry regarding reports
            my (@reports, %reports);
            push(@reports, @{$self->{config}->{run_default}->{$service}}) if exists($self->{config}->{run_default}->{$service});
            push(@reports, @{$self->{config}->{run}->{$service}}) if exists($self->{config}->{run}->{$service});
            $self->debug("Service $service for $client->{name}/$project->{name} will run reports ".(join(', ', @reports) || '(none)'));

            # For each report configured for the service (see report.yaml or the project reports configuration)
            foreach my $report (@reports) {

                # Keep track of reports that we already ran, because running a report twice will not be useful (if configured in both run_default and run)
                $self->warn("Report $report for $client->{name}/$project->{name} will run twice or more, see logs below!") if (exists($reports{$report}));
                $reports{$report} = 1;

                # List all the existing log file that this analyser is interested into, ie. those that match date-logperiod up to date-1
                # Log files are of the form "[service]/_logs/access.log.YYYYMMDD" (uncompressed version).
                # String comparison for dates in this format is OK.
                my @logfiles = ();
                my ($start_date, $end_date) = ('', '');
                $end_date = sprintf('%4d%02d%02d', Add_Delta_Days(@date, -1));
                if (exists($self->{config}->{definitions}->{$report}->{logperiod})) {
                    my $logname = $self->{config}->{definitions}->{$report}->{logname} || 'access.log';
                    $start_date = sprintf('%4d%02d%02d', Add_Delta_YMD(@date, @{$self->{config}->{definitions}->{$report}->{logperiod}}));
                    $self->debug("Searching for log files in $working_dir/$service/_logs/$logname.????????.gz, dated between $start_date and $end_date");
                    foreach my $logfile (glob("$service/_logs/$logname.????????.gz")) {
                        my ($log_date) = ($logfile =~ m/\.(\d{8})\.gz$/);
                        push(@logfiles, $logfile) if (($start_date le $log_date) and ($log_date le $end_date));
                    }

                    # If there is no log files to read, AND the analyser wanted at least some (logperiod was defined), then we can just try the next report...
                    next unless ($#logfiles >= 0);
                    $self->debug("Found log files ".join(' ', @logfiles));
                }

                # For each interval at which the project should run...
                foreach my $interval (@{$self->{config}->{definitions}->{$report}->{interval}}) {
                    # ...matching the day we run on (cf. --date passed as a parameter)
                    next unless (($interval eq 'day')                                   # always run day reports during the daily morning run
                             or (($interval eq 'week') and (Day_of_Week(@date) == 7))   # run the week reports on sunday morning
                             or (($interval eq 'month') and ($date[2] == 1)));          # run the month reports on the 1st of the month

                    # We reach the meat of the framework :
                    #   - change current directory to [client]/[project]/ as a specific base for all code that will run
                    #   - create a directory for the report as [client]/[project]/[service]/_stats/[basedir]/history/[YYYY]/[MM]/[DD]_[interval]/
                    #   - create a handy symlink to it as [client]/[project]/[service]/_stats/[basedir]/[interval]
                    #   - add a directory matching the back-end Alias if needed
                    # All this does not occur if the "basedir" is set to an empty string, denoting a report with no filesystem output at all.
                    my $output_dir = '';
                    if (not exists($self->{config}->{definitions}->{$report}->{basedir}) or ($self->{config}->{definitions}->{$report}->{basedir} gt '')) {
                        my $base_dir = "$service/_stats/".($self->{config}->{definitions}->{$report}->{basedir} || $report);
                        my $report_dir = "history/".join('/', @date)."_$interval";
                        $output_dir = "$base_dir/$report_dir";

                        unless ($dry_run) {
                            # Create the report dir
                            mkpath("$working_dir/$base_dir/$report_dir");
                            # Remove the existing symlink if it exists (a symlink pointing to a non-existent file will return false with -e, true with -l)
                            if (-l "$working_dir/$base_dir/$interval") {
                                unlink("$working_dir/$base_dir/$interval") 
                                    or $self->warn("Can't remove '$working_dir/$base_dir/$interval': $!");
                            }
                            # Then create a new symlink
                            symlink($report_dir, "$working_dir/$base_dir/$interval")
                                or $self->warn("Can't create symbolic link '$working_dir/$base_dir/$interval' to '$report_dir': $!");
                            # In case the apache/_stats/$service directory does not exist, create it, in order to match the corresponding Apache Alias directive
                            if (($service ne 'apache') and not -d "$working_dir/apache/_stats/$service") {
                                mkpath("$working_dir/apache/_stats/$service")
                                    or $self->warn("Can't create '$working_dir/apache/_stats/$service': $!");
                            }
                        }
                    }

                    # Identify each report analyser to call (using either the report name, or the list of report analysers defined)
                    foreach my $analyser ((exists($self->{config}->{definitions}->{$report}->{analyser}) ? @{$self->{config}->{definitions}->{$report}->{analyser}} : $report)) {
                        # Finally, call the report analyser code for each existing configuration string...
                        foreach my $config ((exists($self->{config}->{definitions}->{$report}->{config}) ? @{$self->{config}->{definitions}->{$report}->{config}} : '')) {
                            # Call _report_$analyser with all args above...
                            my $sub = "_report_$analyser";
                            $self->info(($dry_run ? 'Not actually calling' : 'Calling')." report $report ($sub) for $client->{name}/$project->{name}/$service, input from $start_date to $end_date, interval $interval, config '$config', output to ".($output_dir || '(none)').", user ".getpwuid($>).", working directory $working_dir");
                            $self->$sub(
                                client => $client,                  # client configuration hash
                                project => $project,                # project configuration hash
                                service => $service,                # service name to run report on
                                report => $report,                  # report name from config.run.[service].[...]
                                logfiles => \@logfiles,             # list of the log files the analyser is configured to use
                                run_date => join('', @date),        # date of the run, aka --date parameter, ie. "today" (end_date is "yesterday")
                                start_date => $start_date,          # starting date for the log contents, computed from the --date parameter, and the logperiod value from report.yaml
                                end_date => $end_date,              # ending date, computed from --date (basically : yesterday) (YYYYMMDD format)
                                interval => $interval,              # interval string
                                config => $config,                  # config string
                                output => $output_dir,              # output directory, relative to the working directory, containing most of the above strings as necessary
                                working_dir => $working_dir,        # working directory, used only for non-ambiguous warning about files, for instance
                            ) unless $dry_run;
                        }
                    }
                }
            }
        }
    }

    $self->info("Finished analysers for $client->{name}/$project->{name}, run date $date");
}


#=============================================================================
# Methods _report_*
#
# Actual report-generating methods, for each report listed in report.yaml
# These methods are executed :
#   - once for each service, interval, config, etc. ie. potentially many times per project
#   - under the uid/gid of the client, thus files are generated using this uid, and someone
#     cannot manually run a report against a random project he does not have permissions on
# They receive the exact same set of arguments :
#   - current directory is the base directory for the client/project (same as cron jobs)
#   - client and project configuration hashes, for full control
#   - ISO date (YYYY-MM-DD) at which the report is generated (in the morning, or manually for a random date)
#   - current servive, interval, config string, output directory
#   - all the log file names they should read in order to generate their output, as selected by the caller (in compressed format)
# Quick stats input about log file size :
#   - total raw size of logs of some project : 11343.6 MB
#   - total compressed size (gzip -9) : 598.613 MB (5.2% of the raw size)
#   - raw file read over NFS (4.4GB) : 
#       - file read (empty cache) : real 9m32.255s / user 0m2.660s / sys 0m8.665s
#       - loads the NFS client (even when niced) at approx 80% of 1 CPU (iowait + softirq), load average 1
#         AND the NFS server  at 100% of 1 CPU (iowait), load average 8
#       - network transfert 60Mbps (1Gbps ethernet)
#       - local file read (cache probably not empty) : real 1m34.946s / user 0m2.792s / sys 0m10.905s
#       - local file uncompress comparable...
#   - compressed file read over NFS (235MB) :
#       - file read (empty cache) : real 0m59.197s / user 0m0.288s / sys 0m0.736s
#       - file uncompress : 2.92user 2.56system 0:29.17elapsed
#   - conclusion : there is far more CPU available to uncompress, than bandwidth to read raw files
#   - even reading + uncompressing is 6 times faster than reading only
#   - net gain on local FS is much less (and will load a single host, instead of client + server)
#     but log files in such a setup (devl, test) are also tiny, so it does not matter...
# BTW: I don't know exactly why _report_analog() doesn't use the exact same run_job() call as the others _report_*()

sub _report_analog {
    my $self = shift;
    my (%option) = @_;
    my @command = ();

    # This analyser uses analog in the following way :
    #   analog -G +g/etc/doloforge/report.analog.cfg +g_conf/report.analog.cfg -Oapache/_stats/analog/history/2008/09/28_day/index.html +F080828:0000 +T080927:2359 apache/_logs/access.log.20080828.gz ...

    my $elapsed = time;
    push(@command, 'analog', '-G'); # -G : do not use the default configuration file
    # Search for all existing configuration files, and add them in turn :
    #   - report-specific global config
    #   - config-string-specific global config
    #   - project config for analog
    #   - report-specific project config
    #   - config-string-specific project config for the report
    my @conffiles = grep {-f} (
        "$self->{global}->{dir}->{conf}/report.analog.cfg",
        "$self->{global}->{dir}->{conf}/report.analog-$option{report}.cfg",
        "$self->{global}->{dir}->{conf}/report.analog-$option{report}-$option{config}.cfg",
        "_conf/report.analog.cfg",
        "conf/report.analog-$option{report}.cfg",
        "conf/report.analog-$option{report}-$option{config}.cfg",
    );
    push(@command, map("+g$_", @conffiles));

    # Find the lastest OUTFILE directive inside all of the config files, and get its value, to add to the command-line
    my $outfile;
    if ($#conffiles >= 0) {
        $outfile = qx(awk '/^OUTFILE/ {outfile=\$2} END {print outfile}' @conffiles);
        chomp($outfile);
        push(@command, "-O$option{output}/$outfile");
    }

    # Add the starting and ending dates computed from the configuration, as it will allow better use of the --date parameter
    # Analog date format is yyMMdd (the format we have is YYYYMMDD)
    my ($start_date) = ($option{start_date} =~ m/(\d{6})$/);
    my ($end_date) = ($option{end_date} =~ m/(\d{6})$/);
    push(@command, "+F$start_date:0000", "+T$end_date:2359");

    # Add the names of all the log files identified by the caller
    push(@command, @{$option{logfiles}});

    $self->info("Running ".join(' ', @command));

    # ...then actually run the job...
    my $rc = system(@command);
    # ...return code is not of much use, as analog mostly return error text on STDOUT/ERR, with rc=0...

    # Test for return code et al, and report to the running developer
    if ($rc == -1) {
        $self->warn("Failed to execute $command[0]: $!");
    } elsif ($rc & 127) {
        $self->warn("Child $command[0] died with signal=".($rc & 127)." elapsed=".(time - $elapsed)." (".(($rc & 128) ? 'with' : 'without')." coredump)");
    } elsif ($rc >= 256) {
        $self->warn("Child $command[0] exited with rc=".($rc >> 8)." elapsed=".(time - $elapsed));
    } else {
        $self->info("Finished $command[0] rc=$rc elapsed=".(time - $elapsed));
    }

    # Compress the ouput file
    $self->run_job(
        $option{client}, $option{project},  # client and project configuration hashes
        'gzip',                             # name of the command to call
        # command-line arguments, given to the script, making it easy to get position arguments (last argument is in fact a variable list of input log file names) :
        ['-9', "$option{output}/$outfile"],
        # no environment variables...
        {}
    ) if ($outfile);

    return $rc;
}

sub _report_error {
    my $self = shift;
    my (%option) = @_;

    # Run a generic (or specific, if a project manager really wants to) error analyser, with the help of the generic external report calling method
    $option{config} ||= 'report_error';
    $self->_report_external(%option);
}

sub _report_external {
    my $self = shift;
    my (%option) = @_;

    # The analyser script can be a project-specified script, or a generic one if the former doesn't exist
    my @candidate_scripts = (
        "cron/analyser/$option{config}",
        "$self->{global}->{dir}->{shared}/$option{config}",
        "$self->{global}->{dir}->{bin}/$option{config}",
        #"/home/nhuillard/Developpement/plain/doloforge/$self->{global}->{dir}->{bin}/$option{config}", # This one is for development only !!!
    );
    my $script = (grep {-x and -f} @candidate_scripts)[0];

    if ($script) {
        # Analyser scripts are executed the same way cron jobs are executed : same environment, same working dir, etc.
        # They also receive special arguments in the environment, related to the report to be generated :
        $self->info("Running $script $option{service} $option{report} $option{start_date} $option{end_date} $option{interval} $option{output} ".join(' ',@{$option{logfiles}}));
        $self->run_job(
            $option{client}, $option{project},  # client and project configuration hashes
            $script,                            # name of the script to call, based on the "config" parameter for the report (may be set multiple times, leading to multiple calls)
            # command-line arguments, given to the script, making it easy to get position arguments (last argument is in fact a variable list of input log file names) :
            [$option{service}, $option{report}, $option{start_date}, $option{end_date}, $option{interval}, $option{output}, @{$option{logfiles}}],
            # environment variables, added to the classic project_env.yaml env vars, giving the script a chance to use named arguments instead of position arguments :
            {
                REP_SERVICE => $option{service},
                REP_REPORT => $option{report},
                REP_DATE_RUN => $option{run_date},
                REP_DATE_START => $option{start_date},
                REP_DATE_END => $option{end_date},
                REP_INTERVAL => $option{interval},
                REP_OUTPUT_DIR => $option{output},
                REP_LOG_FILES => join(' ', @{$option{logfiles}}),
            }
        );
    } else {
        $self->warn("Can't find a suitable report job out of (".join(' ',@candidate_scripts).") to run for $option{config} $option{service} $option{report} $option{start_date} $option{end_date} $option{interval} $option{output} ".join(' ',@{$option{logfiles}}));
    }
}

sub _report_performance {
    my $self = shift;
    my (%option) = @_;

    $self->info("Running _report_performance $option{service} $option{report} $option{start_date} $option{end_date} $option{interval} $option{output} ".join(' ',@{$option{logfiles}}));
    # report info (+0s): Running _report_performance mason performance 20100915 20100915 day mason/_stats/.performance/history/2010/09/16_day mason/_logs/access.log.20100915.gz

    my %stat;

    foreach my $logfile (@{$option{logfiles}}) {
        my @cmd = ('zcat', $logfile);
        open(LOG, '-|', @cmd) or $self->error("Can't pipe from ".join(' ', @cmd).": $!");
        while (my $line = <LOG>) {
            my ($ip, $login, $date, $method, $uri, $proto, $code, $size, $referer, $browser, $time, $cookies) =
                #($line =~ m/^([^ ]*) - ([^ ]+) (\[[^\]]+\]) "([A-Z]+) (.+) (HTTP\/\d\.\d)" (\d+) ([0-9-]+) "([^"]*)" "([^"]*)" (\d+) "([^"]*)"/);
                ($line =~ m/^([^ ]*) - (.+?) (\[[^\]]+\]) "([A-Z]+) (.+) (HTTP\/\d\.\d)" (\d+) ([0-9-]+) "((?:[^"]|\\")*)" "((?:[^"]|\\")*)" (\d+) "((?:[^"]|\\")*)"/);
            unless ($uri) {
                $self->warn("Unparsable log line in $option{working_dir}/$logfile: '$line'");
                next;
            }

            # TODO : configurable regexp, as of lib/Doloforge/Service/Bulkmail.pm
            $uri =~ s!\?.*!!;
            $uri =~ s!(\.md$)!\.phtml!;
            $uri =~ s!.*(/e-\w+s/)../../../../!$1.../!;
            # FIXME : test previously non traité : /medecin/actualites/medicale/e-docs/foie_gras_et_syndrome_metabolique_142831/document_actu_med.phtml
            $uri =~ s!.*(/e-\w+s/).*/!$1.../!;
            $uri =~ s!http://[^/]+!!;
            $uri =~ s!(/virtual/).*/!$1.../!;

            $size = 0 if ($size eq '-');

            $stat{total}{requests}++;
            $stat{total}{seconds} += $time;
            $stat{total}{bytes} += $size;
            $stat{uri}{$uri}{requests}++;
            $stat{uri}{$uri}{seconds} += $time;
            $stat{uri}{$uri}{bytes} += $size;

            $time = 181 if ($time > 180);
            $stat{totbytime}[$time]++;
            $stat{uribytime}{$uri}[$time]++;
        }
        close(LOG);
    }

#print STDERR Data::Dumper->Dump([\%stat], ['stat']);

    my @report;

    # Expected shape of the CSV results:
    #   URI; number_of_requests; total_time; mean_time; total_bytes; bytes_per_request; bytes_per_second; percent_requests; percent_time; percent_bytes
    push(@report, "$option{output}/report.csv");
    open(REPORT, ">$option{output}/report.csv") or $self->error("Cannot write report file $option{output}/report.csv: $!");
    print REPORT join("\t", qw(uri time_s requests bytes mean_time_ms bytes_per_request bytes_per_second percent_requests percent_time percent_bytes)), "\n";
    foreach (sort {$stat{uri}{$b}{seconds} <=> $stat{uri}{$a}{seconds}} keys(%{$stat{uri}})) {
        print REPORT join("\t", $_,     # uri
            $stat{uri}{$_}{seconds},    # time
            $stat{uri}{$_}{requests},   # requests
            $stat{uri}{$_}{bytes},      # bytes
            ($stat{uri}{$_}{requests} ? int($stat{uri}{$_}{seconds} * 1000 / $stat{uri}{$_}{requests}) : 'N/A'),    # mean_time (ms)
            ($stat{uri}{$_}{requests} ? int($stat{uri}{$_}{bytes} / $stat{uri}{$_}{requests}) : 'N/A'),             # bytes_per_request
            ($stat{uri}{$_}{seconds} ? int($stat{uri}{$_}{bytes} / $stat{uri}{$_}{seconds}) : 'N/A'),               # bytes_per_second
            ($stat{total}{requests} ? int($stat{uri}{$_}{requests} * 100 / $stat{total}{requests}) . '%' : 'N/A'),   # percent_requests
            ($stat{total}{seconds} ? int($stat{uri}{$_}{seconds} * 100 / $stat{total}{seconds}) . '%' : 'N/A'),      # percent_time
            ($stat{total}{bytes} ? int($stat{uri}{$_}{bytes} * 100 / $stat{total}{bytes}) . '%' : 'N/A'),            # percent_bytes
        ), "\n";
    }
    close(REPORT);

    foreach my $report (qw(total)) {
        push(@report, "$option{output}/$report.csv");
        open(REPORT, ">$option{output}/$report.csv") or $self->error("Cannot write report file $option{output}/$report.csv: $!");
        print REPORT join("\t", 'key', $report), "\n";
        foreach (sort {$stat{$report}{$b} <=> $stat{$report}{$a}} keys(%{$stat{$report}})) {
            print REPORT join("\t", $_, $stat{$report}{$_}), "\n";
        }
        close(REPORT);
    }
    push(@report, "$option{output}/totbytime.csv");
    open(REPORT, ">$option{output}/totbytime.csv") or $self->error("Cannot write report file $option{output}/totbytime.csv: $!");
    print REPORT join("\t", 'seconds', 'requests'), "\n";
    for (my $i = 0; $i <= $#{$stat{totbytime}}; $i++) {
        print REPORT join("\t", $i, (defined($stat{totbytime}[$i]) ? $stat{totbytime}[$i] : 0)), "\n";
    }
    close(REPORT);
    push(@report, "$option{output}/uribytime.csv");
    open(REPORT, ">$option{output}/uribytime.csv") or $self->error("Cannot write report file $option{output}/uribytime.csv: $!");
    print REPORT join("\t", 'uri', 'seconds', 'requests'), "\n";
    foreach my $uri (sort {$stat{uri}{$b}{seconds} <=> $stat{uri}{$a}{seconds}} keys(%{$stat{uribytime}})) {
        for (my $i = 0; $i <= $#{$stat{uribytime}{$uri}}; $i++) {
            print REPORT join("\t", $uri, $i, (defined($stat{uribytime}{$uri}[$i]) ? $stat{uribytime}{$uri}[$i] : 0)), "\n";
        }
    }
    close(REPORT);

    # Compress all ouput files
    $self->run_job(
        $option{client}, $option{project},  # client and project configuration hashes
        'gzip',                             # name of the command to call
        # command-line arguments, given to the script, making it easy to get position arguments (last argument is in fact a variable list of input log file names) :
        ['-9', @report],
        # no environment variables...
        {}
    );
}



1;
__END__

=head1 AUTHOR

Nicolas Huillard <F<nicolas@huillard.net>>, Dolomède

=head1 SEE ALSO

doloforge(1)

=cut
