#=============================================================================
# Doloforge::Service::Apache : configure Apache for any client/project
#
# Generate config files from templates, add a config file to the whole,
# reload Apache (well, not yet)
#
# Full help :
#   perldoc Doloforge::Service::Apache
#=============================================================================


package Doloforge::Service::Apache;

use strict;

# This base class implements the constructor and help methods in a generic way.
use base qw(Doloforge::Service);

use File::Basename;
use File::Copy;
use Date::Format;


#=============================================================================
=encoding utf8

=head1 NAME

Doloforge::Service::Apache : Configure Apache for any client/project

=head1 SYNOPSIS

    # Update the full configuration, running each client/project/service update
    doloforge config --update

=head1 DESCRIPTION

Todo

=cut


#=============================================================================
# Method update
#
# Update the service configuration on the host, depending on
# client/project/service config files...
# This method receives :
#   option: context options passesd on the command-line (quite useless there)
#   client_conf: client configuration file contents (client.$name.yaml)
#   project_conf: project configuration file contents (project.$name.client.$name.yaml)

sub update {
    my $self = shift;

    my ($option, $client_conf, $project_conf) = @_;

    # Copy the service skeleton directory into the project directory.
    $self->SUPER::update($option, $client_conf, $project_conf);

    # The following merged templates are all stored on shared storage, so only generated by the NFS servers
    if ($self->has_role('nfs_role')) {
        # Merge the config hashes with the proper template files, and save the result...
        # Apache httpd.conf VirtualHost, and Mason, PHP5, etc. subclasses too.

        # The following files are not generated for the main Doloforge::Service::Apache (apache) class, which is actually replaced by the nginx front-end
        if ($self->{name} ne 'apache') {
            $self->merge_template($option, $client_conf, $project_conf, # Configuration hashes
                # Service-specific template, from [skel]/template/[stage]/apache_*
                "$self->{global}->{dir}->{skel}/template/$self->{name}_httpd.conf.tmpl",
                # Alternate project-service-specific template, to be used if it exists
                "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/conf/$self->{name}_httpd.conf.tmpl",
                # Filled-in file, in [client]/[project]/_conf/ (it's name must be unique among all services)
                "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/_conf/$self->{name}_httpd.conf",
                # Link to the filled-in file, which must be maintained once the project is active
                # (the link location depends on the project port definition, as various Apache processes do not share the same port, thus do not share the same config files)
    #            "$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/"
    #                . (exists($project_conf->{services}->{$self->{name}}->{port}) ? "conf-$project_conf->{services}->{$self->{name}}->{port}.d" : 'conf.d')
    #                . "/$client_conf->{name}_$project_conf->{name}_httpd.conf",
                "$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/conf-".($self->{config}->{stages}->{$self->{global}->{stage}}->{($project_conf->{services}->{$self->{name}}->{version} || 'default')}->{port} || $self->{config}->{port}).".d/$client_conf->{name}_$project_conf->{name}.$self->{name}_httpd.conf",
            );
        }

        # The following files are generated only for the main Doloforge::Service::Apache (apache) class
        if ($self->{name} eq 'apache') {
            # Generate the project's user authentication parts, for the main Apache class only
            if (-f "$self->{global}->{dir}->{skel}/template/$self->{name}_users.tmpl") {
                # Apache user file partial template, allowing Apache authentication for /stats/ access
                $self->merge_template($option, $client_conf, $project_conf, # Configuration hashes
                    # Service-specific template, from [skel]/template/apache_*
                    "$self->{global}->{dir}->{skel}/template/$self->{name}_users.tmpl",
                    # Alternate project-service-specific template, to be used if it exists
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/conf/$self->{name}_users.tmpl",
                    # Filled-in file, in [client]/[project]/_conf/ (it's name must be unique among all services)
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/_conf/$self->{name}_users",
                    # Link to the filled-in file, which must be maintained once the project is active
                    # no need for that "$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/users/$client_conf->{name}_$project_conf->{name}_$self->{name}_users",
                );
            }

            # Generate the project's main Analog configuration, defining project name etc.
            if (-f "$self->{global}->{dir}->{skel}/template/report.analog.cfg.tmpl") {
                # Apache user file partial template, allowing Apache authentication for /stats/ access
                $self->merge_template($option, $client_conf, $project_conf, # Configuration hashes
                    # Service-specific template, from [skel]/template/apache_*
                    "$self->{global}->{dir}->{skel}/template/report.analog.cfg.tmpl",
                    # Alternate project-service-specific template, to be used if it exists
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/conf/report.analog.cfg.tmpl",
                    # Filled-in file, in [client]/[project]/_conf/ (it's name must be unique among all services)
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/_conf/report.analog.cfg",
                    # Link to the filled-in file, which must be maintained once the project is active
                    undef,
                );
            }

            # nginx is an alternative for the Apache front-end : generate it's configuration too
            if (-f "$self->{global}->{dir}->{skel}/template/nginx_httpd.conf.tmpl") {
                # Apache user file partial template, allowing Apache authentication for /stats/ access
                $self->merge_template($option, $client_conf, $project_conf, # Configuration hashes
                    # Service-specific template, from [skel]/template/apache_*
                    "$self->{global}->{dir}->{skel}/template/nginx_httpd.conf.tmpl",
                    # Alternate project-service-specific template, to be used if it exists
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/conf/nginx_httpd.conf.tmpl",
                    # Filled-in file, in [client]/[project]/_conf/ (it's name must be unique among all services)
                    "$self->{global}->{dir}->{shared}/$client_conf->{name}/$project_conf->{name}/_conf/nginx_httpd.conf",
                    # Link to the filled-in file, which must be maintained once the project is active
                    # (the link location depends on the project port definition, as various Apache processes do not share the same port, thus do not share the same config files)
                "$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/conf-".($self->{config}->{stages}->{$self->{global}->{stage}}->{($project_conf->{services}->{$self->{name}}->{version} || 'default')}->{port} || $self->{config}->{port}).".d/$client_conf->{name}_$project_conf->{name}.nginx_httpd.conf",
                );
            }
        }
    }
}


#=============================================================================
# Method finish
#
# Called when the whole processing is finished, when it's time to reload services.
# This generic class will do nothing special...

sub finish {
    my $self = shift;

    # The vlogger logging main directory must pre-exist on application servers before we start Apache
    if ($self->has_role('appserv_role')) {
        $self->dir_entry(1, "$self->{global}->{stage} varlog $self->{name}/vlogger-$self->{config}->{port}.d",   # always create the dir (1), directory description in messages
            "$self->{global}->{dir}->{varlog}/$self->{name}/vlogger-$self->{config}->{port}.d",                  # directory name
            oct($self->{global}->{permissions}->{varlog}->[0]),                     # permission on creation and update (in decimal form, hence the oct())
            $self->{global}->{permissions}->{user}->[0],                            # user is the generic doloforge ID, as in the apache.conf file, in the vlogger command line
            $self->{global}->{permissions}->{group}->[0],                           # group is the generic doloforge group
            undef,                                                                  # there is no skeleton for this directory, as vlogger will manage it completely
        ) if ($self->{config}->{port});
        foreach my $version_conf (values(%{$self->{config}->{stages}->{$self->{global}->{stage}}})) {
            my $port = ($version_conf->{port} || $self->{config}->{port});
            $self->dir_entry(1, "$self->{global}->{stage} varlog $self->{name}/vlogger-$port.d",   # always create the dir (1), directory description in messages
                "$self->{global}->{dir}->{varlog}/$self->{name}/vlogger-$port.d",                  # directory name
                oct($self->{global}->{permissions}->{varlog}->[0]),                 # permission on creation and update (in decimal form, hence the oct())
                $self->{global}->{permissions}->{user}->[0],                        # user is the generic doloforge ID, as in the apache.conf file, in the vlogger command line
                $self->{global}->{permissions}->{group}->[0],                       # group is the generic doloforge group
                undef,                                                              # there is no skeleton for this directory, as vlogger will manage it completely
            ) if ($port);
        }
    }

    # If any modification was made in the configuration for this service, reload everything that depends upon it.
    # It may be too much work, if only a tiny part was modified... We'll optimize later on...
    # This will only be done on machines that have a role for this class, ie. machines where
    # the update() method have updated something...
    if ($self->{modified}) {
        # BEWARE : daemons are reloaded on application servers, while configuration have been regenerated on nfs-server
        #   - we may not know it's been regenerated
        #   - we may reload before the files are actually completed
        $self->warn("Modified $self->{modified} files: need to reload something for service '$self->{name}' ? (reload: $self->{config}->{reload})");
    }
}


#=============================================================================
# Method maintenance_on_appserv
#
# Run maintenance tasks on the application servers. Note that since this class
# is subclassed by many other Apache-based services, all of them using vlogger
# (per the finish() method above), we just implement the logfile management
# stuff once, but it will run as many times as there are subclasses...

sub maintenance_on_appserv {
    my $self = shift;

    my ($interval) = @_;
    $self->info("Maintenance for $self->{name}/$self->{global}->{stage}/$interval on appserv");

    # We need to exclude the log files for the current day, matching the vlogger file format defined in apache.conf et al...
    my $time = time;
    my $today = time2str("%Y%m%d", $time);
    # We also need the current timezone, with a quick hack here ("+HHMM" string format)
    my $tz = sprintf("%+05d", (((localtime($time))[2] * 100 + (localtime($time))[1]) + 2400 - ((gmtime($time))[2] * 100 + (gmtime($time))[1])) % 2400);

    # When moving the files to the shared space, we must change user ID to the doloforge one (same as the vlogger process uid/gid), because :
    #   - it's a bit safer
    #   - root-squash on NFS server would prevent the file copy
    $self->run_sub(
        "$self->{name} sending log files to shared space",
        $self->{global}->{permissions}->{user}->[1], $self->{global}->{permissions}->{group}->[1],
        sub {
            # Move vlogger closed files from the local tree to the shared tree.
            # This is done on each application server, and provides the NFS server with all the logs.
            # The NFS server will then be able to merge them, resolve IPs, analyse, etc.

#            # The vhost names are constructed (see apache_httpd.conf.tmpl) using the local domain name : just use the exact same formula here...
#            my $local_domain = ($self->{global}->{local_domain} || "doloforge");

            # Search for every vhost directory vlogger could have created for the stage we're running on...
            # This include the main VirtualHost "_default_"
#            #   config files should be in {shared}/.doloforge/apache/conf-{port}.d/{localdomain}.conf
#            #       or {localdomain}.{service}.conf or have {localdomain} include {service} : "mason.jim4.jim.local.doloforge.net" or "jim4.jim.mason.doloforge.net" (thus have multiple services served by a single back-end : php5 + mason)
#            #   get back from vlogger-{port} to service name using conf symlink (lstat) :
#            #       /srv/doloforge/shared/.doloforge/mason/conf-84.d/jim_jim4_httpd.conf -> /srv/doloforge/shared/jim/jim4/_conf/mason_httpd.conf
            # TODO : après cette boucle :
            #   - supprimer les liens symboliques "access.log -> access.log.20180921" plus vieux que 1j :
            #       find /var/log/doloforge/*/vlogger-*.d/ -mindepth 2 -type l -xtype l -daystart -mtime +1 -delete
            #   - puis supprimer les répertoire vides :
            #       find /var/log/doloforge/*/vlogger-*.d/ -mindepth 1 -type d -empty -delete
            #   - et alerter des fichiers restants plus vieux que 1j (logs de non-sites dont le host est fictif, sans doute provenant d'une mauvaise identification dans un error.log)
            #       find /var/log/doloforge/*/vlogger-*.d/ -type f -daystart -mtime +0 -ls
            foreach my $vhost (glob("$self->{global}->{dir}->{varlog}/$self->{name}/vlogger*/{*.*.$self->{global}->{stage}.*,_$self->{global}->{stage}_}")) {
                # This glob can find logs that correspond to no project (after rename), or unused services ("Failed to create /srv/doloforge/shared/adversitement/seosem/apache/_logs directory:")
                next unless -d $vhost;
                $self->debug("Checking log files in $vhost");
                # Compute the client and project name from the vhost name (reversed from apache_httpd.conf.tmpl)
                my ($project, $client) = (basename($vhost) =~ m{^([^/\.]+)\.([^/\.]+)\.});
                # Identify the inherited service which effectively triggered those logs, from the port used, which matches the one used in the config symlink name
                my ($port) = ($vhost =~ m{/vlogger-(\d+)\.d/});
#                my ($service);
#                foreach (glob("$self->{global}->{dir}->{shared}/.doloforge/apache/conf-$port.d/${client}_$project.*_httpd.conf")) {
#                    ($service) = (m{/${client}_$project\.([^/\.]+)_httpd.conf$});
#                    last if $service;
#                }
#                $service ||= $self->{name};
                my $service = $self->{name};
                # nginx is actually an alias for the apache service: fix this one
                # FIXME : this service identification should be better handled, and "apache" should actually be the "http" base service
                $service = 'apache' if ($service eq 'nginx');
                $port ||= 'default'; # If the port is not part of the vlogger*/ directory name, we must still avoid "Not moving $file to $destdir/$destfile: destination file already exists" by having ie. ":default" instead of ":80"...
                # Create a destination directory, which is within the service directory in the project tree
                my $destdir = ($self->{global}->{stages}->{$self->{global}->{stage}}->{dir}->{shared} || $self->{global}->{dir}->{shared});
                if ($client) {
                    if ($project) {
                        # If we identified a project name, the logs are stored in something like
                        #   /srv/doloforge/shared/[client]/[project]/php5/_log/
                        #$destdir = "$self->{global}->{dir}->{shared}/$client/$project/$service/_logs";
                        $destdir .= "/$client/$project/$service/_logs";
                    } else {
                        # If we didn't identify a project name, the logs are stored in something like
                        #   /srv/doloforge/shared/[client]/lib/_logs/php5/
                        # (this is to prevent confusing ./lib/php/ and ./lib/perl/ vs. the ./lib/_logs/ and ./lib/_stats/ dirs)
                        #$destdir = "$self->{global}->{dir}->{shared}/$client/lib/_logs/$service";
                        $destdir .= "/$client/lib/_logs/$service";
                    }
                } else {
                    # If there is no client at all (generic Apache warnings and errors), the logs are stored in
                    #   /srv/doloforge/shared/.doloforge/php5/_log/
                    $destdir .= "/.doloforge/$service/_logs-$port.d";
                }
                unless (-d $destdir) {
                    $self->warn("Creating $destdir directory");
                    mkdir($destdir) or $self->warn("Failed to create $destdir directory: $!");
                }

                # In each vhost directory, look for every remaining log files from previous (only) days (there may be none)
                foreach my $file (glob("$vhost/*.log.????????")) { # access.log.YYYYMMDD or error.log.YYYYMMDD
                    next if ($file =~ m{$today$}); # Exclude "today"
                    my ($destfile) = basename($file) . ".host.$self->{global}->{host}:$port"; # access.log.YYYYMMDD.host.[hostname]:[port]
                    my ($tempfile) = ".tmp.$destfile"; # .tmp.access.log.YYYYMMDD.host.[hostname]:[port]

                    unless (-e "$destdir/$destfile") {
                        # Move the files to the shared space, using a temporary name there : we must make sure the NFS server will not pick partially transfered files...
                        $self->info("Moving $file to $destdir/$destfile");
                        if (move($file, "$destdir/$tempfile")) { # ...this is a file copy + delete...
                            unless (move("$destdir/$tempfile", "$destdir/$destfile")) { # ...while this is a simple rename...
                                $self->warn("Failed to rename $destdir/$tempfile to $destdir/$destfile: $!");
                            }
                        } else {
                            # Erreur inexplicable : helium a créé plusieurs répertoires destination, mais pas celui-là ; à la fin, le répertoire existe, mais le fichier n'y est pas !!!
                            # FIXME Explication: _default_ est traité au titre de chaque étage, en concurrence : comme on ne peut pas l'affecter a priori à un étage, il faut décider !!!
                            #   Aug 18 09:39:01 helium doloforge[22912]: mason info: Moving /var/log/doloforge/mason/vlogger-8001.d/_default_/error.log.20180817 to /srv/doloforge/devl/.doloforge/mason/_logs-8001.d/error.log.20180817.host.helium:8001
                            #   Aug 18 09:39:01 helium doloforge[22908]: mason info: Moving /var/log/doloforge/mason/vlogger-8001.d/_default_/error.log.20180817 to /srv/doloforge/test/.doloforge/mason/_logs-8001.d/error.log.20180817.host.helium:8001
                            #   Aug 18 09:39:01 helium doloforge[22912]: mason warning: Failed to move /var/log/doloforge/mason/vlogger-8001.d/_default_/error.log.20180817 to /srv/doloforge/devl/.doloforge/mason/_logs-8001.d/.tmp.error.log.20180817.host.helium:8001: No such file or directory
                            #
                            #   Aug 18 09:39:01 sodium doloforge[18844]: apache info: Moving /var/log/doloforge/apache/vlogger-80.d/_default_/access.log.20180817 to /srv/doloforge/test/.doloforge/apache/_logs-80.d/access.log.20180817.host.sodium:80
                            #   Aug 18 09:39:01 sodium doloforge[18841]: apache info: Moving /var/log/doloforge/apache/vlogger-80.d/_default_/access.log.20180817 to /srv/doloforge/devl/.doloforge/apache/_logs-80.d/access.log.20180817.host.sodium:80
                            #   Aug 18 09:39:01 sodium doloforge[18841]: apache warning: Failed to move /var/log/doloforge/apache/vlogger-80.d/_default_/access.log.20180817 to /srv/doloforge/devl/.doloforge/apache/_logs-80.d/.tmp.access.log.20180817.host.sodium:80: No s
                            #   Aug 18 09:39:01 sodium doloforge[18841]: apache info: Moving /var/log/doloforge/apache/vlogger-80.d/_default_/error.log.20180817 to /srv/doloforge/devl/.doloforge/apache/_logs-80.d/error.log.20180817.host.sodium:80
                            #   Aug 18 09:39:02 sodium doloforge[18844]: apache info: Moving /var/log/doloforge/apache/vlogger-80.d/_default_/error.log.20180817 to /srv/doloforge/test/.doloforge/apache/_logs-80.d/error.log.20180817.host.sodium:80
                            $self->warn("Failed to move $file to $destdir/$tempfile: $!");
                        }
                    } else {
                        # We must be absolutely sure not to overwrite existing files (in case of bugs introduced above in the future; yes, this occured)
                        $self->warn("Not moving $file to $destdir/$destfile: destination file already exists");
                    }
                }
            }

#            # Process the global error.log file, changing date format in the process (enabling mergelog to decode dates), and dealing with date-less lines
#            # WARNING:
#            # This section is deprecated, since error.log files are now piped to vlogger, which splits them to project dirs.
#            # The uncategorized lines are in _default_/error.log. All this means that the above sections does the stuff...
#            my $destdir = "$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/_logs";
#            foreach my $file (glob("$self->{global}->{dir}->{varlog}/$self->{name}/vlogger/error.log.????????")) {
#                next if ($file =~ m{$today$}); # Exclude "today"
#                my ($destfile) = basename($file) . ".host.$self->{global}->{host}"; # error.log.YYYYMMDD.host.[hostname]
#                my ($tempfile) = ".tmp.$destfile"; # .tmp.error.log.YYYYMMDD.host.[hostname]
#
#                # Move the files to the shared space, using a temporary name there : we must make sure the NFS server will not pick partially transfered files...
#                $self->info("Moving $file to $destdir/$destfile");
#                open(INFILE, "<$file") or die("Can't read error log file $file: $!");
#                open(OUTFILE, ">$destdir/$tempfile") or die("Can't write error log file $destdir/$tempfile: $!");
#                # Default first date, if the first line of the file does not contains a date. Next date-less lines will copy previous line's formatted date.
#                # For some reason, mergelog segfaults on 1970-01-01, so any other date will do...
#                my $previous_date = "01/Jan/2000:00:00:00 $tz";
#                while (<INFILE>) {
#                    chomp;
#                    # Identify the timestamp in the source file, as "[Wed Sep  2 09:21:01 2009]"
#                    my ($dow, $month, $date, $hour, $min, $sec, $year, $line) = ($_ =~ m{^\[(\w{3}) (\w{3}) +(\d+) (\d{2}):(\d{2}):(\d{2}) (\d{4})\] (.*)});
#                    if ($line) {
#                        $previous_date = sprintf("%02d/%s/%04d:%02d:%02d:%02d %s", $date, $month, $year, $hour, $min, $sec, $tz);
#                    } else {
#                        $line = "[nodate] $_";
#                    }
#                    # Write the line with the timestamp as "[02/Sep/2009:08:18:02 +0200]" (mergelog really wants "%d/%3c/%d:%d:%d:%d" after a "[" located at least 9 chars from the beginning of the line)
#                    # The line formatted as below allow to pipe the error.log files into mergelog + jdresolve, just like the access.log files, which is GOOD...
#                    print OUTFILE "127.0.0.1 - - [$previous_date] $line\n";
#                }
#                close(INFILE);
#                close(OUTFILE);
#
#                # Remove the original INFILE after processing it
#                unlink($file) or $self->warn("Failed to remove $file: $!");
#                # Rename the temporary file to it's final name (this "move" is a simple "rename")
#                unless (move("$destdir/$tempfile", "$destdir/$destfile")) {
#                    $self->warn("Failed to rename $destdir/$tempfile to $destdir/$destfile");
#                }
#            }
        }
    ) if ($interval eq 'daily');
}

sub maintenance_on_nfs {
    my $self = shift;

    my ($interval) = @_;
    $self->info("Maintenance for $self->{name}/$self->{global}->{stage}/$interval on nfs");

    # Merge all transfered files from application servers (see above), and prepare them for analysis
    # Then we can merge log files from all application servers, resolve IPs, etc.
    my $jdresolvedb = "$self->{global}->{dir}->{shared}/.doloforge/cron/jdresolve.db";

    # Change user ID before merging the files (same as the vlogger process uid/gid)
    $self->run_sub(
        "$self->{name} merging/resolving log files",
        $self->{global}->{permissions}->{user}->[1], $self->{global}->{permissions}->{group}->[1],
        sub {
            # Expire old entries in the jdresolve cache DB (long predefined TTL, because reverse resolution do not usually change very often, and wrong values are not critical)
            # Beware : jdresolve will either resolve IPs OR expire DB, but will not do both in a single run ! So don't add the expire option to the mergelog|jdresolve run below !
            # There's no other way to silence jdresolve than sending STDERR to /dev/null, but it does not have much else to say, so it's nearly OK
            system("$self->{global}->{commands}->{jdresolve} $self->{global}->{commands}->{jdresolve_expire} --database=$jdresolvedb 2>&-");

            # List all active projects (it does not matter that a project is active or not, since logs files will probably not exist in an inactive project)
            my $clients = (&Doloforge::Command::Factory('Client', $self->{global}))->_list();
            my $projects = (&Doloforge::Command::Factory('Project', $self->{global}))->_list();

            # Process log files in [client]/[project]/[service]/_logs/
            foreach (sort(keys(%{$projects}))) {
                my $project = $projects->{$_};
                my $client = $clients->{$project->{client}};

                # Process pending non-merged log files in this project : [service]/_logs/access.log.YYYYMMDD.host.[hostname]
                # This will process both the access.log.* and the error.log.* files
                $self->_merge_resolve_archive_logs("$self->{global}->{dir}->{shared}/$project->{client}/$project->{name}/$self->{name}/_logs", $jdresolvedb);
            }

            # Process log files in [client]/lib/_logs/*/ (probably error logs only)
            foreach (sort(keys(%{$clients}))) {
                my $client = $clients->{$_};

                # Process pending non-merged log files : _logs/[service]/error.log.YYYYMMDD.host.[hostname]
                $self->_merge_resolve_archive_logs("$self->{global}->{dir}->{shared}/$client->{name}/lib/_logs/$self->{name}", $jdresolvedb);
            }

            # Process log files in .doloforge/[service]/_logs-[port].d/
            # FIXME : il faudrait ausso traiter .doloforge/$service/_logs-$port.d/ (cf. l.288)
            #   find /srv/doloforge/*/.doloforge/*/_logs-*.d/ -type f -ls
            foreach my $logsdir (glob("$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/_logs-*.d")) {
                # Process pending non-merged log files : .doloforge/[service]/_logs.[port]-d/error.log.YYYYMMDD.host.[hostname]
                $self->_merge_resolve_archive_logs($logsdir, $jdresolvedb);
            }
        }
    ) if ($interval eq 'daily');

    # Also merge _default_ log files, found in the shared tree
    # This will process both the access.log.* and the error.log.* files
    $self->_merge_resolve_archive_logs("$self->{global}->{dir}->{shared}/.doloforge/$self->{name}/_logs", $jdresolvedb)
        if ($interval eq 'daily');
}

sub _merge_resolve_archive_logs {
    my $self = shift;

    my ($logsdir, $jdresolvedb) = @_;

    # The glob can find 10 files, but we will loop only 5 times, because the inner code will process many files at once (one for each host) : 5 different days * 2 sending hosts...
    foreach (glob("$logsdir/*.log.????????.host.*")) {
        next unless -e $_; # The file may have been removed by the inner "rm", after the glob() was evaluated... This is normal.
        my $elapsed = time;
        my ($file) = ($_ =~ m!/([a-z]+\.log\..{8})\.host\.!); # $file = access.log.YYYYMMDD or error.log.YYYYMMDD

        # We use a subshell to run this complex piped command :
        #   - merge all same-date files
        #   - resolve IP addresses with a shared cache DB (--nostats)
        #   - keep an uncompressed version of the merged file (will be used very soon by log analysis: too bad if we do not keep it, at least temporarily)
        #   - also create a compressed version for archival
        #   - then remove the original file if everything is OK
        #   - make sure the pipe will report the result code of the first failing command (which will probably be mergelog or jdresolve)
        # jdresolve options :
        #   --nostats : don't display stats after processing
        #   --timeout=<seconds> : timeout in seconds for each host resolution (default is 30 seconds) (value based on stats observations on the doloforge.com cluster)
        #   --expiredb=<hours> : expires entries in the database that are older than <hours> hours
#        my $command = join(' ', split(/\s+/, # join(split("")) will simply remove formatting spaces and newlines in the output traces file and actual shell command...
#            "set -o pipefail;
#            mergelog $logsdir/$file.host.*
#            | jdresolve $self->{global}->{commands}->{jdresolve_opts} --database=$jdresolvedb -
#            | gzip --to-stdout --best --rsyncable
#            > $logsdir/$file.gz
#            && rm -f $logsdir/$file.host.*"
#        ));
        # 20180710/NH : replace mergelog/jdresolve with logresolvemerge.pl from awstats, because mergelog can't process dates formated as in ErrorLogFormat
        # FIXME : do not create empty files / why are there empty files ???
        #   /srv/doloforge/prod/.doloforge/apache/_logs-80.d/error.log.20180801.gz
        #   /srv/doloforge/devl/.doloforge/apache/_logs-81.d/error.log.20180721.gz
        # TODO : added ls -l to check for files before logresolvemerge.pl / do not remove source
        # To debug logresolvemerge.pl:
        #   ./bin/logresolvemerge.pl -debug=9 /srv/doloforge/prod/internal/sandbox/apache/_logs/.orig.error.log.20180811.host.localhost\:80
        # 20180930/NH : switch from "logresolvemerge.pl -dnslookup" to "jdresolve" without --nostats
        my $command = join(' ', split(/\s+/, # join(split("")) will simply remove formatting spaces and newlines in the output traces file and actual shell command...
            "set -o pipefail;
            $self->{global}->{dir}->{bin}/logresolvemerge.pl $logsdir/$file.host.*
            | $self->{global}->{commands}->{jdresolve} $self->{global}->{commands}->{jdresolve_opts} --database=$jdresolvedb -
            | gzip --to-stdout --best --rsyncable
            > $logsdir/$file.gz
            && rm -f $logsdir/$file.host.*"
        ));
        # Add this before gzip, if an uncompressed file is needed (beware about their size : compressed files take 5% of the space of the raw files (17744MB vs. 881MB))
        #    | tee $logsdir/$file
        # Add thise before rm, to keep original files under .orig.* name, and not rm them
        #    && rename s/$file/.orig.$file/ $logsdir/$file.host.*

        # Actually run the command...
        $self->info("Running command : $command");
        my $rc = system('/bin/bash', '-c', $command);

        # Test for return code et al, and report to the doloforge administrator
        if ($rc == -1) {
            $self->warn("Failed to execute $command: $!");
        } elsif ($rc & 127) {
            $self->warn("Child $command died with signal ".($rc & 127).", ".(($rc & 128) ? 'with' : 'without')." coredump");
        } elsif ($rc >= 256) {
            $self->warn("Child $command exited with value ".($rc >> 8));
        } else {
            $self->info("Finished $command rc=$rc elapsed=".(time - $elapsed));
        }
    }
}



1;
__END__

=head1 AUTHOR

Nicolas Huillard <F<nicolas@huillard.net>>, Dolomède

=head1 SEE ALSO

doloforge(1)

=cut
