package Thruk::Backend::Provider::Mysql;

use warnings;
use strict;
use Carp qw/confess/;
use Data::Dumper qw/Dumper/;
use Module::Load qw/load/;
use POSIX ();
use Time::HiRes qw/gettimeofday tv_interval/;

use Thruk::Timer qw/timing_breakpoint/;
use Thruk::Utils ();
use Thruk::Utils::Log qw/:all/;

use parent 'Thruk::Backend::Provider::Base';

=head1 NAME

Thruk::Backend::Provider::Mysql - connection provider for Mysql connections

=head1 DESCRIPTION

connection provider for Mysql connections

=head1 METHODS

=cut

$Thruk::Backend::Provider::Mysql::cache_version = 6;

$Thruk::Backend::Provider::Mysql::db_types = {
    'INITIAL HOST STATE'      => 6, # LOGCLASS_STATE
    'CURRENT HOST STATE'      => 6, # LOGCLASS_STATE
    'HOST ALERT'              => 1, # LOGCLASS_ALERT
    'HOST DOWNTIME ALERT'     => 1, # LOGCLASS_ALERT
    'HOST FLAPPING ALERT'     => 1, # LOGCLASS_ALERT

    'INITIAL SERVICE STATE'   => 6, #LOGCLASS_STATE
    'CURRENT SERVICE STATE'   => 6, # LOGCLASS_STATE
    'SERVICE ALERT'           => 1, # LOGCLASS_ALERT
    'SERVICE DOWNTIME ALERT'  => 1, # LOGCLASS_ALERT
    'SERVICE FLAPPING ALERT'  => 1, # LOGCLASS_ALERT

    'TIMEPERIOD TRANSITION'   => 6, # LOGCLASS_STATE

    'HOST NOTIFICATION'       => 3, # LOGCLASS_NOTIFICATION
    'SERVICE NOTIFICATION'    => 3, # LOGCLASS_NOTIFICATION

    'PASSIVE SERVICE CHECK'   => 4, # LOGCLASS_PASSIVECHECK
    'PASSIVE HOST CHECK'      => 4, # LOGCLASS_PASSIVECHECK

    'SERVICE EVENT HANDLER'   => 0, # INFO
    'HOST EVENT HANDLER'      => 0, # INFO

    'EXTERNAL COMMAND'        => 5, # LOGCLASS_COMMAND
    'LOG ROTATION'            => 0, # INFO
};

$Thruk::Backend::Provider::Mysql::db_classes = {
    'INFO'          => 0,
    'ALERT'         => 1,
    'PROGRAMM'      => 2,
    'NOTIFICATION'  => 3,
    'PASSIVE'       => 4,
    'COMMAND'       => 5,
    'STATE'         => 6,
};

use constant {
    MODE_IMPORT         => 1,
    MODE_UPDATE         => 2,
};

@Thruk::Backend::Provider::Mysql::tables = (qw/contact contact_host_rel contact_service_rel host log service status/);

##########################################################

END {
    # close all connections at the end
    return unless $INC{"DBI.pm"};
    DBI->visit_handles(sub {
        my($dbh) = @_;
        if($dbh->{'Type'} eq 'db') {
            $dbh->disconnect;
        }
        return 1;
    });
}

##########################################################

=head2 new

create new manager

=cut
sub new {
    my($class, $peer_config) = @_;

    my $options = $peer_config->{'options'};
    confess('need at least one peer. Minimal options are <options>peer = mysql://user:password@host:port/dbname</options>'."\ngot: ".Dumper($peer_config)) unless defined $options->{'peer'};

    $options->{'name'} = 'mysql' unless defined $options->{'name'};
    if(!defined $options->{'peer_key'}) {
        confess('please provide peer_key');
    }
    my($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock);
    if($options->{'peer'} =~ m/^mysql:\/\/(.*?)(|:.*?)@([^:]+)(|:.*?)\/([^\/]*?)$/mx) {
        $dbuser = $1;
        $dbpass = $2;
        $dbhost = $3;
        $dbport = $4;
        $dbname = $5;
        $dbpass =~ s/^://gmx;
        $dbport =~ s/^://gmx;
        if($dbhost =~ m|/|mx) {
            $dbsock = $dbhost;
            $dbhost = 'localhost';
        }
    } else {
        die('Mysql connection must match this form: mysql://user:password@host:port/dbname');
    }

    my $self = {
        'dbhost'      => $dbhost,
        'dbport'      => $dbport,
        'dbname'      => $dbname,
        'dbuser'      => $dbuser,
        'dbpass'      => $dbpass,
        'dbsock'      => $dbsock,
        'peer_config' => $options,
        'verbose'     => 0,
    };
    bless $self, $class;

    return $self;
}

##########################################################

=head2 reconnect

recreate database connection

=cut
sub reconnect {
    my($self) = @_;
    $self->_disconnect();
    return;
}

##########################################################

=head2 _disconnect

close database connection

=cut
sub _disconnect {
    my($self) = @_;
    if(defined $self->{'mysql'}) {
        &timing_breakpoint('disconnect');
        $self->{'mysql'}->disconnect();
        delete $self->{'mysql'};
    }
    return;
}

##########################################################

=head2 _dbh

try to connect to database and return database handle

=cut
sub _dbh {
    my($self) = @_;
    if(!defined $self->{'mysql'} || $self->{'mysql'}->ping()) {
        &timing_breakpoint('connecting '.$self->{'dbname'}.' '.($self->{'dbsock'} || $self->{'dbhost'}).($self->{'dbport'} ? ':'.$self->{'dbport'} : ''));
        if(!$self->{'modules_loaded'}) {
            load DBI;
            load File::Temp, qw/tempfile/;
            load Encode, qw/encode_utf8/;
            $self->{'modules_loaded'} = 1;
        }
        my $dsn = "DBI:mysql:database=".$self->{'dbname'}.";host=".$self->{'dbhost'};
        $dsn .= ";port=".$self->{'dbport'} if $self->{'dbport'};
        $dsn .= ";mysql_socket=".$self->{'dbsock'} if $self->{'dbsock'};
        $dsn .= ";mysql_ssl=0";
        $self->{'mysql'} = DBI->connect_cached($dsn, $self->{'dbuser'}, $self->{'dbpass'}, {RaiseError => 1, AutoCommit => 0, mysql_enable_utf8 => 1, mysql_local_infile => 1});
        $self->{'mysql'}->do("SET NAMES utf8 COLLATE utf8_bin");
        $self->{'mysql'}->do("SET myisam_stats_method=nulls_ignored");
        &timing_breakpoint('connected');
    }
    return $self->{'mysql'};
}

##########################################################

=head2 peer_key

return the peers key

=cut
sub peer_key {
    my($self, $new_val) = @_;
    if(defined $new_val) {
        $self->{'peer_config'}->{'peer_key'} = $new_val;
    }
    return $self->{'peer_config'}->{'peer_key'};
}


##########################################################

=head2 peer_addr

return the peers address

=cut
sub peer_addr {
    my $self = shift;
    return $self->{'peer_config'}->{'peer'};
}

##########################################################

=head2 peer_name

return the peers name

=cut
sub peer_name {
    my $self = shift;
    return $self->{'peer_config'}->{'name'};
}

##########################################################

=head2 send_command

=cut
sub send_command {
    confess("not implemented");
}

##########################################################

=head2 get_processinfo

=cut
sub get_processinfo {
    confess("not implemented");
}

##########################################################

=head2 get_sites

=cut

sub get_sites {
    confess("unimplemented");
}

##########################################################

=head2 get_can_submit_commands

=cut
sub get_can_submit_commands {
    confess("not implemented");
}

##########################################################

=head2 get_contactgroups_by_contact

=cut
sub get_contactgroups_by_contact {
    confess("not implemented");
}

##########################################################

=head2 get_hosts

=cut
sub get_hosts {
    confess("not implemented");
}

##########################################################

=head2 get_hosts_by_servicequery

=cut
sub get_hosts_by_servicequery {
    confess("not implemented");
}

##########################################################

=head2 get_host_names

=cut
sub get_host_names{
    confess("not implemented");
}

##########################################################

=head2 get_hostgroups

=cut
sub get_hostgroups {
    confess("not implemented");
}

##########################################################

=head2 get_hostgroup_names

=cut
sub get_hostgroup_names {
    confess("not implemented");
}

##########################################################

=head2 get_services

=cut
sub get_services {
    confess("not implemented");
}

##########################################################

=head2 get_service_names

=cut
sub get_service_names {
    confess("not implemented");
}

##########################################################

=head2 get_servicegroups

=cut
sub get_servicegroups {
    confess("not implemented");
}

##########################################################

=head2 get_servicegroup_names

=cut
sub get_servicegroup_names {
    confess("not implemented");
}

##########################################################

=head2 get_comments

=cut
sub get_comments {
    confess("not implemented");
}

##########################################################

=head2 get_downtimes

=cut
sub get_downtimes {
    confess("not implemented");
}

##########################################################

=head2 get_contactgroups

=cut
sub get_contactgroups {
    confess("not implemented");
}

##########################################################

=head2 get_contactgroup_names

=cut
sub get_contactgroup_names {
    confess("not implemented");
}

##########################################################

=head2 get_logs

  get_logs

returns logfile entries

=cut
sub get_logs {
    my($self, %options) = @_;

    my $orderby = '';
    my $sorted  = 0;
    if(defined $options{'sort'}->{'DESC'} and $options{'sort'}->{'DESC'} eq 'time') {
        $orderby = ' ORDER BY l.time DESC';
        $sorted  = 1;
    }
    if(defined $options{'sort'}->{'ASC'} and $options{'sort'}->{'ASC'} eq 'time') {
        $orderby = ' ORDER BY l.time ASC';
        $sorted  = 1;
    }

    my $limit = '';
    my $pager_limit;
    if(defined $options{'options'} && $options{'options'}->{'limit'}) {
        $limit = ' LIMIT '.$options{'options'}->{'limit'};
    } else {
        if($options{'pager'}) {
            $pager_limit = $options{'pager'}->{'entries'} * $options{'pager'}->{'page'};
            $limit = ' LIMIT '.$pager_limit;
        }
    }
    $limit = ' LIMIT 10000000' unless $limit; # not using a limit makes mysql not use an index

    my $prefix = $options{'collection'};
    $prefix    =~ s/^logs_//gmx;
    my $dbh = $self->_dbh;

    $self->{'query_meta'} = {
        dbh     => $dbh,
        prefix  => $prefix,
    };
    my($where,$auth_data) = $self->_get_filter($options{'filter'});

    return unless _tables_exist($dbh, $prefix);

    # check logcache version
    my @versions = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 4 LIMIT 1')};
    if(scalar @versions < 1 || $versions[0] != $Thruk::Backend::Provider::Mysql::cache_version) {
        confess(sprintf("Logcache too old, required version %s but got %s. Run 'thruk logcache update' to upgrade.", $Thruk::Backend::Provider::Mysql::cache_version, $versions[0] // '0'));
    }

    # check compact timerange and set a warning flag
    my $c =$Thruk::Globals::c;
    if($c) {
        my $compact_start_data = Thruk::Utils::get_expanded_start_date($c, $c->config->{'logcache_compact_duration'});
        # get time filter
        my($start, $end) = Thruk::Utils::extract_time_filter($options{'filter'});
        if($start && $start < $compact_start_data) {
            $c->stash->{'logs_from_compacted_zone'} = 1;
        }
        elsif($end && $end < $compact_start_data) {
            $c->stash->{'logs_from_compacted_zone'} = 1;
        }
    }

    my $extra_columns = '';
    if($options{'extra_columns'}) {
        $extra_columns = ',
            (CASE
                WHEN l.type = "HOST NOTIFICATION"    THEN SUBSTRING_INDEX(SUBSTRING_INDEX(l.message, ";", 4), ";", -1)
                WHEN l.type = "SERVICE NOTIFICATION" THEN SUBSTRING_INDEX(SUBSTRING_INDEX(l.message, ";", 5), ";", -1)
                ELSE ""
            END) as command_name
        ';
    }

    my $sql = '
    SELECT
        l.time as time,
        l.class as class,
        l.type as type,
        l.state as state,
        l.state_type as state_type,
        IFNULL(h.host_name, "") as host_name,
        IFNULL(s.service_description, "") as service_description,
        IFNULL(c.name, "") as contact_name,
        l.message as message,
        "'.$prefix.'" as peer_key
        '.$extra_columns.'
    FROM
        `'.$prefix.'_log` l
        LEFT JOIN `'.$prefix.'_host` h ON l.host_id = h.host_id
        LEFT JOIN `'.$prefix.'_service` s ON l.service_id = s.service_id
        LEFT JOIN `'.$prefix.'_contact` c ON l.contact_id = c.contact_id
    '.$where.'
    '.$orderby.'
    '.$limit.'
    ';
    confess($sql) if $sql =~ m/(ARRAY|HASH)/mx;

    ## no critic
    $ENV{'THRUK_DB_LAST_QUERY'} = $sql;
    ## use critic

    # logfiles into tmp file
    my($fh, $filename);
    if($options{'file'}) {
        ($fh, $filename) = tempfile();
        open($fh, '>', $filename) or die('open '.$filename.' failed: '.$!);
    }

    # add performance related debug output
    if(Thruk::Base->verbose >= 3) {
        _trace($sql);

        _trace("EXPLAIN:");
        _trace(_sql_debug("EXPLAIN\n".$sql, $dbh));

        my $debug_sql = "SHOW INDEXES FROM `".$prefix."_log`";
        _trace($debug_sql.":");
        _trace(_sql_debug($debug_sql, $dbh));
    }

    # queries with authorization
    my $data;
    if($auth_data->{'username'}) {
        my($contact,$strict,$authorized_for_all_services,$authorized_for_all_hosts,$authorized_for_system_information) = ($auth_data->{'username'},$auth_data->{'strict'},$auth_data->{'authorized_for_all_services'},$auth_data->{'authorized_for_all_hosts'},$auth_data->{'authorized_for_system_information'});
        my $sth = $dbh->prepare($sql);
        $sth->execute;

        my $hosts_lookup    = $self->_get_log_host_auth($dbh, $prefix, $contact);
        my $services_lookup = $self->_get_log_service_auth($dbh, $prefix, $contact);

        while(my $r = $sth->fetchrow_hashref()) {
            if($r->{'service_description'}) {
                if($authorized_for_all_services) {
                }
                elsif($strict) {
                    next if(!defined $services_lookup->{$r->{'host_name'}}->{$r->{'service_description'}});
                } else {
                    next if(!defined $hosts_lookup->{$r->{'host_name'}} && !defined $services_lookup->{$r->{'host_name'}}->{$r->{'service_description'}});
                }
            }
            elsif($r->{'host_name'}) {
                if($authorized_for_all_hosts) {
                } else {
                    next if !defined $hosts_lookup->{$r->{'host_name'}};
                }
            }
            else {
                next if !$authorized_for_system_information;
            }
            if($fh) {
                print $fh encode_utf8($r->{'message'}),"\n";
            } else {
                push @{$data}, $r;
            }
        }
    }
    else {
        if($fh) {
            my $sth = $dbh->prepare($sql);
            $sth->execute;
            while(my $r = $sth->fetchrow_arrayref()) {
                print $fh encode_utf8($r->[8]),"\n";
            }
        } else {
            $data = $dbh->selectall_arrayref($sql, { Slice => {} });
        }
    }

    my $total_size;
    if($pager_limit && scalar @{$data} >= $pager_limit) {
        # fetch real number of entries here
        my $sql = '
        SELECT
            count(*)
        FROM
            `'.$prefix.'_log` l
            LEFT JOIN `'.$prefix.'_host` h ON l.host_id = h.host_id
            LEFT JOIN `'.$prefix.'_service` s ON l.service_id = s.service_id
            LEFT JOIN `'.$prefix.'_contact` c ON l.contact_id = c.contact_id
        '.$where.'
        '.$limit.'
        ';
        my $counts = $dbh->selectall_arrayref($sql);
        $total_size = $counts->[0]->[0];
    }

    if($fh) {
        my $rc = Thruk::Utils::IO::close($fh, $filename);
        if(!$rc) {
            unlink($filename);
            confess("writing logs to $filename failed: $!");
        }
        return($filename, 'file');
    } else {
        return($data, ($sorted ? 'sorted' : ''), $total_size);
    }
}

##########################################################

=head2 get_timeperiods

=cut
sub get_timeperiods {
    confess("not implemented");
}

##########################################################

=head2 get_timeperiod_names

=cut
sub get_timeperiod_names {
    confess("not implemented");
}

##########################################################

=head2 get_commands

=cut
sub get_commands {
    confess("not implemented");
}

##########################################################

=head2 get_contacts

=cut
sub get_contacts {
    confess("not implemented");
}

##########################################################

=head2 get_contact_names

=cut
sub get_contact_names {
    confess("not implemented");
}

##########################################################

=head2 get_host_stats

=cut
sub get_host_stats {
    confess("not implemented");
}

##########################################################

=head2 get_host_totals_stats

  get_host_totals_stats

returns the host statistics used on the service/host details page

=cut

sub get_host_totals_stats {
    confess("not implemented");
}

##########################################################

=head2 get_host_less_stats

  get_host_less_stats

returns the host statistics but with less columns

=cut

sub get_host_less_stats {
    confess("not implemented");
}

##########################################################

=head2 get_service_stats

=cut
sub get_service_stats {
    confess("not implemented");
}

##########################################################

=head2 get_service_totals_stats

  get_service_totals_stats

returns the services statistics used on the service/host details page

=cut

sub get_service_totals_stats {
    confess("not implemented");
}

##########################################################

=head2 get_service_less_stats

  get_service_less_stats

returns the services statistics but with less columns

=cut

sub get_service_less_stats {
    confess("not implemented");
}

##########################################################

=head2 get_performance_stats

=cut
sub get_performance_stats {
    confess("not implemented");
}

##########################################################

=head2 get_extra_perf_stats

=cut
sub get_extra_perf_stats {
    confess("not implemented");
}

##########################################################

=head2 get_contact_totals_stats

=cut

sub get_contact_totals_stats {
    confess("not implemented");
}

##########################################################

=head2 set_verbose

  set_verbose

sets verbose mode for this backend and returns old value

=cut
sub set_verbose {
    my($self, $val) = @_;
    my $old = $self->{'verbose'};
    $self->{'verbose'} = $val;
    return($old);
}

##########################################################

=head2 renew_logcache

  renew_logcache

renew logcache

=cut
sub renew_logcache {
    return;
}

##########################################################

=head2 _add_peer_data

  _add_peer_data

add peer name, addr and key to result array

=cut
sub _add_peer_data {
    my($self, $data) = @_;
    for my $d (@{$data}) {
        $d->{'peer_name'} = $self->peer_name;
        $d->{'peer_addr'} = $self->peer_addr;
        $d->{'peer_key'}  = $self->peer_key;
    }
    return $data;
}

##########################################################

=head2 _get_filter

  _get_filter

return Mysql filter

=cut
sub _get_filter {
    my($self, $inp) = @_;
    my $auth_data = {};
    if($inp && ref $inp eq 'ARRAY') {
        for my $f (@{$inp}) {
            if(ref $f eq 'HASH' && $f->{'auth_filter'}) {
                $auth_data = $f->{'auth_filter'};
                $f = undef;
            }
        }
    }
    my $filter = $self->_get_subfilter($inp);
    if($filter and ref $filter) {
        $filter = '('.join(' AND ', @{$filter}).')';
    }
    $filter = " WHERE ".$filter if $filter;

    $filter =~ s/WHERE\ \(\((.*)\)\ AND\ \)/WHERE ($1)/gmx;
    $filter =~ s/\ AND\ \)/)/gmx;
    $filter =~ s/\(\ AND\ \(/((/gmx;
    $filter =~ s/AND\s+AND/AND/gmx;
    $filter = '' if $filter eq ' WHERE ';

    return($filter, $auth_data);
}

##########################################################

=head2 _get_subfilter

  _get_subfilter

return Mysql filter

=cut
sub _get_subfilter {
    my($self, $inp, $f) = @_;
    return '' unless defined $inp;
    if(ref $inp eq 'ARRAY') {
        # empty lists
        return '' if scalar @{$inp} == 0;

        # single array items will be stripped from array
        if(scalar @{$inp} == 1) {
            return $self->_get_subfilter($inp->[0]);
        }

        my $x   = 0;
        my $num = scalar @{$inp};
        my $filter = [];
        while($x < $num) {
            # [ 'key', { 'op' => 'value' } ]
            if(exists $inp->[$x+1] and ref $inp->[$x] eq '' and ref $inp->[$x+1] eq 'HASH') {
                my $key = $inp->[$x];
                my $val = $inp->[$x+1];
                if(!defined $key) {
                    $x=$x+1;
                    next;
                }
                push @{$filter}, $self->_get_subfilter({$key => $val});
                $x=$x+2;
                next;
            }
            # [ '-or', [ 'key' => 'value' ] ]
            if(exists $inp->[$x+1] and ref $inp->[$x] eq '' and ref $inp->[$x+1] eq 'ARRAY') {
                my $key = $inp->[$x];
                my $val = $inp->[$x+1];
                if(!defined $key) {
                    $x=$x+1;
                    next;
                }
                push @{$filter}, $self->_get_subfilter({$key => $val});
                $x=$x+2;
                next;
            }

            # [ 'key', 'value' ] => { 'key' => 'value' }
            if(exists $inp->[$x+1] and ref $inp->[$x] eq '' and ref $inp->[$x+1] eq '') {
                my $key = $inp->[$x];
                my $val = $inp->[$x+1];
                push @{$filter}, $self->_get_subfilter({$key => $val});
                $x=$x+2;
                next;
            }

            if(defined $inp->[$x]) {
                my $f =  $self->_get_subfilter($inp->[$x]);
                if($f and ref $f) {
                    $f= '('.join(' AND ', @{$f}).')';
                }
                push @{$filter}, $f;
            }
            $x++;
        }
        if(scalar @{$filter} == 1) {
            return $filter->[0];
        }
        return $filter;
    }
    if(ref $inp eq 'HASH') {
        # single hash elements with an operator
        if(scalar keys %{$inp} == 1) {
            my $k = [keys   %{$inp}]->[0];
            my $v = [values %{$inp}]->[0];
            ($k, $v) = $self->_replace_column_name($k, $v);
            if($k eq 'IS NULL')                     { return $v.' '.$k; }
            if($k eq 'IS NOT NULL')                 { return $v.' '.$k; }
            if($k eq '=')                           { return '= '._quote($v); }
            if($k eq '!=')                          { return '!= '._quote($v); }
            if($k eq '~')                           { return 'RLIKE '._quote_backslash(_quote(Thruk::Utils::clean_regex($v))); }
            if($k eq '~~')                          { return 'RLIKE '._quote_backslash(_quote(Thruk::Utils::clean_regex($v))); }
            if($k eq '!~~')                         { return 'NOT RLIKE '._quote_backslash(_quote(Thruk::Utils::clean_regex($v))); }
            if($k eq '>='  and ref $v eq 'ARRAY')   { confess("whuus") unless defined $f; return '= '.join(' OR '.$f.' = ', @{_quote($v)}); }
            if($k eq '!>=' and ref $v eq 'ARRAY')   { confess("whuus") unless defined $f; return '!= '.join(' OR '.$f.' != ', @{_quote($v)}); }
            if($k eq '!>=')                         { return '!= '._quote($v); }
            if($k eq '>=' and $v !~ m/^[\d\.]+$/mx) { return 'IN ('._quote($v).')'; }
            if($k eq '>=')                          { return '>= '._quote($v); }
            if($k eq '<=')                          { return '<= '._quote($v); }
            if($k eq '>')                           { return '> '._quote($v); }
            if($k eq '<')                           { return '< '._quote($v); }
            if($k eq '-or') {
                my $list = $self->_get_subfilter($v);
                if(ref $list) {
                    # remove empty elements
                    @{$list} = grep(!/^$/mx, @{$list});
                    for my $l (@{$list}) {
                        if(ref $l eq 'ARRAY') {
                            $l = '('.join(' AND ', @{$l}).')';
                        }
                    }
                    return('('.join(' OR ', @{$list}).')');
                }
                return $list;
            }
            if($k eq '-and') {
                my $list = $self->_get_subfilter($v);
                if(ref $list) {
                    @{$list} = grep(!/^$/mx, @{$list});
                    for my $l (@{$list}) {
                        if(ref $l eq 'ARRAY') {
                            $l = '('.join(' AND ', @{$l}).')';
                        }
                    }
                    return('('.join(' AND ', @{$list}).')');
                }
                return $list;
            }
            if(ref $v) {
                $v = $self->_get_subfilter($v, $k);
                if($v =~ m/\ OR\ $k\ /mx) {
                    return '('.$k.' '.$v.')';
                }
                return $k.' '.$v;
            }
            ($k, $v) = $self->_replace_column_name($k, $v);
            return $k.' = '._quote($v);
        }

        # multiple keys will be converted to list
        # { 'key' => 'v', 'key2' => v }
        my $list = [];
        for my $k (sort keys %{$inp}) {
            push @{$list}, {$k => $inp->{$k}};
        }
        return $self->_get_subfilter({'-and' => $list});
    }
    return $inp;
}

##########################################################
# replace column name with correct table column
sub _replace_column_name {
    my($self, $col, $val, $op) = @_;
    $op = '=' unless defined $op;
    if($col eq 'contact_name') {
        return("c.name", $val);
    }

    if($col eq 'service_description' || $col eq 'type') {
        if(ref $val eq 'HASH') {
            my @keys = keys %{$val};
            if(scalar @keys == 1) {
                $op = $keys[0];
                if($op eq '=' || $op eq '!=') {
                    $val = $val->{$op};
                }
            }
        }
        if(!defined $val) {
            if($op eq '=') {
                return("IS NULL", $col);
            }
            if($op eq '!=') {
                return("IS NOT NULL", $col);
            }
        }
    }

    # using ids makes mysql prefer index
    if($col eq 'host_name' && $self->{'query_meta'}->{'prefix'}) {
        # val can be scalar or hash
        if(ref $val eq 'HASH') {
            my @keys = keys %{$val};
            if(scalar @keys == 1) {
                my $op = $keys[0];
                $val = $val->{$op};
                my($col, $id) = $self->_replace_column_name($col, $val, $op);
                return($col, { $op => $id });
            }
            # no replacement possible
            return($col, $val);
        }
        $self->{'query_meta'}->{'host_lookup'} = _get_host_lookup($self->{'query_meta'}->{'dbh'},undef,$self->{'query_meta'}->{'prefix'}, 1) unless defined $self->{'query_meta'}->{'host_lookup'};
        my $id = $self->{'query_meta'}->{'host_lookup'}->{$val} // -1; # always replace host_name. Using the name leads to zero results but skips the index
        if($id == -1 && ($op ne '==' && $op ne '!=')) {
            return($col, $val); # in case of regex matches, fallback to host_name again
        }
        return('l.host_id', $id);
    }
    return($col, $val);
}

##########################################################
sub _quote {
    my($str) = @_;
    return "''" unless defined $str;
    if(ref $str eq 'ARRAY') {
        my $list = [];
        for my $v (@{$str}) {
            push @{$list}, _quote($v);
        }
        return $list;
    }
    if($str =~ m/^\-?(\d+|\d+\.\d+)$/mx) {
        return $str;
    }
    $str =~ s/\\\\/\\/gmx;
    $str =~ s/\'/\\'/gmx;
    $str =~ s/\"/\\"/gmx;
    $str =~ s/\x08/\\b/gmx;
    $str =~ s/\n/\\n/gmx;
    $str =~ s/\r/\\r/gmx;
    $str =~ s/\t/\\t/gmx;
    $str =~ s/\x1A/\\Z/gmx;
    return("'".$str."'");
}

##########################################################
sub _quote_backslash {
    return '' unless defined $_[0];
    $_[0] =~ s|\\|\\\\|gmx;
    return($_[0]);
}

##########################################################

=head2 get_logs_start_end

  get_logs_start_end

returns first and last logfile entry

=cut
sub get_logs_start_end {
    return(_get_logs_start_end(@_));
}

##########################################################

=head2 _get_logs_start_end

  _get_logs_start_end

returns the min/max timestamp for given logs

=cut
sub _get_logs_start_end {
    my($self, %options) = @_;
    my($start, $end);
    my $prefix = $options{'collection'} || $self->{'peer_config'}->{'peer_key'};
    $prefix    =~ s/^logs_//gmx;
    my $dbh  = $options{'dbh'} || $self->_dbh();
    return([$start, $end]) unless _tables_exist($dbh, $prefix);
    my $where = "";
    ($where) = $self->_get_filter($options{'filter'}) if $options{'filter'};
    my @data = @{$dbh->selectall_arrayref('SELECT MIN(l.time) as mi, MAX(l.time) as ma FROM `'.$prefix.'_log` l '.$where.' LIMIT 1', { Slice => {} })};
    $start   = $data[0]->{'mi'} if defined $data[0];
    $end     = $data[0]->{'ma'} if defined $data[0];
    return([$start, $end]);
}

##########################################################

=head2 get_existing_caches

  get_existing_caches

returns list of already created caches

=cut
sub get_existing_caches {
    my($self, $c) = @_;

    # use first logcache connection
    my $dbh;
    for my $peer ( @{ $c->db->get_peers() } ) {
        next unless $peer->{'logcache'};
        $peer->logcache->reconnect();
        $dbh = $peer->logcache->_dbh();
        last;
    }

    return unless $dbh;

    my @list;
    # check if our tables exist
    my @tables = @{$dbh->selectcol_arrayref('SHOW TABLES')};
    for my $table (@tables) {
        if($table =~ m/^(.*?)_log$/mx) {
            push @list, $1;
        }
    }

    return(\@list);
}

##########################################################

=head2 _log_stats

  _log_stats

gather log statistics

=cut

sub _log_stats {
    my($self, $c, $backends) = @_;

    $c->stats->profile(begin => "Mysql::_log_stats");

    ($backends) = $c->db->select_backends('get_logs') unless defined $backends;
    $backends  = Thruk::Base::list($backends);
    push @{$backends}, keys %{$c->stash->{'failed_backends'}} if $c->stash->{'failed_backends'};

    my @result;
    for my $key (@{$backends}) {
        my $peer = $c->db->get_peer_by_key($key);
        my $msg = "OK";
        my($index_size, $data_size, $items, $last_entry);
        my $status  = {};
        if(!$peer->{'logcache'}) {
            $msg = "logcache is disabled";
        } else {
            $peer->logcache->reconnect();
            my $dbh  = $peer->logcache->_dbh();
            my $res  = $dbh->selectall_hashref("SHOW TABLE STATUS LIKE '".$key."%'", 'Name');
            if(!defined $res->{$key.'_log'}) {
                $msg = "logcache not yet created";
            } else {
                $index_size = $res->{$key.'_log'}->{'Index_length'};
                $data_size  = $res->{$key.'_log'}->{'Data_length'};
                $items      = $res->{$key.'_log'}->{'Rows'};
                $status     = $dbh->selectall_hashref("SELECT name, value FROM `".$key."_status`", 'name');
                (undef, $last_entry) = @{$self->_get_logs_start_end(collection => $key, dbh => $dbh)};
                if($status->{'lock_mode'}->{'value'}) {
                    $msg = sprintf("running %s since %s (pid: %s)", $status->{'lock_mode'}->{'value'}, $status->{'last_update'}->{'value'} ? scalar localtime($status->{'last_update'}->{'value'}) : '?', $status->{'update_pid'}->{'value'}//'?');
                }
            }
        }
        push @result, {
            key              => $key,
            name             => $peer->{'name'},
            enabled          => $peer->{'logcache'} ? 1 : 0,
            index_size       => $index_size     // 0,
            data_size        => $data_size      // 0,
            items            => $items          // 0,
            cache_version    => $status->{'cache_version'}->{'value'}       // '',
            last_update      => $status->{'last_update'}->{'value'}         // '',
            last_reorder     => $status->{'last_reorder'}->{'value'}        // '',
            last_compact     => $status->{'last_compact'}->{'value'}        // '',
            reorder_duration => $status->{'reorder_duration'}->{'value'}    // '',
            update_duration  => $status->{'update_duration'}->{'value'}     // '',
            compact_duration => $status->{'compact_duration'}->{'value'}    // '',
            compact_till     => $status->{'compact_till'}->{'value'}        // '',
            last_entry       => $last_entry                                 // '',
            mode             => $status->{'lock_mode'}->{'value'}           // '',
            status           => $msg,
        };
    }

    $c->stats->profile(end => "Mysql::_log_stats");
    return @result if wantarray;
    return Thruk::Utils::text_table(
        keys => [['Backend', 'name'],
                 { name => 'Data Size',   key => 'data_size',  type => 'bytes', format => "%.1f" },
                 { name => 'Index Size',  key => 'index_size', type => 'bytes', format => "%.1f" },
                 ['Items', 'items'],
                 { name => 'Last Update', key => 'last_update', type => 'date', format => '%Y-%m-%d %H:%M:%S' },
                 { name => 'Last Item',   key => 'last_entry',  type => 'date', format => '%Y-%m-%d %H:%M:%S' },
                 { name => 'Status',      key => 'status' },
                ],
        data => \@result,
    );
}

##########################################################

=head2 _logcache_stats_types

  _logcache_stats_types

gather log type statistics

=cut

sub _logcache_stats_types {
    my($self, $c, $groupby, $backends) = @_;

    $c->stats->profile(begin => "Mysql::_logcache_stats_types: ".$groupby);

    ($backends) = $c->db->select_backends('get_logs') unless defined $backends;
    $backends  = Thruk::Base::list($backends);

    my @result;
    for my $key (@{$backends}) {
        my $peer = $c->db->get_peer_by_key($key);
        next unless $peer->{'logcache'};
        $peer->logcache->reconnect();
        my $dbh  = $peer->logcache->_dbh();
        my $res  = $dbh->selectall_hashref("SHOW TABLE STATUS LIKE '".$key."%'", 'Name');
        next unless defined $res->{$key.'_log'};
        my $types  = [values %{$dbh->selectall_hashref("SELECT IFNULL(".$groupby.", '') as $groupby, count(*) as total FROM `".$key."_log` GROUP BY ".$groupby, $groupby)}];
        $types     = [reverse sort { $a->{'total'} <=> $b->{'total'} } @{$types}];
        my $total = 0;
        for my $t (@{$types}) {
            $total += $t->{'total'};
        }
        for my $t (@{$types}) {
            $t->{'procent'} = 0;
            if($total > 0) {
                $t->{'procent'} = $t->{'total'} * 100 / $total;
            }
        }
        push @result, {
            key     => $key,
            name    => $peer->{'name'},
            types   => $types,
        };
    }

    $c->stats->profile(end => "Mysql::_logcache_stats_types: ".$groupby);
    return \@result;
}

##########################################################

=head2 _log_removeunused

  _log_removeunused

remove logcache tables from backends which do no longer exist

=cut

sub _log_removeunused {
    my($self, $c, $print_only) = @_;
    $c->stats->profile(begin => "Mysql::_log_removeunused");

    # use first peers logcache
    my $peer;
    for my $key (@{$c->stash->{'backends'}}) {
        $peer = $c->db->get_peer_by_key($key);
        last if $peer->{'logcache'};
    }
    return "no logcache configured?" unless(defined $peer and defined $peer->{'logcache'});

    $peer->logcache->reconnect();
    my $dbh  = $peer->logcache->_dbh();
    my $res  = $dbh->selectall_hashref("SHOW TABLE STATUS", 'Name');

    # gather backend ids
    my $backends = {};
    for my $tbl (keys %{$res}) {
        if($tbl =~ m/^(.*?)_(status|log)/mx) {
            $backends->{$1} = 1;
        }
    }

    # do not remove the ones still existing
    for my $key (@{$c->stash->{'backends'}}) {
        delete $backends->{$key};
    }
    if($print_only) {
        $c->stats->profile(end => "Mysql::_log_removeunused");
        return($backends);
    }

    my $removed = 0;
    my $tables  = 0;
    for my $key (keys %{$backends}) {
        for my $tbl (keys %{$res}) {
            next unless $tbl =~ m/^${key}_/mx;
            $tables++;
            $dbh->do("DROP TABLE `".$tbl."`");
        }
        $removed++;
    }
    $dbh->commit || confess $dbh->errstr;

    $c->stats->profile(end => "Mysql::_log_removeunused");
    return "no old tables found in logcache" if $removed == 0;

    return $removed." old backends removed (".$tables." tables) from logcache";
}

##########################################################

=head2 _log_check_inconsistency

  _log_check_inconsistency

report inconsistancies

=cut
sub _log_check_inconsistency {
    my($self, $c, $backends, $heal) = @_;

    $c->stats->profile(begin => "Mysql::_log_check_inconsistency()");

    if(!defined $backends) {
        ($backends) = $c->db->select_backends('get_logs');
    }
    $backends = Thruk::Base::list($backends);
    my @peer_keys;
    for my $key (@{$backends}) {
        my $peer   = $c->db->get_peer_by_key($key);
        next unless $peer->{'enabled'};
        push @peer_keys, $key;
    }

    my $errors = {};
    for my $key (@{$backends}) {
        my $prefix = $key;
        my $peer   = $c->db->get_peer_by_key($key);
        next unless $peer->{'enabled'};
        next unless $peer->{'logcache'};
        _debug("checking inconsistancies for site ".$peer->{'name'});

        $c->stats->profile(begin => "$key");
        $peer->logcache->reconnect();
        my $dbh = $peer->logcache->_dbh;

        my $sth = $dbh->prepare("select count(host_id) as count, host_name from `".$prefix."_host` group by host_name having count > 1");
        $sth->execute;
        my $num = 0;
        for my $r (@{$sth->fetchall_arrayref()}) {
            $num++;
            _debug("  - %s: got %d ids for host: '%s'", $peer->{'name'}, $r->[0], $r->[1]);
        }
        if($num > 0) {
            $errors->{$key} = "host_id table integrity broken (hint: run selfcheck with --heal to fix)";
            if($heal) {
                $errors->{$key} = "host_id table integrity broken, trying autofix";
                _info("[%s] trying to autofix host_id for backend %s", $peer->{'name'}, $prefix);
                _fix_multiple_host_ids($c, $prefix, $num);
            }
        }

        # cleanup connection
        eval {
            $peer->logcache->_disconnect();
        };

        $c->stats->profile(end => "$key");
    }

    $c->stats->profile(end => "Mysql::_log_check_inconsistency()");

    return $errors;
}

##########################################################
sub _fix_multiple_host_ids {
    my($c, $prefix, $total) = @_;

    my $peer = $c->db->get_peer_by_key($prefix);
    my $dbh  = $peer->logcache->_dbh;

    my $sth = $dbh->prepare("select count(host_id) as count, host_name from `".$prefix."_host` group by host_name having count > 1");
    $sth->execute;

    my $sp  = length("$total");
    my $num = 0;
    for my $r (@{$sth->fetchall_arrayref()}) {
        my $hostname = $r->[1];
        $num++;

        if($num%100 == 0) {
            $dbh->commit || confess $dbh->errstr;
        }

        _infos("[%0".$sp."d/%0".$sp."d] %s ", $num, $total, $hostname);

        my $sth2 = $dbh->prepare("select host_id from `".$prefix."_host` WHERE host_name = "._quote($hostname));
        $sth2->execute;
        my $keepId;
        for my $r (@{$sth2->fetchall_arrayref()}) {
            _infoc(".");
            my $id = $r->[0];
            if(!defined $keepId) {
                $keepId = $id;
                next;
            }
            $dbh->do("UPDATE `".$prefix."_log` SET host_id = ".$keepId." WHERE host_id = ".$id);
            $dbh->do("UPDATE `".$prefix."_service` SET host_id = ".$keepId." WHERE host_id = ".$id);
            $dbh->do("DELETE FROM `".$prefix."_contact_host_rel` WHERE host_id = ".$id);
            $dbh->do("DELETE FROM `".$prefix."_host` WHERE host_id = ".$id);
        }
        _info("OK");
    }
    $dbh->commit || confess $dbh->errstr;

    return;
}

##########################################################

=head2 _import_logs

  _import_logs

imports logs into Mysql

=cut

sub _import_logs {
    my($self, $c, $mode, $backends, $blocksize, $options) = @_;
    my $files = $options->{'files'} || [];
    $c->stats->profile(begin => "Mysql::_import_logs($mode)");

    my $forcestart;
    if($options->{'start'}) {
        $forcestart = time() - Thruk::Utils::expand_duration($options->{'start'});
    }

    my $backend_count = 0;
    my $log_count     = 0;

    if(!defined $backends) {
        ($backends) = $c->db->select_backends('get_logs');
    }
    $backends = Thruk::Base::list($backends);
    my @peer_keys;
    for my $key (@{$backends}) {
        my $peer   = $c->db->get_peer_by_key($key);
        next unless $peer->{'enabled'};
        push @peer_keys, $key;
    }

    if(scalar @peer_keys > 1 and scalar @{$files} > 0) {
        _error("you must specify a backend (-b) when importing files.");
        return(0, -1);
    }

    my $errors = [];
    for my $key (@{$backends}) {
        my $prefix = $key;
        my $peer   = $c->db->get_peer_by_key($key);
        next unless $peer->{'enabled'};
        next unless $peer->{'logcache'};
        $c->stats->profile(begin => "$key");
        $backend_count++;
        $peer->logcache->reconnect();
        my $dbh = $peer->logcache->_dbh;

        if($mode eq 'update') {
            $mode = 'import' if _update_logcache_version($c, $dbh, $prefix);
        }

        _info("running ".$mode." for site ".$peer->{'name'});

        # backends maybe down, we still want to continue updates
        eval {
            my $count;
            if($mode eq 'update' or $mode eq 'import') {
                $count = $peer->logcache->_update_logcache($c, $mode, $peer, $dbh, $prefix, $blocksize, $files, $forcestart);
            }
            elsif($mode eq 'clean') {
                my $tmp = $peer->logcache->_update_logcache($c, $mode, $peer, $dbh, $prefix, $blocksize, $files, $forcestart);
                $log_count = [0,0] unless ref $log_count eq 'ARRAY';
                if(ref $tmp eq 'ARRAY') {
                    $log_count->[0] += $tmp->[0];
                    $log_count->[1] += $tmp->[1];
                }
            }
            elsif($mode eq 'compact') {
                my $tmp = $peer->logcache->_update_logcache($c, $mode, $peer, $dbh, $prefix, $blocksize, $files, $forcestart, $options->{'force'});
                $log_count = [0,0] unless ref $log_count eq 'ARRAY';
                if(ref $tmp eq 'ARRAY') {
                    $log_count->[0] += $tmp->[0];
                    $log_count->[1] += $tmp->[1];
                }
            }
            elsif($mode eq 'drop') {
                $peer->logcache->_update_logcache($c, $mode, $peer, $dbh, $prefix, $blocksize, $files, $forcestart);
            }
            elsif($mode eq 'authupdate') {
                $count = $peer->logcache->_update_logcache_auth($c, $peer, $dbh, $prefix);
            }
            elsif($mode eq 'optimize') {
                $count = $peer->logcache->_update_logcache_optimize($c, $peer, $prefix, $options);
            } else {
                die("unknown mode: ".$mode."\n");
            }
            $log_count += $count if($count && $count > 0);
        };
        my $err = $@;
        if($err) {
            _debug($err);
            if($err =~ m/(.*\Qplease come back later\E)/mx) {
                _warn(sprintf("skipping %s, remote site is currently running an cache import, please try again later.", $peer->{'name'}));
                push @{$errors}, ""; # count it as failed
            } else {
                push @{$errors}, $err;
            }
        }

        # cleanup connection
        eval {
            $peer->logcache->_disconnect();
        };

        $c->stats->profile(end => "$key");
    }

    our $global_lock_created;
    if($global_lock_created) {
        unlink($c->config->{'tmp_path'}."/logcache_import.lock");
    }

    $c->stats->profile(end => "Mysql::_import_logs($mode)");
    return($backend_count, $log_count, $errors);
}

##########################################################
sub _update_logcache {
    my($self, $c, $mode, $peer, $dbh, $prefix, $blocksize, $files, $forcestart,$force) = @_;

    &timing_breakpoint('_update_logcache');
    unless(defined $blocksize) {
        $blocksize = 86400;
        if($mode eq 'clean') {
            $blocksize = Thruk::Utils::expand_duration($c->config->{'logcache_clean_duration'}) / 86400;
        }
        if($mode eq 'compact') {
            $blocksize = Thruk::Utils::expand_duration($c->config->{'logcache_compact_duration'}) / 86400;
        }
    }

    if($mode eq 'drop') {
        _drop_tables($dbh, $prefix);
        return;
    }

    # check tables
    _drop_tables($dbh, $prefix) if $mode eq 'import';
    my $fresh_created = 0;
    if(_create_tables_if_not_exist($dbh, $prefix)) {
        $fresh_created = 1;
    }

    return(-1) unless _check_lock($dbh, $prefix, $c, $mode);
    my $start = time();

    if($mode eq 'clean') {
        my $res = _update_logcache_clean($c, $dbh, $prefix, $blocksize);
        _finish_update($c, $dbh, $prefix, time() - $start);
        return($res);
    }
    if($mode eq 'compact') {
        my $res = $self->_update_logcache_compact($c, $dbh, $prefix, $blocksize, $force);
        _finish_update($c, $dbh, $prefix, time() - $start);
        return($res);
    }

    $mode = 'import' if $fresh_created;

    my $log_count = 0;
    eval {
        my $host_lookup    = _get_host_lookup(   $dbh,$peer,$prefix,               $mode eq 'import' ? 0 : 1);
        my $service_lookup = _get_service_lookup($dbh,$peer,$prefix, $host_lookup, $mode eq 'import' ? 0 : 1);
        my $contact_lookup = _get_contact_lookup($dbh,$peer,$prefix,               $mode eq 'import' ? 0 : 1);

        if(defined $files && scalar @{$files} > 0) {
            $log_count += $self->_import_logcache_from_file($mode,$dbh,$files,$host_lookup,$service_lookup,$prefix,$contact_lookup,$c);
        } else {
            $log_count += $self->_import_peer_logfiles($c,$mode,$peer,$blocksize,$dbh,$host_lookup,$service_lookup,$prefix,$contact_lookup,$forcestart);
        }

        if($mode eq 'import') {
            _debug2("updateing auth cache");
            $self->_update_logcache_auth($c, $peer, $dbh, $prefix);
        }
    };
    my $error = $@ || '';

    _finish_update($c, $dbh, $prefix, time() - $start) or $error .= $dbh->errstr;

    if($error) {
        my($short_err, undef) = Thruk::Utils::extract_connection_error($error);
        if(defined $short_err) {
            _debug($error);
            $error = $short_err;
        }
        _cronerror('logcache '.$mode.' failed: '._strip_line($error)); # don't fill the log with errors from cronjobs
        die($error);
    }

    return $log_count;
}

##########################################################
sub _finish_update {
    my($c, $dbh, $prefix, $duration) = @_;
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(1,'last_update',UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE value=UNIX_TIMESTAMP()");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(2,'update_pid',NULL) ON DUPLICATE KEY UPDATE value=NULL");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(6,'update_duration','".$duration."') ON DUPLICATE KEY UPDATE value='".$duration."'");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(10,'lock_mode','') ON DUPLICATE KEY UPDATE value=''");
    _release_write_locks($dbh) unless $c->config->{'logcache_pxc_strict_mode'};
    $dbh->commit || return;
    return 1;
}

##########################################################
# returns 1 if tables have been newly created or undef if already exist
sub _create_tables_if_not_exist {
    my($dbh, $prefix) = @_;

    return if _tables_exist($dbh, $prefix);

    _debug2("creating logcache tables");
    _create_tables($dbh, $prefix);
    return 1;
}

##########################################################
# returns 1 if logcache tables exist, undef if not
sub _tables_exist {
    my($dbh, $prefix) = @_;

    # check if our tables exist
    my @tables = @{$dbh->selectcol_arrayref('SHOW TABLES LIKE "'.$prefix.'\_%"')};
    if(scalar @tables >= 5) {
        return 1;
    }

    return;
}

##########################################################
sub _check_lock {
    my($dbh, $prefix, $c, $mode) = @_;

    # import locks all other operations
    return unless check_global_lock($c);

    # check if there is already a update / import running
    my $skip          = 0;
    eval {
        $dbh->do('LOCK TABLES `'.$prefix.'_status` READ') unless $c->config->{'logcache_pxc_strict_mode'};
        my @pids = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 2 LIMIT 1')};
        if(scalar @pids > 0 and $pids[0]) {
            if(kill(0, $pids[0])) {
                _info("WARNING: logcache update already running with pid ".$pids[0]);
                $skip = 1;
            }
        }
    };
    $dbh->do('UNLOCK TABLES') unless $c->config->{'logcache_pxc_strict_mode'};
    if($@) {
        _debug($@);
        return;
    }
    if($skip) {
        return;
    }

    $dbh->do('LOCK TABLES `'.$prefix.'_status` WRITE') unless $c->config->{'logcache_pxc_strict_mode'};
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(1,'last_update',UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE value=UNIX_TIMESTAMP()");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(2,'update_pid',".$$.") ON DUPLICATE KEY UPDATE value=".$$);
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(10,'lock_mode','".$mode."') ON DUPLICATE KEY UPDATE value='".$mode."'");
    $dbh->commit || confess $dbh->errstr;
    $dbh->do('UNLOCK TABLES') unless $c->config->{'logcache_pxc_strict_mode'};

    if(($mode eq 'import' || $ENV{'THRUK_CRON'}) && !-f $c->config->{'tmp_path'}."/logcache_import.lock") {
        our $global_lock_created = 1;
        Thruk::Utils::IO::write($c->config->{'tmp_path'}."/logcache_import.lock", $$);
    }

    return(1);
}

##########################################################

=head2 check_global_lock

  check_global_lock($c)

returns true if no global lock exists

=cut

sub check_global_lock {
    my($c) = @_;
    # import locks all other operations
    my $pid = Thruk::Utils::IO::saferead($c->config->{'tmp_path'}."/logcache_import.lock");
    if($pid && $pid != $$) {
        if($pid && kill(0, $pid)) {
            _info(sprintf("WARNING: logcache import currently running with pid %d", $pid));
            return;
        }
        _warn("WARNING: removing stale lock file: ".$c->config->{'tmp_path'}."/logcache_import.lock");
        unlink($c->config->{'tmp_path'}."/logcache_import.lock");
    }
    return(1);
}

##########################################################
sub _update_logcache_version {
    my($c, $dbh, $prefix) = @_;

    return 1 unless _tables_exist($dbh, $prefix);

    my $cache_version = 1;
    my @versions = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 4 LIMIT 1')};
    if(scalar @versions > 0 and $versions[0]) {
        $cache_version = $versions[0];
    }

    if($cache_version < $Thruk::Backend::Provider::Mysql::cache_version) {
        # only log message if not importing already
        my $msg = 'logcache version too old: '.$cache_version.', recreating with version '.$Thruk::Backend::Provider::Mysql::cache_version.'...';
        _warn($msg);
        return 1;
    }

    return;
}

##########################################################
sub _update_logcache_clean {
    my($c, $dbh, $prefix, $blocksize) = @_;

    if($blocksize =~ m/^\d+[a-z]{1}/mx) {
        # blocksize is in days
        $blocksize = int(Thruk::Utils::expand_duration($blocksize) / 86400);
    }

    _check_index($c, $dbh, $prefix);

    my $start = time() - ($blocksize * 86400);
    _debug2("cleaning logs older than: ", scalar localtime $start);
    my $plugin_ref_count = 0;
    my $log_count = $dbh->do("DELETE FROM `".$prefix."_log` WHERE time < ".$start);
    return([$log_count, $plugin_ref_count]) if $log_count == 0;

    $dbh->commit || confess $dbh->errstr;
    return([$log_count, $plugin_ref_count]);
}

##########################################################
sub _update_logcache_compact {
    my($self, $c, $dbh, $prefix, $blocksize, $force) = @_;
    my $log_count = 0;
    my $log_clear = 0;

    # since we usually backtrack 4 days in reports, use 3days plus 2 extra hours to compensate timeshifts to compact state changes
    my $offset = 74*3600;

    if($blocksize =~ m/^\d+[a-z]{1}/mx) {
        # blocksize is in days
        $blocksize = int(Thruk::Utils::expand_duration($blocksize) / 86400);
    }
    my $t1     = time();
    my $end  = Thruk::Utils::DateTime::start_of_day(time() - ($blocksize * 86400));
    _debug("compacting logs older than: ".(scalar localtime $end));
    my $status = $dbh->selectall_hashref("SELECT name, value FROM `".$prefix."_status`", 'name');
    my $start  = $status->{'compact_till'}->{'value'};
    if(!$start || $force) {
        my($mstart) = @{$self->_get_logs_start_end()};
        $start = $mstart;
    }
    if(!$start) {
        return([$log_count, $log_clear]);
    }

    _check_index($c, $dbh, $prefix);

    my $import_filter = _get_exclude_filter($c->config);
    my $current = Thruk::Utils::DateTime::start_of_day($start - $offset);
    while(1) {
        if($current >= $end) {
            last;
        }

        _infos("compacting ".(scalar localtime $current));
        my $next = Thruk::Utils::DateTime::start_of_day($current + $offset);

        my $sth = $dbh->prepare("SELECT log_id, class, type, state, state_type, host_id, service_id, message FROM `".$prefix."_log` WHERE time >= $current and time < $next");
        $sth->execute;
        _infoc(': ');
        my $processed = 0;
        my $removed = 0;
        my @delete;
        my $alerts = {};
        for my $l (@{$sth->fetchall_arrayref({})}) {
            $processed++;
            _infoc('.') if $processed%10000 == 0;
            if($removed%10000 == 0 && scalar @delete > 0) {
                $dbh->do("DELETE FROM `".$prefix."_log` WHERE log_id IN (".join(",", @delete).")");
                $dbh->commit || confess $dbh->errstr;
                $log_clear += scalar @delete;
                @delete = ();
            }
            if(_is_compactable($l, $alerts, $import_filter)) {
                $removed++;
                push @delete, $l->{'log_id'};
            }
        }

        _info(sprintf("%d of %d removed (%.1f%%). done", $removed, $processed, $removed > 0 ? (($removed / $processed) * 100) : 0));
        $current  = $next;
        $log_count += $processed;
        $log_clear += $removed;

        if(scalar @delete > 0) {
            $dbh->do("DELETE FROM `".$prefix."_log` WHERE log_id IN (".join(",", @delete).")");
            $dbh->commit || confess $dbh->errstr;
        }

        $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(9,'compact_till','".$end."') ON DUPLICATE KEY UPDATE value='".$next."'");
        $dbh->commit || confess $dbh->errstr;
    }

    my $duration = time() - $t1;
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(7,'last_compact',UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE value=UNIX_TIMESTAMP()");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(8,'compact_duration','".$duration."') ON DUPLICATE KEY UPDATE value='".$duration."'");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(9,'compact_till','".$end."') ON DUPLICATE KEY UPDATE value='".$end."'");

    $dbh->commit || confess $dbh->errstr;
    return([$log_count, $log_clear]);
}

##########################################################
# returns true if log entry can be removed during compact
sub _is_compactable {
    my($l, $alertstore, $excludepattern) = @_;
    return 1 if($excludepattern && $l->{'message'} =~ $excludepattern);

    if($l->{'class'} == 2 || $l->{'class'} == 3 || $l->{'class'} == 5 || $l->{'class'} == 6) {
        # keep program, notifications, external commands, timeperiod transitions
        return;
    }
    elsif($l->{'class'} == 1) {
        if($l->{'type'} eq 'HOST DOWNTIME ALERT' || $l->{'type'} eq 'SERVICE DOWNTIME ALERT') {
            # keep downtimes
            return;
        }
        # remove duplicate alerts
        my $uniq = sprintf("%s;%s", $l->{'state_type'}//'', $l->{'state'}//'');
        if($l->{'type'} eq 'SERVICE ALERT' || $l->{'type'} eq 'CURRENT SERVICE STATE' || $l->{'type'} eq 'INITIAL SERVICE STATE') {
            my $host_id    = $l->{'host_id'} // $l->{'host_name'} // '';
            my $service_id = $l->{'service_id'} // $l->{'service_description'} // '';
            my $chk = $alertstore->{'svc'}->{$host_id}->{$service_id};
            if(!$chk || $chk ne $uniq) {
                $alertstore->{'svc'}->{$host_id}->{$service_id} = $uniq;
                return;
            }
        }
        elsif($l->{'type'} eq 'HOST ALERT' || $l->{'type'} eq 'CURRENT HOST STATE' || $l->{'type'} eq 'INITIAL HOST STATE') {
            my $host_id = $l->{'host_id'} // $l->{'host_name'};
            my $chk     = $alertstore->{'hst'}->{$host_id};
            if(!$chk || $chk ne $uniq) {
                $alertstore->{'hst'}->{$host_id} = $uniq;
                return;
            }
        }
    }
    return 1;
}

##########################################################
sub _update_logcache_auth {
    #my($self, $c, $peer, $dbh, $prefix) = @_;
    my($self, undef, $peer, $dbh, $prefix) = @_;

    # bad idea, this moves contact ids which are also used in notifications log entries
    #$dbh->do("TRUNCATE TABLE `".$prefix."_contact`");
    my $contact_lookup = _get_contact_lookup($dbh,$peer,$prefix);
    my $host_lookup    = _get_host_lookup($dbh,$peer,$prefix);
    my $service_lookup = _get_service_lookup($dbh,$peer,$prefix,$host_lookup);

    # update hosts
    my($hosts) = $peer->{'class'}->get_hosts(columns => [qw/name contacts/]);
    _debugs("hosts: ");
    my $stm = "INSERT INTO `".$prefix."_contact_host_rel` (contact_id, host_id) VALUES";
    $dbh->do("TRUNCATE TABLE `".$prefix."_contact_host_rel`");
    my $count = 0;
    for my $host (@{$hosts}) {
        my $host_id = &_host_lookup($host_lookup, $host->{'name'}, $dbh, $prefix);
        my @values;
        for my $contact (@{Thruk::Base::array_uniq($host->{'contacts'})}) {
            my $contact_id = &_contact_lookup($contact_lookup, $contact, $dbh, $prefix);
            push @values, '('.$contact_id.','.$host_id.')';
        }
        $dbh->do($stm.join(',', @values)) if scalar @values > 0;
        $count++;
        _debugc(".") if $count%100 == 0;
    }
    _debug("done");

    # update services
    _debugs("services: ");
    $dbh->do("TRUNCATE TABLE `".$prefix."_contact_service_rel`");
    $stm = "INSERT INTO `".$prefix."_contact_service_rel` (contact_id, service_id) VALUES";
    my($services) = $peer->{'class'}->get_services(columns => [qw/host_name description contacts/]);
    $count = 0;
    for my $service (@{$services}) {
        my $service_id = &_service_lookup($service_lookup, $host_lookup, $service->{'host_name'}, $service->{'description'}, $dbh, $prefix);
        next unless $service_id;
        my @values;
        for my $contact (@{Thruk::Base::array_uniq($service->{'contacts'})}) {
            my $contact_id = &_contact_lookup($contact_lookup, $contact, $dbh, $prefix);
            push @values, '('.$contact_id.','.$service_id.')';
        }
        $dbh->do($stm.join(',', @values)) if scalar @values > 0;
        $count++;
        _debugc(".") if $count%1000 == 0;
    }

    _debug("done");

    $dbh->commit || confess $dbh->errstr;

    return(scalar @{$hosts} + scalar @{$services});
}

##########################################################
sub _update_logcache_optimize {
    my($self, $c, $peer, $prefix, $options) = @_;

    my $dbh = $peer->logcache->_dbh;
    return(-1) unless _tables_exist($dbh, $prefix);

    # update sort order / optimize every day
    my @times = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 3 LIMIT 1')};
    if(!$options->{'force'} && scalar @times > 0 && $times[0] && $times[0] > time()-86400) {
        _info("no optimize necessary, last optimize: ".(scalar localtime $times[0]).", use -f to force");
        return(-1);
    }
    my $start = time();

    # start logcache self heal
    $self->_log_check_inconsistency($c, [$prefix], 1);

    my $disk_space_ok = $self->_check_db_fs($c, $peer, $prefix);
    if($disk_space_ok) {
        my $t1 = [gettimeofday];
        _infos("update %s logs table order...", $prefix);
        $dbh = $peer->logcache->_dbh; # reconnect
        $dbh->do("ALTER TABLE `".$prefix."_log` ORDER BY time");
        $dbh->commit || confess $dbh->errstr;
        _info("done. (duration: %s)", Thruk::Utils::Filter::duration(tv_interval($t1), 6));
    }

    unless($c->config->{'logcache_pxc_strict_mode'}) {
        # repair / optimize tables
        $dbh = $peer->logcache->_dbh; # reconnect

        _debug("optimizing / repairing tables");
        for my $table (@Thruk::Backend::Provider::Mysql::tables) {
            my $t1 = [gettimeofday];
            _infos('maintain table %20s...', $table);
            _infos(', check...');
            my $res = $dbh->selectall_arrayref("CHECK TABLE `".$prefix."_".$table.'`', { Slice => {} });
            if(!$res || !$res->[0] || $res->[0]->{"Msg_text"} ne 'OK') {
                _infos('running repair, check returned: %s', $res->[0]->{"Msg_text"});
                $dbh->do("REPAIR TABLE `".$prefix."_".$table.'`');
            }
            if($disk_space_ok && $table ne 'log') { # log table is optimized by the alter alter tyle order by... already
                _infos(', optimize...');
                $dbh->do("OPTIMIZE TABLE `".$prefix."_".$table.'`');
            }
            _infos(', analyze...');
            $dbh->do("ANALYZE TABLE `".$prefix."_".$table.'`');
            _info("OK. (duration: %s)", Thruk::Utils::Filter::duration(tv_interval($t1), 6));
        }
    }

    $dbh->commit || confess $dbh->errstr;
    my $duration = time() - $start;
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(3,'last_reorder',UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE value=UNIX_TIMESTAMP()");
    $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(5,'reorder_duration','".$duration."') ON DUPLICATE KEY UPDATE value='".$duration."'");
    $dbh->commit || confess $dbh->errstr;
    return(-1);
}

##########################################################
sub _get_host_lookup {
    my($dbh,$peer,$prefix, $noupdate) = @_;

    my $sth = $dbh->prepare("SELECT host_id, host_name FROM `".$prefix."_host`");
    $sth->execute;
    my $hosts_lookup = {};
    for my $r (@{$sth->fetchall_arrayref()}) { $hosts_lookup->{$r->[1]} = $r->[0]; }
    return $hosts_lookup if $noupdate;

    my($hosts) = $peer->{'class'}->get_hosts(columns => [qw/name/]);
    my $stm = "INSERT INTO `".$prefix."_host` (host_name) VALUES";
    my @values;
    for my $h (@{$hosts}) {
        next if defined $hosts_lookup->{$h->{'name'}};
        push @values, '('.$dbh->quote($h->{'name'}).')';
    }
    if(scalar @values > 0) {
        for my $chunk (@{Thruk::Utils::array_chunk_fixed_size(\@values, 50)}) {
            $dbh->do($stm.join(',', @{$chunk}));
            $sth->execute;
        }
        for my $r (@{$sth->fetchall_arrayref()}) { $hosts_lookup->{$r->[1]} = $r->[0]; }
    }
    return $hosts_lookup;
}


##########################################################
sub _get_service_lookup {
    my($dbh,$peer,$prefix,$hosts_lookup,$noupdate, $auto_increments, $foreign_key_stash) = @_;

    my $sth = $dbh->prepare("SELECT s.service_id, h.host_name, s.service_description FROM `".$prefix."_service` s, `".$prefix."_host` h WHERE s.host_id = h.host_id");
    $sth->execute;
    my $services_lookup = {};
    for my $r (@{$sth->fetchall_arrayref()}) { $services_lookup->{$r->[1]}->{$r->[2]} = $r->[0]; }
    return $services_lookup if $noupdate;

    my($services) = $peer->{'class'}->get_services(columns => [qw/host_name description/]);
    my $stm = "INSERT INTO `".$prefix."_service` (host_id, service_description) VALUES";
    my @values;
    for my $s (@{$services}) {
        next if defined $services_lookup->{$s->{'host_name'}}->{$s->{'description'}};
        my $host_id = &_host_lookup($hosts_lookup, $s->{'host_name'}, $dbh, $prefix, $auto_increments, $foreign_key_stash);
        push @values, '('.$host_id.','.$dbh->quote($s->{'description'}).')';
    }
    if(scalar @values > 0) {
        for my $chunk (@{Thruk::Utils::array_chunk_fixed_size(\@values, 50)}) {
            $dbh->do($stm.join(',', @{$chunk}));
            $sth->execute;
        }
        for my $r (@{$sth->fetchall_arrayref()}) { $services_lookup->{$r->[1]}->{$r->[2]} = $r->[0]; }
    }
    return $services_lookup;
}

##########################################################
sub _get_contact_lookup {
    my($dbh,$peer,$prefix,$noupdate) = @_;

    my $sth = $dbh->prepare("SELECT contact_id, name FROM `".$prefix."_contact`");
    $sth->execute;
    my $contact_lookup = {};
    for my $r (@{$sth->fetchall_arrayref()}) { $contact_lookup->{$r->[1]} = $r->[0]; }
    return $contact_lookup if $noupdate;

    my($contacts) = $peer->{'class'}->get_contacts(columns => [qw/name/]);
    my $stm = "INSERT INTO `".$prefix."_contact` (name) VALUES";
    my @values;
    for my $c (@{$contacts}) {
        next if defined $contact_lookup->{$c->{'name'}};
        push @values, '('.$dbh->quote($c->{'name'}).')';
    }
    if(scalar @values > 0) {
        for my $chunk (@{Thruk::Utils::array_chunk_fixed_size(\@values, 50)}) {
            $dbh->do($stm.join(',', @{$chunk}));
            $sth->execute;
        }
        for my $r (@{$sth->fetchall_arrayref()}) { $contact_lookup->{$r->[1]} = $r->[0]; }
    }
    return $contact_lookup;
}

##########################################################
sub _host_lookup {
    my($host_lookup, $host_name, $dbh, $prefix, $auto_increments, $foreign_key_stash) = @_;
    return unless $host_name;

    my $id = $host_lookup->{$host_name};
    return $id if $id;

    if($auto_increments) {
        $id = $auto_increments->{$prefix.'_host'}->{'AUTO_INCREMENT'}++;
        push @{$foreign_key_stash->{'host'}}, '('.$id.', '.$dbh->quote($host_name).')';
        $host_lookup->{$host_name} = $id;
        return $id;
    }

    $dbh->do("INSERT INTO `".$prefix."_host` (host_name) VALUES(".$dbh->quote($host_name).")");
    $id = $dbh->last_insert_id(undef, undef, undef, undef);
    $host_lookup->{$host_name} = $id;

    return $id;
}

##########################################################
sub _get_log_host_auth {
    my($self,$dbh, $prefix, $contact) = @_;
    my @hosts = @{$dbh->selectall_arrayref("SELECT h.host_name FROM `".$prefix."_host` h, `".$prefix."_contact_host_rel` chr, `".$prefix."_contact` c WHERE h.host_id = chr.host_id AND c.contact_id = chr.contact_id AND c.name = ".$dbh->quote($contact))};
    my $hosts_lookup = {};
    for my $h (@hosts) { $hosts_lookup->{$h->[0]} = 1; }
    return $hosts_lookup;
}

##########################################################
sub _get_log_service_auth {
    my($self,$dbh, $prefix, $contact) = @_;

    # Select all Services where the host is allowed by contact
    my $sql1 = "SELECT h.host_name, s.service_description
               FROM
                 `".$prefix."_service` s,
                 `".$prefix."_host` h,
                 `".$prefix."_contact_host_rel` chr,
                 `".$prefix."_contact` c1,
                 `".$prefix."_contact_service_rel` csr
               WHERE
                 s.host_id = h.host_id
                 AND h.host_id = chr.host_id
                 AND c1.contact_id = chr.contact_id
                 AND s.service_id = csr.service_id
                 AND c1.name = ".$dbh->quote($contact)
               ;
    # Select all Services which are directly allowed by contact
    my $sql2 = "SELECT h.host_name, s.service_description
               FROM
                 `".$prefix."_service` s,
                 `".$prefix."_host` h,
                 `".$prefix."_contact_host_rel` chr,
                 `".$prefix."_contact` c1,
                 `".$prefix."_contact_service_rel` csr
               WHERE
                 s.host_id = h.host_id
                 AND h.host_id = chr.host_id
                 AND c1.contact_id = csr.contact_id
                 AND s.service_id = csr.service_id
                 AND c1.name = ".$dbh->quote($contact)
                ;
    my $services1        = $dbh->selectall_arrayref($sql1);
    my $services2        = $dbh->selectall_arrayref($sql2);
    # Make them unique
    my $services_lookup = {};
    for my $s (@{$services1}) { $services_lookup->{$s->[0]}->{$s->[1]} = 1; }
    for my $s (@{$services2}) { $services_lookup->{$s->[0]}->{$s->[1]} = 1; }
    return $services_lookup;
}

##########################################################
sub _service_lookup {
    my($service_lookup, $host_lookup, $host_name, $service_description, $dbh, $prefix, $host_id, $auto_increments, $foreign_key_stash) = @_;
    return unless $service_description;
    return unless $host_name;

    my $id = $service_lookup->{$host_name}->{$service_description};
    return $id if $id;

    $host_id = &_host_lookup($host_lookup, $host_name, $dbh, $prefix, $auto_increments, $foreign_key_stash) unless $host_id;

    if($auto_increments) {
        $id = $auto_increments->{$prefix.'_service'}->{'AUTO_INCREMENT'}++;
        push @{$foreign_key_stash->{'service'}}, '('.$id.', '.$host_id.','.$dbh->quote($service_description).')';
        $service_lookup->{$host_name}->{$service_description} = $id;
        return $id;
    }

    $dbh->do("INSERT INTO `".$prefix."_service` (host_id, service_description) VALUES(".$host_id.", ".$dbh->quote($service_description).")");
    $id = $dbh->last_insert_id(undef, undef, undef, undef);
    $service_lookup->{$host_name}->{$service_description} = $id;

    return $id;
}

##########################################################
sub _contact_lookup {
    my($contact_lookup, $contact_name, $dbh, $prefix, $auto_increments, $foreign_key_stash) = @_;
    return unless $contact_name;

    my $id = $contact_lookup->{$contact_name};
    return $id if $id;

    if($auto_increments) {
        $id = $auto_increments->{$prefix.'_contact'}->{'AUTO_INCREMENT'}++;
        push @{$foreign_key_stash->{'contact'}}, '('.$id.', '.$dbh->quote($contact_name).')';
        $contact_lookup->{$contact_name} = $id;
        return $id;
    }

    $dbh->do("INSERT INTO `".$prefix."_contact` (name) VALUES(".$dbh->quote($contact_name).")");
    $id = $dbh->last_insert_id(undef, undef, undef, undef);
    $contact_lookup->{$contact_name} = $id;

    return $id;
}

##########################################################
sub _fill_lookup_logs {
    my($self,$prefix,$start,$end) = @_;
    my $lookup = {};
    my($mlogs) = $self->get_logs(
                                filter  => [{ '-and' => [
                                                        { time => { '>=' => $start } },
                                                        { time => { '<=' => $end } },
                                           ]}],
                                collection => $prefix,
                              );
    for my $l (@{$mlogs}) {
        next unless defined $l->{'message'};
        $lookup->{$l->{'message'}} = 1;
    }

    return $lookup;
}

##########################################################
sub _import_peer_logfiles {
    my($self,$c,$mode,$peer,$blocksize,$dbh,$host_lookup,$service_lookup,$prefix,$contact_lookup,$forcestart) = @_;

    return(-1) unless _tables_exist($dbh, $prefix);

    if($blocksize > 86400) {
        _warn("logcache blocksize too long, reduced to 1 day");
        $blocksize = 86400;
    }

    # get start / end timestamp
    my($mstart, $mend);
    if($mode eq 'update' && !$forcestart) {
        $c->stats->profile(begin => "get last mysql timestamp");
        # get last timestamp from Mysql
        ($mstart, $mend) = @{$peer->logcache->get_logs_start_end(collection => $prefix)};
        _debug("last entry from logcache: ".(scalar localtime $mend)) if $mend;
        $c->stats->profile(end => "get last mysql timestamp");
    }

    my $log_count = 0;
    my($start, $end);
    if($forcestart) {
        $start = $forcestart;
    } elsif($mend) {
        $start = $mend;
    } else {
        if($mode eq 'import') {
            # it does not make sense to import more than we would clean immediatly again
            $mend = Thruk::Utils::get_expanded_start_date($c, $c->config->{'logcache_clean_duration'});
        }
        # fetching logs without any filter is a terrible bad idea
        $c->stats->profile(begin => "get livestatus timestamp no filter");
        ($start, $end) = Thruk::Backend::Provider::Base::get_logs_start_end_no_filter($peer->{'class'}, $mend);
        if(defined $mend && $start < $mend) {
            $start = $mend;
        }
        $c->stats->profile(end => "get livestatus timestamp no filter");
    }
    if(!$start) {
        die("something went wrong, cannot get start from logfiles (".(defined $start ? $start : "undef").")\nIf this is an Icinga2 please have a look at: https://thruk.org/documentation/logfile-cache.html#icinga-2 for a workaround.\n");
    }

    _info("importing from ".(scalar localtime $start));
    _info("until latest entry in logfile: ".(scalar localtime $end)) if $end;
    my $time = $start;
    $end = time() unless $end;

    # add import filter again, even if it should have been filtered in the logs query already, but it seems like not all backends handle them correctly
    my $import_filter = [];
    for my $f (@{Thruk::Base::list($c->config->{'logcache_import_exclude'})}) {
        push @{$import_filter}, { message => { '!~~' => $f } }
    }
    if($mode eq 'import') {
        $dbh->do('SET foreign_key_checks = 0');
        $dbh->do('SET unique_checks = 0');
        $dbh->do('ALTER TABLE `'.$prefix.'_log` DISABLE KEYS');
    }
    my $compact_start_data = Thruk::Utils::get_expanded_start_date($c, $c->config->{'logcache_compact_duration'});
    my $alertstore = {};
    my $last_day = "";
    my $consecutive_errors = 0;
    my $good               = 0;
    my $err;

    my @columns = qw/class time type state host_name service_description message state_type contact_name/;
    my $reordered = 0;
    while($time <= $end) {
        my $stime = scalar localtime $time;
        $c->stats->profile(begin => $stime);
        my $duplicate_lookup = {};
        _infos(scalar localtime $time);

        my $today = POSIX::strftime("%Y-%m-%d", localtime($time));
        if($last_day ne $today) {
            $alertstore = {};
            $last_day = $today;
        }
        my $block_end = $time + $blocksize - 1;
        # round block end by days to avoid requesting multiple log files
        my $endday = POSIX::strftime("%Y-%m-%d", localtime($block_end));
        if($today ne $endday) {
            $block_end = Thruk::Utils::parse_date_string($endday." 00:00:00") - 1;
        }

        my $import_compacted = 0;
        if(($block_end) < $compact_start_data) {
            $import_compacted = 1;
        }

        my $logs = [];
        my $file = $peer->{'class'}->{'fetch_command'} ? 1 : undef;
        &timing_breakpoint('_get_logs');
        eval {
            # get logs from peer
            ($logs) = $peer->{'class'}->get_logs(nocache => 1,
                                                 filter  => [{ '-and' => [
                                                                    { time => { '>=' => $time } },
                                                                    { time => { '<=' => $block_end } },
                                                            ]}, @{$import_filter} ],
                                                 columns => \@columns,
                                                 file => $file,
                                                );
            &timing_breakpoint('_get_logs done');
            if($mode eq 'update') {
                # get already stored logs to filter duplicates
                $duplicate_lookup = $self->_fill_lookup_logs($prefix,$time,$block_end);
                &timing_breakpoint('_fill_lookup_logs_logs done');
            }
            _infoc(":");
        };
        if($@) {
            $err = $@;
            chomp($err);
            if($mode eq 'import') {
                die($err);
            } else {
                _debug($err);
                my $short;
                ($short, $err) = Thruk::Utils::extract_connection_error($err);
                $err = $short if $short;
                _infoc(" %s", $err);
                $consecutive_errors++;
                if($consecutive_errors >= 3) {
                    die("failed to update 3 times in a row, bailing out: ".$err);
                }
            }
        } else {
            $consecutive_errors = 0;
            $good++;
        }

        $time = $block_end + 1;

        if($file) {
            $file = $logs;
            $log_count += $self->_import_logcache_from_file($mode,$dbh,[$file],$host_lookup,$service_lookup,$prefix,$contact_lookup,$c, $import_compacted, $alertstore);
        } else {
            $log_count += $self->_insert_logs($dbh,$mode,$logs,$host_lookup,$service_lookup,$duplicate_lookup,$prefix,$contact_lookup,$c, undef, $import_compacted, $alertstore);
            $reordered = 1;
        }

        $c->stats->profile(end => $stime);
    }

	# all blocks failed
    if($err && $good == 0) {
        die(_strip_line($err));
    }

    if($mode eq 'import') {
        _infos("creating index...");
        _enable_index($dbh, $prefix);
        if($reordered) {
            $dbh->do("INSERT INTO `".$prefix."_status` (status_id,name,value) VALUES(3,'last_reorder',UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE value=UNIX_TIMESTAMP()");
        }
        _info("done");
        &timing_breakpoint('_import_peer_logfiles enable index done');
    }

    return $log_count;
}

##########################################################
sub _enable_index {
    my($dbh,$prefix) = @_;
    &timing_breakpoint('_import_peer_logfiles enable index');
    $dbh->do('SET foreign_key_checks = 1');
    $dbh->do('SET unique_checks = 1');
    $dbh->do('ALTER TABLE `'.$prefix.'_log` ENABLE KEYS');
    for my $table (@Thruk::Backend::Provider::Mysql::tables) {
        $dbh->do("ANALYZE TABLE `".$prefix."_".$table.'`');
        $dbh->do("CHECK TABLE `".$prefix."_".$table.'`');
    }
    return;
}

##########################################################
sub _get_exclude_filter {
    my($config) = @_;
    if(scalar @{Thruk::Base::list($config->{'logcache_import_exclude'})} == 0) {
        return;
    }
    my $import_filter;
    my $f = join('|', @{Thruk::Base::list($config->{'logcache_import_exclude'})});
    ## no critic
    $import_filter = qr/($f)/i;
    ## use critic
    return($import_filter);
}

##########################################################
sub _import_logcache_from_file {
    my($self,$mode,$dbh,$files,$host_lookup,$service_lookup,$prefix,$contact_lookup, $c, $import_compacted, $alertstore) = @_;
    my $log_count = 0;

    require Monitoring::Availability::Logs;

    # get current auto increment values
    my $auto_increments = _get_autoincrements($dbh, $prefix);
    my $foreign_key_stash = {};

    # add import filter
    my $import_filter = _get_exclude_filter($c->config);

    my $stm = "INSERT INTO `".$prefix."_log` (time,class,type,state,state_type,contact_id,host_id,service_id,message) VALUES";

    # make import with relative paths work (thruk chdirs into OMD at start)
    if($ENV{'THRUKOLDPWD'}) {
        chdir($ENV{'THRUKOLDPWD'});
    }

    my $failed = 0;
    for my $p (@{$files}) {
        my $expanded = [];
        if(-f $p) {
            $expanded = [$p];
        } elsif(-d $p.'/.') {
            $expanded = [glob($p.'/*')];
        } else {
            $expanded = [glob($p)];
            if(scalar @{$expanded} == 0) {
                $failed++;
                _warn("no files found for: $p");
                next;
            }
        }
        for my $f (@{$expanded}) {
            if(!-f $f) {
                $failed++;
                _warn("skipping $f: $!");
                next;
            }
            _infos($f);
            my $duplicate_lookup  = {};
            my $last_duplicate_ts = 0;
            my @values;

            open(my $fh, '<', $f) or die("cannot open ".$f.": ".$!);
            while(my $line = <$fh>) {
                chomp($line);
                &Thruk::Utils::Encode::decode_any($line);
                &Thruk::Utils::Encode::remove_utf8_surrogates($line);
                my $original_line = $line;
                my $l = &Monitoring::Availability::Logs::parse_line($line); # do not use xs here, unchanged $line breaks the _set_class later
                next unless($l && $l->{'time'});
                next if $import_filter && $original_line =~ $import_filter;

                if($mode eq 'update') {
                    if($last_duplicate_ts < $l->{'time'}) {
                        $self->_safe_insert($dbh, $stm, \@values);
                        $self->_safe_insert_stash($dbh, $prefix, $foreign_key_stash);
                        @values = ();
                        $duplicate_lookup = $self->_fill_lookup_logs($prefix,$l->{'time'},$l->{'time'}+86400);
                        $last_duplicate_ts = $l->{'time'}+86400;
                    }
                    next if defined $duplicate_lookup->{$original_line};
                }

                $log_count++;
                $l->{'message'} = $original_line;
                my($host, $svc, $contact) = _fix_import_log($l, $host_lookup, $service_lookup, $contact_lookup, $dbh, $prefix, $auto_increments, $foreign_key_stash);

                # commit every 1000th to avoid to large blocks
                if($log_count%1000 == 0) {
                    $self->_safe_insert($dbh, $stm, \@values);
                    $self->_safe_insert_stash($dbh, $prefix, $foreign_key_stash);
                    @values = ();
                    _infoc('.');
                }

                if($import_compacted && _is_compactable($l, $alertstore, $import_filter)) {
                    # skip insert
                    next;
                }

                push @values, sprintf('(%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                        $l->{'time'},
                        $l->{'class'},
                        $dbh->quote($l->{'type'}),
                        $dbh->quote($l->{'state'}),
                        $dbh->quote($l->{'state_type'}),
                        $dbh->quote($contact),
                        $dbh->quote($host),
                        $dbh->quote($svc),
                        $dbh->quote($l->{'message'}),
                );
            }
            $self->_safe_insert($dbh, $stm, \@values);
            $self->_safe_insert_stash($dbh, $prefix, $foreign_key_stash);
            CORE::close($fh);
            _info(". OK");
        }
    }

    unless ($c->config->{'logcache_pxc_strict_mode'}) {
        _release_write_locks($dbh);
        _info("it is recommended to run logcacheoptimize after importing logfiles.");
    }

    # restore old working dir
    if($ENV{'THRUKOLDPWD'}) {
        chdir($ENV{'HOME'});
    }

    if($failed == scalar @{$files}) {
        die("no file imported");
    }

    return $log_count;
}

##########################################################
sub _insert_logs {
    my($self,$dbh,$mode,$logs,$host_lookup,$service_lookup,$duplicate_lookup,$prefix,$contact_lookup,$c,$use_extended_inserts, $import_compacted, $alertstore) = @_;
    my $log_count = 0;
    my $compacted = 0;

    my $dots_each = 1000;
    if($mode eq 'update') {
        $mode = MODE_UPDATE;
    } elsif($mode eq 'import') {
        $mode = MODE_IMPORT;
        $dots_each = 10000;
    }

    if(!defined $use_extended_inserts) {
        $use_extended_inserts = $mode == MODE_IMPORT ? 0 : 1;
    }

    # check pid / lock
    my @pids = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 2 LIMIT 1')};
    if(scalar @pids == 1 && $pids[0] && $pids[0] != $$) {
        _warn("logcache update already running with pid ".$pids[0]);
        return $log_count;
    }

    # get current auto increment values
    my $auto_increments = _get_autoincrements($dbh, $prefix);
    my $foreign_key_stash = {};

    # add import filter
    my $import_filter = _get_exclude_filter($c->config);

    my $stm = "INSERT INTO `".$prefix."_log` (time,class,type,state,state_type,contact_id,host_id,service_id,message) VALUES";

    my @values;
    my($fh, $datafilename);
    if(!$use_extended_inserts) {
        ($fh, $datafilename) = tempfile();
        $fh->binmode (":encoding(utf-8)");
    }
    &timing_breakpoint('_insert_logs');
    for my $l (@{$logs}) {
        next unless $l->{'message'};
        &Thruk::Utils::Encode::remove_utf8_surrogates($l->{'message'});
        if($mode == MODE_UPDATE) {
            next if defined $duplicate_lookup->{$l->{'message'}};
        }

        next if $import_filter && $l->{'message'} =~ $import_filter;

        $log_count++;
        _infoc('.') if $log_count % $dots_each == 0;

        my($host, $svc, $contact) = _fix_import_log($l, $host_lookup, $service_lookup, $contact_lookup, $dbh, $prefix, $auto_increments, $foreign_key_stash);

        if($import_compacted && _is_compactable($l, $alertstore, $import_filter)) {
            # skip insert
            $compacted++;
            next;
        }

        if($use_extended_inserts) {
            push @values, sprintf('(%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                    $l->{'time'},
                    $l->{'class'},
                    $dbh->quote($l->{'type'}),
                    $dbh->quote($l->{'state'}),
                    $dbh->quote($l->{'state_type'}),
                    $dbh->quote($contact),
                    $dbh->quote($host),
                    $dbh->quote($svc),
                    $dbh->quote($l->{'message'}),
            );
        } else {
            printf($fh "%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\n",
                    $l->{'time'},
                    $l->{'class'},
                    $l->{'type'}       // '\N',
                    $l->{'state'}      // '\N',
                    $l->{'state_type'} // '\N',
                    $contact           // '\N',
                    $host              // '\N',
                    $svc               // '\N',
                    $l->{'message'},
            );
        }

        # commit every 1000th to avoid to large blocks
        if($use_extended_inserts && $log_count%1000 == 0) {
            &timing_breakpoint('_insert_logs logs calculated');
            $self->_safe_insert($dbh, $stm, \@values);
            @values = ();
            &timing_breakpoint('_insert_logs logs inserted');
            $self->_safe_insert_stash($dbh, $prefix, $foreign_key_stash);
        }
    }
    if($use_extended_inserts) {
        $self->_safe_insert($dbh, $stm, \@values);
    } else {
        &timing_breakpoint('_insert_logs load data local');
        CORE::close($fh);
        my $stm = sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE `%s_log` FIELDS TERMINATED BY '\0' ENCLOSED BY '' (time,class,type,state,state_type,contact_id,host_id,service_id,message)", $datafilename, $prefix);
        eval {
            $dbh->do($stm);
        };
        my $err = $@;
        unlink($datafilename);
        if($err) {
            _error("ERROR DETAIL: ".$err);
            _error("ERROR SQL: ".$stm);
            # retry with extended inserts
            return(_insert_logs($self,$dbh,$mode,$logs,$host_lookup,$service_lookup,$duplicate_lookup,$prefix,$contact_lookup,$c,1));
        }
        $dbh->commit || confess $dbh->errstr;
        &timing_breakpoint('_insert_logs load data local done');
    }

    $self->_safe_insert_stash($dbh, $prefix, $foreign_key_stash);
    # release locks, unless in import mode. Import releases lock later
    if($mode != MODE_IMPORT) {
        _release_write_locks($dbh) unless $c->config->{'logcache_pxc_strict_mode'};
    }

    if($compacted > 0) {
        _info('. '.($log_count-$compacted) . " entries added and ".$compacted." compacted rows skipped");
    } else {
        _info('. '.$log_count . " entries added");
    }
    return $log_count;
}

##########################################################
sub _create_tables {
    my($dbh, $prefix) = @_;
    for my $stm (@{_get_create_statements($prefix)}) {
        $dbh->do($stm);
    }
    $dbh->commit || confess $dbh->errstr;
    return;
}

##########################################################
sub _drop_tables {
    my($dbh, $prefix) = @_;
    for my $table (@Thruk::Backend::Provider::Mysql::tables) {
        $dbh->do("DROP TABLE IF EXISTS `".$prefix."_".$table.'`');
    }
    $dbh->do("DROP TABLE IF EXISTS `".$prefix."_plugin_output`");
    $dbh->commit || confess $dbh->errstr;
    return;
}

##########################################################
sub _safe_insert {
    my($self, $dbh, $stm, $values) = @_;
    return if scalar @{$values} == 0;
    eval {
        $dbh->do($stm.join(',', @{$values}));
    };
    if($@) {
        _error("ERROR INSERT: ".$@);

        # insert failed for some reason, try them one by one to see which one breaks
        for my $v (@{$values}) {
            eval {
                $dbh->do($stm.$v);
            };
            if ($@) {
                _error("ERROR DETAIL: ".$@);
                _error("ERROR SQL: ".$stm.$v);
            }
        }
    }
    $dbh->commit || confess $dbh->errstr;
    return;
}

##########################################################
sub _safe_insert_stash {
    my($self, $dbh, $prefix, $foreign_key_stash) = @_;

    if($foreign_key_stash->{'host'}) {
        $self->_safe_insert($dbh, "INSERT INTO `".$prefix."_host` (host_id, host_name) VALUES", \@{$foreign_key_stash->{'host'}});
        delete $foreign_key_stash->{'host'};
    }

    if($foreign_key_stash->{'service'}) {
        $self->_safe_insert($dbh, "INSERT INTO `".$prefix."_service` (service_id, host_id, service_description) VALUES", \@{$foreign_key_stash->{'service'}});
        delete $foreign_key_stash->{'service'};
    }

    if($foreign_key_stash->{'contact'}) {
        $self->_safe_insert($dbh, "INSERT INTO `".$prefix."_contact` (contact_id, name) VALUES", \@{$foreign_key_stash->{'contact'}});
        delete $foreign_key_stash->{'contact'};
    }

    return;
}

##########################################################
sub _get_autoincrements {
    my($dbh, $prefix) = @_;
    my $auto_increments = $dbh->selectall_hashref(
        'SELECT
            TABLE_NAME,
            AUTO_INCREMENT
         FROM
            INFORMATION_SCHEMA.TABLES
         WHERE
            TABLE_SCHEMA = Database()
            AND TABLE_NAME LIKE "%'.$prefix.'_%"
        ', 'TABLE_NAME');
    return($auto_increments);
}

##########################################################
sub _release_write_locks {
    my($dbh) = @_;
    $dbh->do('UNLOCK TABLES');
    return;
}

##########################################################
sub _fix_import_log {
    my($l, $host_lookup, $service_lookup, $contact_lookup, $dbh, $prefix, $auto_increments, $foreign_key_stash) = @_;
    my($host, $svc, $contact);

    if(exists $l->{'hard'}) {
        if($l->{'hard'}) {
            $l->{'state_type'} = 'HARD';
        } else {
            $l->{'state_type'} = 'SOFT';
        }
    }
    if(!$l->{'state_type'} || ($l->{'state_type'} ne 'HARD' && $l->{'state_type'} ne 'SOFT')) {
        $l->{'state_type'} = undef;
    }

    $l->{'state'} = undef unless(defined $l->{'state'} && $l->{'state'} ne '');
    &_set_class($l);
    &_set_type($l);

    if($l->{'class'} == 5) { &_set_external_command($l); }

    if($l->{'service_description'}) {
        $host = $host_lookup->{$l->{'host_name'}} || &_host_lookup($host_lookup, $l->{'host_name'}, $dbh, $prefix, $auto_increments, $foreign_key_stash);
        $svc  = $service_lookup->{$l->{'host_name'}}->{$l->{'service_description'}} || &_service_lookup($service_lookup, $host_lookup, $l->{'host_name'}, $l->{'service_description'}, $dbh, $prefix, $host, $auto_increments, $foreign_key_stash);
    }
    elsif($l->{'host_name'}) {
        $host = $host_lookup->{$l->{'host_name'}} || &_host_lookup($host_lookup, $l->{'host_name'}, $dbh, $prefix, $auto_increments, $foreign_key_stash);
    }
    if($l->{'contact_name'}) {
        $contact = $contact_lookup->{$l->{'contact_name'}} || &_contact_lookup($contact_lookup, $l->{'contact_name'}, $dbh, $prefix, $auto_increments, $foreign_key_stash);
    }
    return($host, $svc, $contact);
}

##########################################################
sub _set_class {
    my($l) = @_;
    return if $l->{'class'};
    my $type = $l->{'type'};
    $l->{'class'} = $Thruk::Backend::Provider::Mysql::db_types->{$type} if defined $type;
    return if $l->{'class'};

    if(!defined $l->{'message'}) {
        $l->{'class'}   = 0; # LOGCLASS_INFO
        $l->{'message'} = $type;
        $l->{'type'}    = '';
        return;
    }

    if(   $l->{'message'} =~ m/starting\.\.\./mxo
       or $l->{'message'} =~ m/shutting\ down\.\.\./mxo
       or $l->{'message'} =~ m/Bailing\ out/mxo
       or $l->{'message'} =~ m/active\ mode\.\.\./mxo
       or $l->{'message'} =~ m/standby\ mode\.\.\./mxo
    ) {
        $l->{'class'} = 2; # LOGCLASS_PROGRAM
        $l->{'message'} = $l->{'type'}.': '.$l->{'message'} if($l->{'type'} && $l->{'message'} !~ m/^\[\d+\]/mx);
        $l->{'type'}    = '';
        return;
    }

    $l->{'type'}    = '';
    $l->{'class'}   = 0; # LOGCLASS_INFO
    return;
}

##########################################################
sub _set_type {
    my($l) = @_;

    if($l->{'message'} =~ m/^\[\d+\]\s+TIMEPERIOD\ TRANSITION/mxo) {
        $l->{'type'}  = 'TIMEPERIOD TRANSITION';
        $l->{'class'} = 6; # LOGCLASS_STATE
        return;
    }

    if(defined $l->{'type'}) {
        if(!defined $Thruk::Backend::Provider::Mysql::db_types->{$l->{'type'}}) {
            # Set type to NULL to prevent SQL insert errors if type is not a special type.
            undef $l->{'type'};
        }
        return;
    }

    return;
}

##########################################################
sub _check_index {
    my($c, $dbh, $prefix) = @_;
    $c->stats->profile(begin => "update index statistics");
    _debugs("running check/analyse...");

    my $data = $dbh->selectall_hashref("SHOW INDEXES FROM `".$prefix."_log`", "Key_name");
    if($data && $data->{'host_id'}) {
        if(exists $data->{'host_id'}->{'Cardinality'} && !defined $data->{'host_id'}->{'Cardinality'}) {
            _warn("table index was disabled, enabling...");
            _enable_index($dbh, $prefix);
            _warn("done.");
        }
        my($hostcount) = @{$dbh->selectcol_arrayref("SELECT COUNT(*) as total FROM `".$prefix."_host`")};
        if(!$hostcount || !$data->{'host_id'}->{'Cardinality'} || $data->{'host_id'}->{'Cardinality'} < $hostcount * 5) {
            $c->stats->profile(end => "update index statistics");
            _debug("not required");
            return;
        }
    }

    for my $table (@Thruk::Backend::Provider::Mysql::tables) {
        $dbh->do("ANALYZE TABLE `".$prefix."_".$table.'`');
        $dbh->do("CHECK TABLE `".$prefix."_".$table.'`');
    }
    _debug("done");
    $c->stats->profile(end => "update index statistics");
    return;
}

##########################################################
sub _check_db_fs {
    my($self, $c, $peer, $prefix) =  @_;

    _debug2("[%s] checking required disk space", $prefix);

    # fetch mysql datadir
    my $dbh = $self->_dbh();
    my $res  = $dbh->selectall_arrayref("SHOW VARIABLES LIKE 'datadir'", { Slice => {} });
    if(scalar @{$res} != 1) {
        _debug2("[%s] cannot fetch datadir variable.", $prefix);
        return 1;
    }
    my $datadir = $res->[0]->{'Value'};

    # fetch mysql hostname
    $res  = $dbh->selectall_arrayref('SELECT @@hostname', { Slice => {} });
    if(scalar @{$res} != 1) {
        _debug2("[%s] cannot fetch hostname variable.", $prefix);
        return 1;
    }
    my $dbhost = $res->[0]->{'@@hostname'};

    _debug("[%s] db runs on %s with datadir %s", $prefix, $dbhost, $datadir);

    chomp(my $hostname = Thruk::Utils::IO::cmd("hostname"));
    chomp(my $fqdn     = Thruk::Utils::IO::cmd("hostname --fqdn"));

    # not on the same host, no chance to access the filesystem
    if($dbhost ne $hostname && $dbhost ne $fqdn) {
        _debug2("[%s] database is not on the same host, cannot check filesystem. (%s != %s)", $prefix, $dbhost, $fqdn);
        return 1;
    }

    # try to get free disk space
    my($rc, $diskspace) = Thruk::Utils::IO::cmd("df -k $datadir 2>&1"); # use -k to force 1k blocks
    if($rc != 0) {
        _debug2("[%s] cannot check filesystem available space: %s", $prefix, $diskspace);
        return 1;
    }
    chomp($diskspace);

    my @lines = split(/\n/mx, $diskspace);
    my(undef, undef, undef, $disk_available) = split(/\s+/mx, $lines[scalar @lines -1]);
    if(!$disk_available || $disk_available !~ m/^\d+$/mx) {
        _debug2("[%s] cannot check filesystem available space.", $prefix);
        return 1;
    }
    $disk_available = $disk_available * 1024;

    my @stats = $self->_log_stats($c, $prefix);
    if(scalar @stats != 1) {
        _debug2("[%s] cannot fetch log stats", $prefix);
        return 1;
    }

    # add 10% safety
    my $required = 1.1 * ($stats[0]->{'data_size'} + $stats[0]->{'index_size'});
    if($required > $disk_available) {
        _warn("[%s] not enough disk space for table optimization: required: %5.1f %2s, available: %5.1f %2s", $prefix, Thruk::Utils::reduce_number($required, "B"), Thruk::Utils::reduce_number($disk_available, "B"));
        return;
    }

    _info("[%s] disk space sufficient for table optimization: required: %5.1f %2s, available: %5.1f %2s", $prefix, Thruk::Utils::reduce_number($required, "B"), Thruk::Utils::reduce_number($disk_available, "B"));
    return 1;
}

##########################################################
sub _set_external_command {
    my($l) = @_;
    # add hosts/services to external commands
    my $msg = $l->{'message'};
    $msg =~ s/^\[\d+\]\ EXTERNAL\ COMMAND:\ //gmxo;
    $msg =~ s/^(.*?);//gmxo;
    my $cmd;
    if($1) {
        $cmd = $1;
    }
    return unless $cmd;
    if($cmd =~ m/_HOST(_|$)/mx) {
        if($msg =~ m/^([^;]+);(;|$)/gmx) {
            $l->{'host_name'} = $1;
        }
    }
    elsif($cmd =~ m/_SVC(_|$)/mx) {
        if($msg =~ m/^([^;]+);([^;]+)(;|$)/gmx) {
            $l->{'host_name'} = $1;
            $l->{'service_description'} = $2;
        }
    }
    elsif($cmd =~ m/_CONTACT(_|$)/mx) {
        if($msg =~ m/^([^;]+);(;|$)/gmx) {
            $l->{'contact_name'} = $1;
        }
    }
    return;
}

##########################################################
sub _sql_debug {
    my($sql, $dbh) = @_;

    my $sth = $dbh->prepare($sql);
    $sth->execute;
    my $data = $sth->fetchall_arrayref({});

    return(Thruk::Utils::text_table(
        keys => $sth->{'NAME'},
        data => $data,
    ));
}

##########################################################
sub _get_create_statements {
    my($prefix) = @_;
    my @statements = (
    # contact
        "DROP TABLE IF EXISTS `".$prefix."_contact`",
        "CREATE TABLE `".$prefix."_contact` (
          contact_id mediumint(9) unsigned NOT NULL AUTO_INCREMENT,
          name varchar(150) NOT NULL,
          PRIMARY KEY (contact_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

    # contact_host_rel
        "DROP TABLE IF EXISTS `".$prefix."_contact_host_rel`",
        "CREATE TABLE `".$prefix."_contact_host_rel` (
          contact_id mediumint(9) unsigned NOT NULL,
          host_id mediumint(9) unsigned NOT NULL,
          PRIMARY KEY (contact_id,host_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

    # contact_service_rel
        "DROP TABLE IF EXISTS `".$prefix."_contact_service_rel`",
        "CREATE TABLE `".$prefix."_contact_service_rel` (
          contact_id mediumint(9) unsigned NOT NULL,
          service_id mediumint(9) unsigned NOT NULL,
          PRIMARY KEY (contact_id,service_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

    # host
        "DROP TABLE IF EXISTS `".$prefix."_host`",
        "CREATE TABLE `".$prefix."_host` (
          host_id mediumint(9) unsigned NOT NULL AUTO_INCREMENT,
          host_name varchar(150) NOT NULL,
          PRIMARY KEY (host_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

    # log
        "DROP TABLE IF EXISTS `".$prefix."_log`",
        "CREATE TABLE IF NOT EXISTS `".$prefix."_log` (
          log_id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
          time int(11) unsigned NOT NULL,
          class tinyint(4) unsigned NOT NULL,
          type enum('CURRENT SERVICE STATE','CURRENT HOST STATE','SERVICE NOTIFICATION','HOST NOTIFICATION','SERVICE ALERT','HOST ALERT','SERVICE EVENT HANDLER','HOST EVENT HANDLER','EXTERNAL COMMAND','PASSIVE SERVICE CHECK','PASSIVE HOST CHECK','SERVICE FLAPPING ALERT','HOST FLAPPING ALERT','SERVICE DOWNTIME ALERT','HOST DOWNTIME ALERT','LOG ROTATION','INITIAL HOST STATE','INITIAL SERVICE STATE','TIMEPERIOD TRANSITION') DEFAULT NULL,
          state tinyint(4) unsigned DEFAULT NULL,
          state_type enum('HARD','SOFT') DEFAULT NULL,
          contact_id mediumint(9) unsigned DEFAULT NULL,
          host_id mediumint(9) unsigned DEFAULT NULL,
          service_id mediumint(9) unsigned DEFAULT NULL,
          message mediumtext NOT NULL,
          PRIMARY KEY (log_id),
          KEY time (time),
          KEY host_id (host_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci PACK_KEYS=1",    # using utf8_bin here would break case-insensitive rlike queries

    # service
        "DROP TABLE IF EXISTS `".$prefix."_service`",
        "CREATE TABLE `".$prefix."_service` (
          service_id mediumint(9) unsigned NOT NULL AUTO_INCREMENT,
          host_id mediumint(9) unsigned NOT NULL,
          service_description varchar(150) NOT NULL,
          PRIMARY KEY (service_id),
          KEY host_id (host_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

    # status
        "DROP TABLE IF EXISTS `".$prefix."_status`",
        "CREATE TABLE `".$prefix."_status` (
          status_id smallint(6) unsigned NOT NULL AUTO_INCREMENT,
          name varchar(150) NOT NULL,
          value varchar(150) DEFAULT NULL,
          PRIMARY KEY (status_id)
        ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin",

        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(1, 'last_update', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(2, 'update_pid', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(3, 'last_reorder', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(4, 'cache_version', '".$Thruk::Backend::Provider::Mysql::cache_version."')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(5, 'reorder_duration', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(6, 'update_duration', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(7, 'last_compact', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(8, 'compact_duration', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(9, 'compact_till', '')",
        "INSERT INTO `".$prefix."_status` (status_id, name, value) VALUES(10,'lock_mode', '')",
    );
    return \@statements;
}

##########################################################

1;
