Merge branch 'dev'

Author: casperr0

mamonsu/lib/zbx_template.py

@@ -6,7 +6,7 @@
 
 class ZbxTemplate(object):
     plg_type = 'all'
-    mainTemplate = u"""<?xml version="1.0" encoding="UTF-8"?>
+    mainTemplate = """<?xml version="1.0" encoding="UTF-8"?>
 <zabbix_export>
 <version>2.0</version>
     <groups>

mamonsu/plugins/pgsql/__init__.py

@@ -4,7 +4,6 @@
 __all__ += ['checkpoint', 'oldest', 'pg_locks']
 __all__ += ['cfs']
 __all__ += ['archive_command']
-__all__ += ['relations_size']
 __all__ += ['prepared_transaction']
 
 from . import *

mamonsu/plugins/pgsql/driver/pg8000/__init__.py

@@ -1,10 +1,18 @@
-from .core import (
-    Warning, DataError, DatabaseError, InterfaceError, ProgrammingError,
-    Error, OperationalError, IntegrityError, InternalError, NotSupportedError,
-    ArrayContentNotHomogenousError, ArrayDimensionsNotConsistentError,
-    ArrayContentNotSupportedError, Connection, Cursor, Binary, Date,
-    DateFromTicks, Time, TimeFromTicks, Timestamp, TimestampFromTicks, BINARY,
-    Interval, PGEnum, PGJson, PGJsonb, PGTsvector, PGText, PGVarchar)
+from .converters import (
+    BIGINTEGER, BINARY, BOOLEAN, BOOLEAN_ARRAY, BYTES, Binary, CHAR,
+    CHAR_ARRAY, DATE, DATETIME, DECIMAL, DECIMAL_ARRAY, Date, DateFromTicks,
+    FLOAT, FLOAT_ARRAY, INET, INT2VECTOR, INTEGER, INTEGER_ARRAY, INTERVAL,
+    JSON, JSONB, MACADDR, NAME, NAME_ARRAY, NULLTYPE, NUMBER, OID, PGEnum,
+    PGInterval, PGJson, PGJsonb, PGText, PGTsvector, PGVarchar, ROWID, STRING,
+    TEXT, TEXT_ARRAY, TIME, TIMEDELTA, TIMESTAMP, TIMESTAMPTZ, Time,
+    TimeFromTicks, Timestamp, TimestampFromTicks, UNKNOWN, UUID_TYPE, VARCHAR,
+    VARCHAR_ARRAY, XID)
+from .core import Connection, Cursor
+from .exceptions import (
+    DataError, DatabaseError, Error, IntegrityError, InterfaceError,
+    InternalError, NotSupportedError, OperationalError, ProgrammingError,
+    Warning)
+
 from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
@@ -43,16 +51,13 @@
 def connect(
         user, host='localhost', database=None, port=5432, password=None,
         source_address=None, unix_sock=None, ssl_context=None, timeout=None,
-        max_prepared_statements=1000, tcp_keepalive=True,
-        application_name=None, replication=None):
+        tcp_keepalive=True, application_name=None, replication=None):
 
     return Connection(
         user, host=host, database=database, port=port, password=password,
         source_address=source_address, unix_sock=unix_sock,
-        ssl_context=ssl_context, timeout=timeout,
-        max_prepared_statements=max_prepared_statements,
-        tcp_keepalive=tcp_keepalive, application_name=application_name,
-        replication=replication)
+        ssl_context=ssl_context, timeout=timeout, tcp_keepalive=tcp_keepalive,
+        application_name=application_name, replication=replication)
 
 
 apilevel = "2.0"
@@ -74,34 +79,16 @@ def connect(
 
 paramstyle = 'format'
 
-# I have no idea what this would be used for by a client app.  Should it be
-# TEXT, VARCHAR, CHAR?  It will only compare against row_description's
-# type_code if it is this one type.  It is the varchar type oid for now, this
-# appears to match expectations in the DB API 2.0 compliance test suite.
-
-STRING = 1043
-"""String type oid."""
-
-
-NUMBER = 1700
-"""Numeric type oid"""
-
-DATETIME = 1114
-"""Timestamp type oid"""
-
-ROWID = 26
-"""ROWID type oid"""
 
 __all__ = [
     Warning, DataError, DatabaseError, connect, InterfaceError,
     ProgrammingError, Error, OperationalError, IntegrityError, InternalError,
-    NotSupportedError, ArrayContentNotHomogenousError,
-    ArrayDimensionsNotConsistentError, ArrayContentNotSupportedError,
-    Connection, Cursor, Binary, Date, DateFromTicks, Time, TimeFromTicks,
-    Timestamp, TimestampFromTicks, BINARY, Interval, PGEnum, PGJson, PGJsonb,
-    PGTsvector, PGText, PGVarchar]
-
-"""Version string for pg8000.
-
-    .. versionadded:: 1.9.11
-"""
+    NotSupportedError, Connection, Cursor, Binary, Date, DateFromTicks, Time,
+    TimeFromTicks, Timestamp, TimestampFromTicks, BINARY, PGInterval, PGEnum,
+    PGJson, PGJsonb, PGTsvector, PGText, PGVarchar, STRING, NUMBER, DATETIME,
+    TIME, BOOLEAN, INTEGER, BIGINTEGER, INTERVAL, JSON, JSONB, UNKNOWN,
+    NULLTYPE, ROWID, BOOLEAN_ARRAY, BYTES, CHAR, CHAR_ARRAY, DATE, DECIMAL,
+    DECIMAL_ARRAY, FLOAT, FLOAT_ARRAY, INET, INT2VECTOR, INTEGER_ARRAY,
+    MACADDR, NAME, NAME_ARRAY, OID, TEXT, TEXT_ARRAY, TIMEDELTA, TIMESTAMP,
+    TIMESTAMPTZ, UUID_TYPE, VARCHAR, VARCHAR_ARRAY, XID
+]

mamonsu/plugins/pgsql/driver/pg8000/_version.py

@@ -0,0 +1,113 @@
+class Warning(Exception):
+    """Generic exception raised for important database warnings like data
+    truncations.  This exception is not currently used by pg8000.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class Error(Exception):
+    """Generic exception that is the base exception of all other error
+    exceptions.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class InterfaceError(Error):
+    """Generic exception raised for errors that are related to the database
+    interface rather than the database itself.  For example, if the interface
+    attempts to use an SSL connection but the server refuses, an InterfaceError
+    will be raised.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class DatabaseError(Error):
+    """Generic exception raised for errors that are related to the database.
+    This exception is currently never raised by pg8000.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class DataError(DatabaseError):
+    """Generic exception raised for errors that are due to problems with the
+    processed data.  This exception is not currently raised by pg8000.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class OperationalError(DatabaseError):
+    """
+    Generic exception raised for errors that are related to the database's
+    operation and not necessarily under the control of the programmer. This
+    exception is currently never raised by pg8000.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class IntegrityError(DatabaseError):
+    """
+    Generic exception raised when the relational integrity of the database is
+    affected.  This exception is not currently raised by pg8000.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class InternalError(DatabaseError):
+    """Generic exception raised when the database encounters an internal error.
+    This is currently only raised when unexpected state occurs in the pg8000
+    interface itself, and is typically the result of a interface bug.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class ProgrammingError(DatabaseError):
+    """Generic exception raised for programming errors.  For example, this
+    exception is raised if more parameter fields are in a query string than
+    there are available parameters.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class NotSupportedError(DatabaseError):
+    """Generic exception raised in case a method or database API was used which
+    is not supported by the database.
+
+    This exception is part of the `DBAPI 2.0 specification
+    <http://www.python.org/dev/peps/pep-0249/>`_.
+    """
+    pass
+
+
+class ArrayContentNotSupportedError(NotSupportedError):
+    """
+    Raised when attempting to transmit an array where the base type is not
+    supported for binary data transfer by the interface.
+    """
+    pass

mamonsu/plugins/pgsql/driver/pg8000/converters.py

@@ -16,7 +16,7 @@ class PreparedTransaction(Plugin):
     key_prepared = {
         'state': 'oldest_prepared',
         'key': 'pgsql.prepared.oldest',
-        'name': 'PostgreSQL: oldest prepared transaction running time in sec',
+        'name': 'PostgreSQL: oldest prepared transaction time in sec',
         'color': '0000BB',
         'yaxisside': 1,
     }
@@ -71,7 +71,7 @@ def graphs(self, template):
 
     def triggers(self, template):
         result = template.trigger({
-            'name': 'PostgreSQL prepared transaction running is too old on {HOSTNAME}',
+            'name': 'PostgreSQL prepared transaction is too old on {HOSTNAME}',
             'expression': '{#TEMPLATE:' + self.key_prepared['key'] +
                           '.last()}>' + self.plugin_config('max_prepared_transaction_time')
         })

mamonsu/plugins/pgsql/driver/pg8000/core.py

@@ -56,7 +56,7 @@ def run(self, zbx):
             dir_size = self.dir_size(_dir)
             if self.os_walk_error:
                 self.log.error(
-                    "Error in count size backup catalog: {backup_catalog}. Error: {error}".format(
+                    "Error in count size pg_probackup dir: {backup_catalog}. Error: {error}".format(
                         backup_catalog=_dir, error=str(self.os_walk_error)))
             else:
                 zbx.send(self.key_dir_size.format('[' + _dir + ']'), dir_size)
@@ -78,17 +78,20 @@ def run(self, zbx):
             except Exception as e:
                 self.log.error('Error in convert data: {stdout} \n {e}'.format(stdout=stdout, e=e))
                 continue
-
+            no_error= True
             for instance in result:
                 for backup in instance.get('backups', []):
                     status = backup['status']
                     if status in ['ERROR', 'CORRUPT', 'ORPHAN']:
-                        error = 'Backup with id: {backup_id} in instance: {instance_name} in backup catalog: ' \
+                        error = 'Backup with id: {backup_id} in instance: {instance_name} in pg_probackup dir: ' \
                                 '{backup_catalog}  has status: {status}.'.format(backup_id=backup['id'],
                                                                                  instance_name=instance['instance'],
                                                                                  status=status, backup_catalog=_dir)
                         self.log.info(error)
+                        no_error = False
                         zbx.send(self.key_dir_error.format('[' + _dir + ']'), error)
+            if no_error:
+                zbx.send(self.key_dir_error.format('[' + _dir + ']'), 'ok')
 
         zbx.send(self.key_main.format('[]'), zbx.json({'data': dirs}))
         del dirs
@@ -114,12 +117,12 @@ def discovery_rules(self, template):
             ]
         items = [
             {'key': self.right_type(self.key_dir_size, var_discovery="{#BACKUPDIR},"),
-             'name': 'Pg_probackup catalog {#BACKUPDIR}: size',
+             'name': 'Pg_probackup dir {#BACKUPDIR}: size',
              'units': Plugin.UNITS.bytes,
              'value_type': Plugin.VALUE_TYPE.numeric_unsigned,
              'delay': self.plugin_config('interval')},
             {'key': self.right_type(self.key_dir_error, var_discovery="{#BACKUPDIR},"),
-             'name': 'Pg_probackup catalog {#BACKUPDIR}: error',
+             'name': 'Pg_probackup dir {#BACKUPDIR}: error',
              'value_type': Plugin.VALUE_TYPE.text,
              'delay': self.plugin_config('interval')},
         ]
@@ -133,8 +136,8 @@ def discovery_rules(self, template):
             },
         ]
         triggers = [{
-            'name': 'Error in backup dir  '
+            'name': 'Error in pg_probackup dir  '
                     '{#BACKUPDIR} (hostname={HOSTNAME} value={ITEM.LASTVALUE})',
-            'expression': '{#TEMPLATE:pg_probackup.dir.error[{#BACKUPDIR}].strlen()}>1'}
+            'expression': '{#TEMPLATE:pg_probackup.dir.error[{#BACKUPDIR}].str(ok)}<>1'}
         ]
         return template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs, triggers=triggers)

mamonsu/plugins/pgsql/driver/pg8000/exceptions.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 from mamonsu.lib.plugin import Plugin
-
 from mamonsu.lib.const import API
 
 from urllib.parse import urlparse as parse
 from urllib.parse import parse_qs
 from http.server import BaseHTTPRequestHandler, HTTPServer
+import logging
 
 
 class AgentApi(Plugin):
@@ -36,8 +36,13 @@ class AgentApiHandler(BaseHTTPRequestHandler):
 
     def __init__(self, config, *args):
         self.sender = config.sender
+        self.log = logging.getLogger('AGENTAPI')
         BaseHTTPRequestHandler.__init__(self, *args)
 
+    # rewrite log_message method of http lib to avoid getting log messages in console
+    def log_message(self, format, *args):
+        self.log.info(format, *args)
+
     def _set_header(self):
         self.send_response(200)
         self.send_header("Content-type", "text/html")

mamonsu/plugins/pgsql/prepared_transaction.py

@@ -167,17 +167,6 @@ interval = 2
 [agentapi]
 interval = 60
 
-# Get size of relations defined in this section
-# Relations - comma separated list of objects - tables and endexes (schema.relation) to calculate relations size.
-# Example:
-# relations=pg_catalog.pg_class,pg_catalog.pg_user
-# If the relation is blocked by some process such as vacuum full or create index, the result will be -1
-# by default this plugin disabled. To enable this plugin - set bellow "enabled = False" and define a list of relations.
-[relationssize]
-enabled = False
-relations=pg_catalog.pg_class,pg_catalog.pg_user
-interval = 500
-
 # Get age (in seconds) of the oldest running prepared transaction and number of all prepared transactions for two-phase commit.
 # https://www.postgresql.org/docs/current/sql-prepare-transaction.html
 # https://www.postgresql.org/docs/12/view-pg-prepared-xacts.html
@@ -188,10 +177,10 @@ max_prepared_transaction_time = 60
 interval = 60
 
 # Get size of backup catalogs stroring all WAL and backup files using pg_probackup
-# S(https://github.com/postgrespro/pg_probackup)
+# (https://github.com/postgrespro/pg_probackup)
 # Trigger fires if some backup has bad status e.g. (ERROR,CORRUPT,ORPHAN).
 [pgprobackup]
-enabled = false
+enabled = False
 interval = 300
 backup_dirs = /backup_dir1,/backup_dir2
 pg_probackup_path = /usr/bin/pg_probackup-11

mamonsu/plugins/pgsql/relations_size.py

@@ -169,11 +169,6 @@ directory = /tmp
 file = /var/log/mamonsu/agent.log
 level = DEBUG
 
-[relationssize]
-enabled = True
-relations=pg_catalog.pg_class,pg_catalog.pg_user
-interval = 60
-
 [preparedtransaction]
 max_prepared_transaction_time = 60
 interval = 60
@@ -194,7 +189,6 @@ mamonsu agent -c /etc/mamonsu/agent.conf metric-list | grep system
 ## metric log
 grep utilization /tmp/localhost.log || exit 7
 grep 'pgsql\.uptime' /tmp/localhost.log || exit 7
-grep 'pgsql\.relation\.size' /tmp/localhost.log || exit 7
 grep 'pgsql\.prepared\.count' /tmp/localhost.log || exit 7
 grep 'pgsql\.prepared\.oldest' /tmp/localhost.log || exit 7