Skip to content

Commit

Permalink
2.3.2 release:
Browse files Browse the repository at this point in the history
  - bootstrap new options: -M (mamonsu non-priviledged user), -v(verbose)
  - get oldest query xid and age via functions
  - fix missing UpdateExisting for applications in Zabbix API 3.4
  • Loading branch information
gsmolk committed Dec 12, 2017
1 parent 65656fa commit 3134dfb
Show file tree
Hide file tree
Showing 10 changed files with 338 additions and 234 deletions.
10 changes: 1 addition & 9 deletions Makefile.pkg
Expand Up @@ -37,7 +37,7 @@ build/all: build/pkg_debian build/pkg_ubuntu build/pkg_centos
build/pkg_debian: build/pkg_debian_7 build/pkg_debian_8
@echo Debian: done

build/pkg_ubuntu: build/pkg_ubuntu_14_04 build/pkg_ubuntu_15_10 build/pkg_ubuntu_16_04 build/pkg_ubuntu_16_10
build/pkg_ubuntu: build/pkg_ubuntu_14_04 build/pkg_ubuntu_16_04
@echo Ubuntu: done

build/pkg_centos: build/pkg_centos_6 build/pkg_centos_7
Expand Down Expand Up @@ -72,18 +72,10 @@ build/pkg_ubuntu_14_04:
$(call build_deb,ubuntu,trusty)
touch build/pkg_ubuntu_14_04

build/pkg_ubuntu_15_10:
$(call build_deb,ubuntu,wily)
touch build/pkg_ubuntu_15_10

build/pkg_ubuntu_16_04:
$(call build_deb,ubuntu,xenial)
touch build/pkg_ubuntu_16_04

build/pkg_ubuntu_16_10:
$(call build_deb,ubuntu,yakkety)
touch build/pkg_ubuntu_16_10

build/pkg_centos_6:
$(call build_rpm,centos,6)
touch build/pkg_centos_6
Expand Down
8 changes: 5 additions & 3 deletions Makefile.tests
@@ -1,6 +1,8 @@
test: run_functional_tests
test: run_builds run_functional_tests

run_functional_tests:
run_builds:
docker run --rm -v "$(WORKDIR)":/var/tmp centos:7 /bin/bash -ex "/var/tmp/tests/build_rpm.sh"
docker run --rm -v "$(WORKDIR)":/var/tmp debian:7 /bin/bash -ex "/var/tmp/tests/build_deb.sh"
docker run --rm -v "$(WORKDIR)":/var/tmp centos:6 /bin/bash -ex "/var/tmp/tests/check.sh"

run_functional_tests:
docker run --rm -v "$(WORKDIR)":/var/tmp centos:6 /bin/bash -ex "/var/tmp/tests/check.sh"
2 changes: 1 addition & 1 deletion mamonsu/__init__.py
@@ -1,7 +1,7 @@
__author__ = 'Dmitry Vasilyev'
__author_email__ = 'info@postgrespro.ru'
__description__ = 'Monitoring agent for PostgreSQL'
__version__ = '2.3.1'
__version__ = '2.3.2'
__licence__ = 'BSD'

__url__ = 'https://github.com/postgrespro/mamonsu'
Expand Down
32 changes: 24 additions & 8 deletions mamonsu/plugins/pgsql/driver/pool.py
Expand Up @@ -44,7 +44,7 @@ def __init__(self, params={}):
self._connections = {}
self._cache = {
'server_version': {'storage': {}},
'bootstrap': {'storage': {}, 'counter': 0, 'cache': 10},
'bootstrap': {'storage': {}, 'counter': 0, 'cache': 10, 'version': False},
'recovery': {'storage': {}, 'counter': 0, 'cache': 10},
'pgpro': {'storage': {}},
'pgproee': {'storage': {}}
Expand Down Expand Up @@ -72,13 +72,20 @@ def server_version(self, db=None):
result.decode('ascii'))
return self._cache['server_version']['storage'][db]

def server_version_greater(self, version, db=None):
db = self._normalize_db(db)
return self.server_version(db) >= LooseVersion(version)
def server_version_greater(self, version, db=None, bootstrap=False):
if not bootstrap:
db = self._normalize_db(db)
return self.server_version(db) >= LooseVersion(version)
else:
return str(self._cache['bootstrap']['version']) >= LooseVersion(version)

def server_version_less(self, version, db=None):
db = self._normalize_db(db)
return self.server_version(db) <= LooseVersion(version)
def server_version_less(self, version, db=None, bootstrap=False):
if not bootstrap:
db = self._normalize_db(db)
return self.server_version(db) <= LooseVersion(version)
else:
print(self._cache['bootstrap']['version'])
return self._cache['bootstrap']['version'] <= LooseVersion(version)

def in_recovery(self, db=None):
db = self._normalize_db(db)
Expand All @@ -104,12 +111,21 @@ def is_bootstraped(self, db=None):
self._cache['bootstrap']['storage'][db] = (result == 1)
if self._cache['bootstrap']['storage'][db]:
self._connections[db].log.info('Found mamonsu bootstrap')
sql = 'select max(version) from public.mamonsu_config'
self._cache['bootstrap']['version'] = self.query(sql, db)[0][0]
else:
self._connections[db].log.info('Can\'t found mamonsu bootstrap')
self._connections[db].log.info('Mamonsu bootstrap is not found')
self._connections[db].log.info(
'hint: run `mamonsu bootstrap` if you want to run without superuser rights')
return self._cache['bootstrap']['storage'][db]

def is_superuser(self, db=None):
db = self._normalize_db(db)
if self.query("select current_setting('is_superuser')")[0][0] == 'on':
return True
else:
return False

def is_pgpro(self, db=None):
db = self._normalize_db(db)
if db in self._cache['pgpro']:
Expand Down
17 changes: 15 additions & 2 deletions mamonsu/plugins/pgsql/oldest.py
Expand Up @@ -10,12 +10,20 @@ class Oldest(Plugin):
select
greatest(max(age(backend_xmin)), max(age(backend_xid)))
from pg_catalog.pg_stat_activity;
"""

OldestXidSql_bootstrap = """
select public.mamonsu_get_oldest_xid();
"""

OldestQuerySql = """
select
extract(epoch from max(now() - xact_start))
from pg_catalog.pg_stat_activity;
"""

OldestQuerySql_bootstrap = """
select public.mamonsu_get_oldest_query();
"""

DEFAULT_CONFIG = {
Expand All @@ -24,9 +32,14 @@ class Oldest(Plugin):
}

def run(self, zbx):
xid = Pooler.query(self.OldestXidSql)[0][0]
if Pooler.is_bootstraped() and Pooler.server_version_greater('2.3.2', bootstrap=True):
xid = Pooler.query(self.OldestXidSql_bootstrap)[0][0]
query = Pooler.query(self.OldestQuerySql_bootstrap)[0][0]
else:
xid = Pooler.query(self.OldestXidSql)[0][0]
query = Pooler.query(self.OldestQuerySql)[0][0]

zbx.send('pgsql.oldest[xid_age]', xid)
query = Pooler.query(self.OldestQuerySql)[0][0]
zbx.send('pgsql.oldest[query_time]', query)

def graphs(self, template):
Expand Down
185 changes: 110 additions & 75 deletions mamonsu/tools/bootstrap/sql.py
@@ -1,75 +1,110 @@
from mamonsu import __version__ as mamonsu_version

QuerySplit = """
"""

CreateSchemaSQL = """
CREATE TABLE IF NOT EXISTS public.mamonsu_config (
version text,
inserted_at timestamp DEFAULT NOW()
);
INSERT INTO public.mamonsu_config(version) VALUES('{0}');
DROP TABLE IF EXISTS public.mamonsu_timestamp_master_{1};
CREATE TABLE public.mamonsu_timestamp_master_{1}(
id int primary key,
ts double precision,
lsn pg_lsn
);
INSERT INTO public.mamonsu_timestamp_master_{1} (id) values (1);
CREATE OR REPLACE FUNCTION public.mamonsu_timestamp_master_update()
RETURNS void AS $$
UPDATE public.mamonsu_timestamp_master_{1} SET
ts = extract(epoch from now() at time zone 'utc')::double precision,
lsn = pg_catalog.pg_current_xlog_location()
WHERE
id = 1;
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_timestamp_get()
RETURNS double precision AS $$
SELECT
(extract(epoch from now() at time zone 'utc') - ts)::double precision
FROM public.mamonsu_timestamp_master_{1}
WHERE id = 1 LIMIT 1;
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_count_autovacuum()
RETURNS BIGINT AS $$
SELECT
count(*)::BIGINT
FROM pg_catalog.pg_stat_activity
WHERE
query like '%%autovacuum%%'
and state <> 'idle'
and pid <> pg_catalog.pg_backend_pid()
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_count_xlog_files()
RETURNS BIGINT AS $$
WITH list(filename) as (SELECT * FROM pg_catalog.pg_ls_dir('pg_xlog'))
SELECT
COUNT(*)::BIGINT
FROM
list
WHERE filename similar to '{2}'
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE EXTENSION IF NOT EXISTS pg_buffercache;
CREATE OR REPLACE FUNCTION public.mamonsu_buffer_cache()
RETURNS TABLE(SIZE BIGINT, TWICE_USED BIGINT, DIRTY BIGINT) AS $$
SELECT
SUM(1) * 8 * 1024,
SUM(CASE WHEN usagecount > 1 THEN 1 ELSE 0 END) * 8 * 1024,
SUM(CASE isdirty WHEN true THEN 1 ELSE 0 END) * 8 * 1024
FROM public.pg_buffercache
$$ LANGUAGE SQL SECURITY DEFINER;
""".format(
mamonsu_version,
mamonsu_version.replace('.', '_'), '[0-9A-F]{24}')
from mamonsu import __version__ as mamonsu_version

QuerySplit = """
"""

CreateSchemaSQL = """
CREATE TABLE IF NOT EXISTS public.mamonsu_config (
version text,
inserted_at timestamp DEFAULT NOW()
);
INSERT INTO public.mamonsu_config(version) VALUES('{0}');
DROP TABLE IF EXISTS public.mamonsu_timestamp_master_{1};
CREATE TABLE public.mamonsu_timestamp_master_{1}(
id int primary key,
ts double precision,
lsn pg_lsn
);
INSERT INTO public.mamonsu_timestamp_master_{1} (id) values (1);
CREATE OR REPLACE FUNCTION public.mamonsu_timestamp_master_update()
RETURNS void AS $$
UPDATE public.mamonsu_timestamp_master_{1} SET
ts = extract(epoch from now() at time zone 'utc')::double precision,
lsn = pg_catalog.pg_current_xlog_location()
WHERE
id = 1;
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_timestamp_get()
RETURNS double precision AS $$
SELECT
(extract(epoch from now() at time zone 'utc') - ts)::double precision
FROM public.mamonsu_timestamp_master_{1}
WHERE id = 1 LIMIT 1;
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_count_autovacuum()
RETURNS BIGINT AS $$
SELECT
count(*)::BIGINT
FROM pg_catalog.pg_stat_activity
WHERE
query like '%%autovacuum%%'
and state <> 'idle'
and pid <> pg_catalog.pg_backend_pid()
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE or REPLACE FUNCTION public.mamonsu_get_oldest_xid()
RETURNS BIGINT AS $$
SELECT
greatest(max(age(backend_xmin)),
max(age(backend_xid)))::BIGINT
FROM pg_catalog.pg_stat_activity
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE or REPLACE FUNCTION public.mamonsu_get_oldest_query()
RETURNS DOUBLE PRECISION AS $$
SELECT
extract(epoch from max(now() - xact_start))
FROM pg_catalog.pg_stat_activity
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE OR REPLACE FUNCTION public.mamonsu_count_xlog_files()
RETURNS BIGINT AS $$
WITH list(filename) as (SELECT * FROM pg_catalog.pg_ls_dir('pg_xlog'))
SELECT
COUNT(*)::BIGINT
FROM
list
WHERE filename similar to '{2}'
$$ LANGUAGE SQL SECURITY DEFINER;
CREATE EXTENSION IF NOT EXISTS pg_buffercache;
CREATE OR REPLACE FUNCTION public.mamonsu_buffer_cache()
RETURNS TABLE(SIZE BIGINT, TWICE_USED BIGINT, DIRTY BIGINT) AS $$
SELECT
SUM(1) * 8 * 1024,
SUM(CASE WHEN usagecount > 1 THEN 1 ELSE 0 END) * 8 * 1024,
SUM(CASE isdirty WHEN true THEN 1 ELSE 0 END) * 8 * 1024
FROM public.pg_buffercache
$$ LANGUAGE SQL SECURITY DEFINER;
""".format(
mamonsu_version,
mamonsu_version.replace('.', '_'), '[0-9A-F]{24}')

GrantsOnSchemaSQL = """
ALTER TABLE public.mamonsu_config OWNER TO {1};
ALTER TABLE public.mamonsu_timestamp_master_{0} OWNER TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_timestamp_master_update() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_timestamp_get() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_count_autovacuum() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_get_oldest_xid() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_get_oldest_query() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_count_xlog_files() TO {1};
GRANT EXECUTE ON FUNCTION public.mamonsu_buffer_cache() TO {1};
"""

0 comments on commit 3134dfb

Please sign in to comment.