Syncing with the original project

Author: Art

README.rst

@@ -58,7 +58,9 @@ Usage
       -h, --help            show this help message and exit
       -c FILE, --config FILE
                             Configuration file
-      -d, --debug           Debug mode
+      -d, --debug           Prints statsd metrics next to sending them
+      --dry-run             Print the output that would be sent to statsd without
+                            actually sending data somewhere
       -f, --foreground      Dont fork main program
 
 At the moment there is also a `deamon
@@ -236,11 +238,12 @@ https://www.percona.com/live/europe-amsterdam-2015/sessions/mysql-performance-mo
 Contributors
 ------------
 
-~~spil-jasper~~
+spil-jasper **no longer works on the project**
 
-~~thijsdezoete~~
+thijsdezoete **no longer works on the project**
 
-~~art-spilgames~~
+art-spilgames **now known as banpei-dbart**
 
 banpei-dbart
 
+bnkr

mysql_statsd/mysql_statsd.py

@@ -12,7 +12,7 @@
 
 from thread_manager import ThreadManager
 from thread_mysql import ThreadMySQL
-from thread_statsd import ThreadStatsd
+from thread_statsd import ThreadStatsd, ThreadFakeStatsd
 
 
 class MysqlStatsd():
@@ -23,8 +23,19 @@ class MysqlStatsd():
     def __init__(self):
         """Program entry point"""
         op = argparse.ArgumentParser()
-        op.add_argument("-c", "--config", dest="cfile", default="/etc/mysql-statsd.conf", help="Configuration file")
-        op.add_argument("-d", "--debug", dest="debug", help="Debug mode", default=False, action="store_true")
+        op.add_argument("-c", "--config", dest="cfile",
+                default="/etc/mysql-statsd.conf",
+                help="Configuration file"
+        )
+        op.add_argument("-d", "--debug", dest="debug",
+                help="Prints statsd metrics next to sending them",
+                default=False, action="store_true"
+        )
+        op.add_argument("--dry-run", dest="dry_run",
+                default=False,
+                action="store_true",
+                help="Print the output that would be sent to statsd without actually sending data somewhere"
+        )
 
         # TODO switch the default to True, and make it fork by default in init script.
         op.add_argument("-f", "--foreground", dest="foreground", help="Dont fork main program", default=False, action="store_true")
@@ -60,6 +71,13 @@ def __init__(self):
         # Spawn Statsd flushing thread
         statsd_thread = ThreadStatsd(queue=self.queue, **statsd_config)
 
+        if opt.dry_run:
+            statsd_thread = ThreadFakeStatsd(queue=self.queue, **statsd_config)
+
+        if opt.debug:
+            """ All debug settings go here """
+            statsd_thread.debug = True
+
         # Get thread manager
         tm = ThreadManager(threads=[mysql_thread, statsd_thread])
         tm.run()

mysql_statsd/preprocessors/innodb_preprocessor.py

@@ -67,7 +67,7 @@ def process(self, rows):
 
         # Process the individual buffer pool
         bufferpool = 'bufferpool_0.'
-        for line in chunks['INDIVIDUAL BUFFER POOL INFO']:
+        for line in chunks.get('INDIVIDUAL BUFFER POOL INFO', []):
             # Buffer pool stats are preceded by:
             # ---BUFFER POOL X
             if line.startswith('---'):

mysql_statsd/thread_statsd.py

@@ -25,6 +25,8 @@ def run(self):
 
 
 class ThreadStatsd(ThreadBase):
+    debug = False
+
     def configure(self, config):
         host = config.get('host', 'localhost')
         port = int(config.get('port', 8125))
@@ -73,11 +75,19 @@ def run(self):
             try:
                 # Timeout after 1 second so we can respond to quit events
                 item = self.queue.get(True, 1)
+                if self.debug:
+                    print(item)
                 self.send_stat(item)
             except Queue.Empty:
                 continue
 
 
+class ThreadFakeStatsd(ThreadStatsd):
+    """Prints metrics instead of sending them to statsd."""
+    def send_stat(self, item):
+        print item
+
+
 if __name__ == '__main__':
     # Run standalone to test this module, it will generate garbage
     from thread_manager import ThreadManager

setup.py

@@ -19,12 +19,12 @@
 
 setup(
     name='mysql-statsd',
-    version='0.1.2',
+    version='0.1.4',
     description='Daemon that gathers statistics from MySQL and sends them to statsd.',
     long_description=readme + '\n\n' + history,
-    author='Jasper Capel, Thijs de Zoete',
-    author_email='jasper.capel@spilgames.com',
-    url='https://github.com/spilgames/mysql_statsd',
+    author='Art van Scheppingen',
+    author_email='art@dbart.net',
+    url='https://github.com/db-art/mysql-statsd',
     packages=[
         'mysql_statsd',
     ],

tests/__init__.py

@@ -1,2 +1,62 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+import unittest
+import os
+from mysql_statsd.preprocessors import InnoDBPreprocessor
+
+class InnoDBPreprocessorTest(unittest.TestCase):
+    def test_values_read_from_vanilla_install(self):
+        """
+        For this mysqld version in default setup::
+
+            $ /usr/sbin/mysqld --version
+            /usr/sbin/mysqld  Ver 5.5.41-0+wheezy1 for debian-linux-gnu on x86_64 ((Debian))
+
+        A basic coverage test.
+        """
+        fixture = os.path.join(os.path.dirname(__file__), 'fixtures',
+                               'show-innodb-status-5.5-vanilla')
+        row = open(fixture, 'rb').read()
+
+        processor = InnoDBPreprocessor()
+        processed = processor.process([('InnoDB', '', row)])
+        expected = {
+                'additional_pool_alloc': '0',
+                'current_transactions': 1,
+                'database_pages': '142',
+                'empty': '',
+                'free_pages': '8049',
+                'hash_index_cells_total': '276671',
+                'hash_index_cells_used': 0,
+                'history_list': '0',
+                'ibuf_cell_count': '2',
+                'ibuf_free_cells': '0',
+                'ibuf_merges': '0',
+                'ibuf_used_cells': '1',
+                'innodb_transactions': 1282,
+                'last_checkpoint': '1595685',
+                'log_bytes_flushed': '1595685',
+                'log_bytes_written': '1595685',
+                'modified_pages': '0',
+                'os_waits': '0',
+                'pages_created': '0',
+                'pages_read': '142',
+                'pages_written': '1',
+                'pending_buf_pool_flushes': '0',
+                'pending_log_flushes': '0',
+                'pending_normal_aio_reads': '0',
+                'pending_normal_aio_writes': '0',
+                'pool_size': '8191',
+                'read_views': '1',
+                'rows_deleted': '0',
+                'rows_inserted': '0',
+                'rows_read': '0',
+                'rows_updated': '0',
+                'spin_rounds': '0',
+                'spin_waits': '0',
+                'total_mem_alloc': '137363456',
+                'unpurged_txns': 1282, }
+        self.assertEquals(expected, dict(processed))
+
+if __name__ == "__main__":
+    unittest.main()

tests/fixtures/show-innodb-status-5.5-vanilla

@@ -0,0 +1,95 @@
+=====================================
+150221 12:34:19 INNODB MONITOR OUTPUT
+=====================================
+Per second averages calculated from the last 16 seconds
+-----------------
+BACKGROUND THREAD
+-----------------
+srv_master_thread loops: 2 1_second, 2 sleeps, 0 10_second, 3 background, 3 flush
+srv_master_thread log flush and writes: 3
+----------
+SEMAPHORES
+----------
+OS WAIT ARRAY INFO: reservation count 3, signal count 3
+Mutex spin waits 0, rounds 0, OS waits 0
+RW-shared spins 3, rounds 90, OS waits 3
+RW-excl spins 0, rounds 0, OS waits 0
+Spin rounds per wait: 0.00 mutex, 30.00 RW-shared, 0.00 RW-excl
+------------
+TRANSACTIONS
+------------
+Trx id counter 502
+Purge done for trx's n:o < 0 undo n:o < 0
+History list length 0
+LIST OF TRANSACTIONS FOR EACH SESSION:
+---TRANSACTION 0, not started
+MySQL thread id 48, OS thread handle 0x7fc0ebabd700, query id 694 localhost root
+show engine innodb status
+--------
+FILE I/O
+--------
+I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
+I/O thread 1 state: waiting for completed aio requests (log thread)
+I/O thread 2 state: waiting for completed aio requests (read thread)
+I/O thread 3 state: waiting for completed aio requests (read thread)
+I/O thread 4 state: waiting for completed aio requests (read thread)
+I/O thread 5 state: waiting for completed aio requests (read thread)
+I/O thread 6 state: waiting for completed aio requests (write thread)
+I/O thread 7 state: waiting for completed aio requests (write thread)
+I/O thread 8 state: waiting for completed aio requests (write thread)
+I/O thread 9 state: waiting for completed aio requests (write thread)
+Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
+ ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
+Pending flushes (fsync) log: 0; buffer pool: 0
+153 OS file reads, 7 OS file writes, 7 OS fsyncs
+0.00 reads/s, 0 avg bytes/read, 0.00 writes/s, 0.00 fsyncs/s
+-------------------------------------
+INSERT BUFFER AND ADAPTIVE HASH INDEX
+-------------------------------------
+Ibuf: size 1, free list len 0, seg size 2, 0 merges
+merged operations:
+ insert 0, delete mark 0, delete 0
+discarded operations:
+ insert 0, delete mark 0, delete 0
+Hash table size 276671, node heap has 0 buffer(s)
+0.00 hash searches/s, 0.00 non-hash searches/s
+---
+LOG
+---
+Log sequence number 1595685
+Log flushed up to   1595685
+Last checkpoint at  1595685
+0 pending log writes, 0 pending chkp writes
+10 log i/o's done, 0.00 log i/o's/second
+----------------------
+BUFFER POOL AND MEMORY
+----------------------
+Total memory allocated 137363456; in additional pool allocated 0
+Dictionary memory allocated 33650
+Buffer pool size   8191
+Free buffers       8049
+Database pages     142
+Old database pages 0
+Modified db pages  0
+Pending reads 0
+Pending writes: LRU 0, flush list 0, single page 0
+Pages made young 0, not young 0
+0.00 youngs/s, 0.00 non-youngs/s
+Pages read 142, created 0, written 1
+0.00 reads/s, 0.00 creates/s, 0.00 writes/s
+No buffer pool page gets since the last printout
+Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
+LRU len: 142, unzip_LRU len: 0
+I/O sum[0]:cur[0], unzip sum[0]:cur[0]
+--------------
+ROW OPERATIONS
+--------------
+0 queries inside InnoDB, 0 queries in queue
+1 read views open inside InnoDB
+Main thread process no. 8651, id 140466251474688, state: waiting for server activity
+Number of rows inserted 0, updated 0, deleted 0, read 0
+0.00 inserts/s, 0.00 updates/s, 0.00 deletes/s, 0.00 reads/s
+----------------------------
+END OF INNODB MONITOR OUTPUT
+============================
+