[Integration] Add a Twemproxy integration

dd-official-check/README.md

@@ -1,2 +1,38 @@
-# dd-official-check
-some prototyping
+About this readme document: The README.md file should be formatted in valid markdown. All of the main sections listed below should be H1's. They should be the only H1's in the document otherwise they will not be used.
+
+# Overview
+**Absolutely Required.**
+
+The overview section is required and should be a paragraph or two with some bullets of what is interesting about this integration. For example, the following comes from the Docker integration.
+
+Get metrics from Docker in real time to:
+
+* Visualize your containers' performance.
+* Correlate the performance of containers with the applications running inside.
+
+There are three ways to setup the Docker integration: install the agent on the host, on a single priviledged container, and on each individual container.
+
+# Installation
+**Required with some exceptions**
+
+The installation section should cover anything that needs to be installed on the agent host. For instance, in the Docker installation section you learn about installing the agent into a container. If there is nothing to install on the agent host, this section can be left out. To be a complete integration, either an installation section or a configuration section must be included.
+
+# Configuration
+**Required with some exceptions**
+
+The configuration section should cover anything that you can configure in the Datadog interface or the agent configuration files. In almost every case this section should be included since there is almost always something to configure. To be a complete integration, either an installation section or a configuration section must be included.
+
+# Validation
+**Required**
+
+The validation section should include instructions on how to validate that the integration is successfully working.
+
+# Troubleshooting
+**Optional**
+
+The troubleshooting section should include anything that answers a question a user might have about the integration. If there is a question that comes up in support about the integration, it should be added here.
+
+# Compatibility
+**Required**
+
+The compatibility section should include which versions this integration has been tested and validated on.

twemproxy/README.md

@@ -0,0 +1,27 @@
+# Overview
+
+Get metrics from Twitter's twemproxy in real time to:
+
+* Visualize client and server connectivity
+* Correlate performance of the proxy to the Redis or Memcached server behind it
+
+# Installation
+
+Install the integration using `apt-get install dd-check-twemproxy`
+
+# Configuration
+
+Edit the `twemproxy.yaml` file to point to your server and port
+
+# Validation
+
+When you run `datadog-agent info` you should see something like the following:
+
+    Checks
+    ======
+
+        twemproxy
+        ---------
+          - instance #0 [OK]
+          - Collected 17 metrics, 0 events & 1 service check
+

twemproxy/check.py

@@ -0,0 +1,132 @@
+# stdlib
+import urllib2
+
+# 3rd party
+import simplejson as json
+
+# project
+from checks import AgentCheck
+
+GLOBAL_STATS = set([
+    'curr_connections',
+    'total_connections'
+])
+
+POOL_STATS = set([
+    'client_eof',
+    'client_err',
+    'client_connections',
+    'server_ejects',
+    'forward_error'
+])
+
+SERVER_STATS = set([
+    'in_queue',
+    'out_queue',
+    'server_connections',
+    'server_err',
+    'server_timedout',
+    'server_eof'
+])
+
+
+class Twemproxy(AgentCheck):
+    """Tracks twemproxy metrics via the stats monitoring port
+
+    Connects to twemproxy via the configured stats port.
+    See https://github.com/twitter/twemproxy#observability
+
+    $ curl localhost:22222
+    {
+        "service":"nutcracker",
+        "source":"i-deadbeef",
+        "version":"0.4.1",
+        "uptime":4018,
+        "timestamp":1446677875,
+        "total_connections":244,
+        "curr_connections":23,
+        "poolA": {
+            # pool stats
+            "client_eof":221,
+            "client_err":0,
+            "client_connections":15,
+            "server_ejects":0,
+            "forward_error":0,
+            "fragments":0,
+            "serverA": {
+                # server stats
+                "server_eof":0,
+                "server_err":0,
+                "server_timedout":0,
+                "server_connections":1,
+                "server_ejected_at":0,
+                "requests":46873,
+                "request_bytes":127685831,
+                "responses":46872,
+                "response_bytes":194030,
+                "in_queue":1,
+                "in_queue_bytes":190,
+                "out_queue":0,
+                "out_queue_bytes":0
+            }
+        }
+    }
+
+    """
+    def check(self, instance):
+        if 'url' not in instance:
+            raise Exception('Twemproxy instance missing "url" value.')
+        tags = instance.get('tags', [])
+
+        response = self._get_data(instance)
+        self.log.debug(u"Nginx status `response`: {0}".format(response))
+
+        metrics = self.parse_json(response, tags)
+        for row in metrics:
+            try:
+                name, value, tags = row
+                self.gauge(name, value, tags)
+            except Exception, e:
+                self.log.error(
+                    u'Could not submit metric: %s: %s',
+                    (repr(row), str(e))
+                )
+
+    def _get_data(self, instance):
+        url = instance.get('url')
+
+        self.log.debug(u"Querying URL: {0}".format(url))
+        response = urllib2.urlopen(url)
+        return response.read()
+
+    @classmethod
+    def parse_json(cls, raw, tags=None):
+        if tags is None:
+            tags = []
+        parsed = json.loads(raw)
+        metric_base = 'twemproxy'
+        output = []
+
+        for key, val in parsed.iteritems():
+            if isinstance(val, dict):
+                # server pool
+                pool_tags = tags + ['pool:%s' % key]
+                for pool_key, pool_val in val.iteritems():
+                    if isinstance(pool_val, dict):
+                        # server
+                        server_tags = pool_tags + ['server:%s' % pool_key]
+                        for stat in SERVER_STATS:
+                            metric_name = '%s.%s' % (metric_base, stat)
+                            output.append(
+                                (metric_name, pool_val.get(stat), server_tags)
+                            )
+
+                    elif pool_key in POOL_STATS:
+                        metric_name = '%s.%s' % (metric_base, pool_key)
+                        output.append((metric_name, pool_val, pool_tags))
+
+            elif key in GLOBAL_STATS:
+                metric_name = '%s.%s' % (metric_base, key)
+                output.append((metric_name, val, tags))
+
+        return output

twemproxy/conf.yaml.example

@@ -0,0 +1,5 @@
+init_config:
+
+instances:
+    - url: http://localhost:22222
+

twemproxy/manifest.json

@@ -0,0 +1,8 @@
+{
+  "manifest_version": "0.1.0",
+  "name": "twemproxy",
+  "version": "0.1.0",
+  "min_agent_version": "5.6.3",
+  "max_agent_version": "6.0.0",
+  "support": "contrib"
+}

twemproxy/requirements.txt

@@ -0,0 +1 @@
+simplejson

twemproxy/twemproxy_metadata.csv

@@ -0,0 +1,12 @@
+metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name
+twemproxy.client_connections,guage,,,The number of client connections,0,twemproxy,client connections
+twemproxy.client_eof,guage,,,something,0,twemproxy,client eof
+twemproxy.client_err,guage,,,something,0,twemproxy,client err
+twemproxy.forward_error,guage,,,something,0,twemproxy,forward error
+twemproxy.in_queue,guage,,,something,0,twemproxy,in queue
+twemproxy.out_queue,guage,,,something,0,twemproxy,out queue
+twemproxy.server_connections,guage,,,something,0,twemproxy,server connections
+twemproxy.server_ejects,guage,,,something,0,twemproxy,server ejects
+twemproxy.server_eof,guage,,,something,0,twemproxy,server eof
+twemproxy.server_err,guage,,,something,0,twemproxy,server errors
+twemproxy.server_timedout,guage,,,something,0,twemproxy,server timed out