From 0760af71ad4e0bfb92816433d41ad8f0ff93a520 Mon Sep 17 00:00:00 2001
From: Matt Jordan <mjordan@digium.com>
Date: Thu, 2 May 2019 19:45:27 -0500
Subject: [PATCH] res_prometheus: Add Asterisk channel metrics
This patch adds basic Asterisk channel statistics to the res_prometheus
module. This includes:
* asterisk_calls_sum: A running sum of the total number of
processed calls
* asterisk_calls_count: The current number of calls
* asterisk_channels_count: The current number of channels
* asterisk_channels_state: The state of any particular channel
* asterisk_channels_duration_seconds: How long a channel has existed,
in seconds
In all cases, enough information is provided with each channel metric
to determine a unique instance of Asterisk that provided the data, as
well as the name, type, unique ID, and - if present - linked ID of each
channel.
ASTERISK-28403
Change-Id: I0db306ec94205d4f58d1e7fbabfe04b185869f59
---
configs/samples/prometheus.conf.sample | 9 +-
include/asterisk/res_prometheus.h | 37 ++++
res/Makefile | 1 +
res/prometheus/channels.c | 236 +++++++++++++++++++++++++
res/prometheus/prometheus_internal.h | 41 +++++
res/res_prometheus.c | 50 +++++-
tests/test_res_prometheus.c | 8 +-
7 files changed, 371 insertions(+), 11 deletions(-)
create mode 100644 res/prometheus/channels.c
create mode 100644 res/prometheus/prometheus_internal.h
diff --git a/configs/samples/prometheus.conf.sample b/configs/samples/prometheus.conf.sample
index 63e9bd6d42..3ee9290db9 100644
--- a/configs/samples/prometheus.conf.sample
+++ b/configs/samples/prometheus.conf.sample
@@ -3,12 +3,9 @@
;
;
-; Note that this configuration file is consumed by res_prometheus, which
-; provides core functionality for serving up Asterisk statistics to a
-; Prometheus server. By default, this only includes basic information about
-; the Asterisk instance that is running. Additional modules can be loaded to
-; provide specific statistics. In all cases, configuration of said statistics
-; is done through this configuration file.
+; This configuration file is consumed by res_prometheus, which
+; provides the functionality for serving up Asterisk statistics to a
+; Prometheus server.
;
; Because Prometheus scrapes statistics from HTTP servers, this module requires
; Asterisk's built-in HTTP server to be enabled and configured properly.
diff --git a/include/asterisk/res_prometheus.h b/include/asterisk/res_prometheus.h
index cf62b7b783..4b6ce16be5 100644
--- a/include/asterisk/res_prometheus.h
+++ b/include/asterisk/res_prometheus.h
@@ -83,6 +83,36 @@ struct prometheus_general_config {
);
};
+/*!
+ * \brief A function table for a metrics provider
+ *
+ * \details
+ * It's generally nice to separate out things that provide metrics
+ * from the core of this module. For those that want to be notified
+ * when things happen in the core module, they can provide an instance
+ * of this function table using \c prometheus_metrics_provider_register
+ * and be notified when module affecting changes occur.
+ */
+struct prometheus_metrics_provider {
+ /*!
+ * \brief Handy name of the provider for debugging purposes
+ */
+ const char *name;
+ /*!
+ * \brief Reload callback
+ *
+ * \param config The reloaded config
+ *
+ * \retval 0 success
+ * \retval -1 error
+ */
+ int (* const reload_cb)(struct prometheus_general_config *config);
+ /*!
+ * \brief Unload callback.
+ */
+ void (* const unload_cb)(void);
+};
+
/*!
* \brief Prometheus metric type
*
@@ -437,6 +467,13 @@ int prometheus_callback_register(struct prometheus_callback *callback);
*/
void prometheus_callback_unregister(struct prometheus_callback *callback);
+/*!
+ * \brief Register a metrics provider
+ *
+ * \param provider The provider function table to register
+ */
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider);
+
/*!
* \brief Retrieve the current configuration of the module
*
diff --git a/res/Makefile b/res/Makefile
index 5ff38ad1b5..78410ad01c 100644
--- a/res/Makefile
+++ b/res/Makefile
@@ -66,6 +66,7 @@ $(call MOD_ADD_C,res_stasis,$(wildcard stasis/*.c))
$(call MOD_ADD_C,res_snmp,snmp/agent.c)
$(call MOD_ADD_C,res_parking,$(wildcard parking/*.c))
$(call MOD_ADD_C,res_pjsip,$(wildcard res_pjsip/*.c))
+$(call MOD_ADD_C,res_prometheus,$(wildcard prometheus/*.c))
$(call MOD_ADD_C,res_ari,ari/cli.c ari/config.c ari/ari_websockets.c)
$(call MOD_ADD_C,res_ari_model,ari/ari_model_validators.c)
$(call MOD_ADD_C,res_stasis_recording,stasis_recording/stored.c)
diff --git a/res/prometheus/channels.c b/res/prometheus/channels.c
new file mode 100644
index 0000000000..97b7519c0d
--- /dev/null
+++ b/res/prometheus/channels.c
@@ -0,0 +1,236 @@
+
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*!
+ * \file
+ * \brief Prometheus Channel Metrics
+ *
+ * \author Matt Jordan <mjordan@digium.com>
+ *
+ */
+
+#include "asterisk.h"
+#include "asterisk/res_prometheus.h"
+#include "asterisk/stasis_channels.h"
+#include "asterisk/pbx.h"
+#include "prometheus_internal.h"
+
+#define CHANNELS_STATE_HELP "Individual channel states. 0=down; 1=reserved; 2=offhook; 3=dialing; 4=ring; 5=ringing; 6=up; 7=busy; 8=dialing_offhook; 9=prering."
+
+#define CHANNELS_DURATION_HELP "Individual channel durations (in seconds)."
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current state
+ *
+ * \param metric The metric to populate
+ * \snapshot Channel snapshot
+ */
+static void get_channel_state(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+ snprintf(metric->value, sizeof(metric->value), "%d", snapshot->state);
+}
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current duration
+ *
+ * \param metric The metric to populate
+ * \param snapshot Channel snapshot
+ */
+static void get_channel_duration(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+ struct timeval now = ast_tvnow();
+ int64_t duration = ast_tvdiff_sec(now, snapshot->base->creationtime);
+
+ snprintf(metric->value, sizeof(metric->value), "%" PRIu64, duration);
+}
+
+/*!
+ * \internal
+ * \brief Helper struct for generating individual channel stats
+ */
+struct channel_metric_defs {
+ /*!
+ * \brief Help text to display
+ */
+ const char *help;
+ /*!
+ * \brief Name of the metric
+ */
+ const char *name;
+ /*!
+ * \brief Callback function to generate a metric value for a given channel
+ */
+ void (* const get_value)(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot);
+} channel_metric_defs[] = {
+ {
+ .help = CHANNELS_STATE_HELP,
+ .name = "asterisk_channels_state",
+ .get_value = get_channel_state,
+ },
+ {
+ .help = CHANNELS_DURATION_HELP,
+ .name = "asterisk_channels_duration_seconds",
+ .get_value = get_channel_duration,
+ },
+};
+
+static void get_total_call_count(struct prometheus_metric *metric)
+{
+ snprintf(metric->value, sizeof(metric->value), "%d", ast_processed_calls());
+}
+
+static void get_current_call_count(struct prometheus_metric *metric)
+{
+ snprintf(metric->value, sizeof(metric->value), "%d", ast_active_calls());
+}
+
+/*!
+ * \internal
+ * \brief Channel based metrics that are always available
+ */
+static struct prometheus_metric global_channel_metrics[] = {
+ PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+ PROMETHEUS_METRIC_COUNTER,
+ "asterisk_calls_sum",
+ "Total call count.",
+ &get_total_call_count
+ ),
+ PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+ PROMETHEUS_METRIC_GAUGE,
+ "asterisk_calls_count",
+ "Current call count.",
+ &get_current_call_count
+ ),
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when Prometheus scrapes the server
+ *
+ * \param response The response to populate with formatted metrics
+ */
+static void channels_scrape_cb(struct ast_str **response)
+{
+ struct ao2_container *channels;
+ struct ao2_iterator it_chans;
+ struct ast_channel_snapshot *snapshot;
+ struct prometheus_metric *channel_metrics;
+ char eid_str[32];
+ int num_channels;
+ int i, j;
+ struct prometheus_metric channel_count = PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+ PROMETHEUS_METRIC_GAUGE,
+ "asterisk_channels_count",
+ "Current channel count.",
+ NULL
+ );
+
+ ast_eid_to_str(eid_str, sizeof(eid_str), &ast_eid_default);
+
+ channels = ast_channel_cache_all();
+ num_channels = ao2_container_count(channels);
+
+ /* Channel count */
+ PROMETHEUS_METRIC_SET_LABEL(&channel_count, 0, "eid", eid_str);
+ snprintf(channel_count.value, sizeof(channel_count.value), "%d", num_channels);
+ prometheus_metric_to_string(&channel_count, response);
+
+ /* Global call values */
+ for (i = 0; i < ARRAY_LEN(global_channel_metrics); i++) {
+ PROMETHEUS_METRIC_SET_LABEL(&global_channel_metrics[i], 0, "eid", eid_str);
+ global_channel_metrics[i].get_metric_value(&global_channel_metrics[i]);
+ prometheus_metric_to_string(&global_channel_metrics[i], response);
+ }
+
+ if (num_channels == 0) {
+ ao2_ref(channels, -1);
+ return;
+ }
+
+ /* Channel dependent values */
+ channel_metrics = ast_calloc(ARRAY_LEN(channel_metric_defs) * num_channels, sizeof(*channel_metrics));
+ if (!channel_metrics) {
+ ao2_ref(channels, -1);
+ return;
+ }
+
+ it_chans = ao2_iterator_init(channels, 0);
+ for (i = 0; (snapshot = ao2_iterator_next(&it_chans)); ao2_ref(snapshot, -1), i++) {
+ for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+ int index = i * ARRAY_LEN(channel_metric_defs) + j;
+
+ channel_metrics[index].type = PROMETHEUS_METRIC_GAUGE;
+ ast_copy_string(channel_metrics[index].name, channel_metric_defs[j].name, sizeof(channel_metrics[index].name));
+ channel_metrics[index].help = channel_metric_defs[j].help;
+ PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 0, "eid", eid_str);
+ PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 1, "name", (snapshot->base->name));
+ PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 2, "id", (snapshot->base->uniqueid));
+ PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 3, "type", (snapshot->base->type));
+ if (snapshot->peer) {
+ PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 4, "linkedid", (snapshot->peer->linkedid));
+ }
+ channel_metric_defs[j].get_value(&channel_metrics[index], snapshot);
+
+ if (i > 0) {
+ AST_LIST_INSERT_TAIL(&channel_metrics[j].children, &channel_metrics[index], entry);
+ }
+ }
+ }
+ ao2_iterator_destroy(&it_chans);
+
+ for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+ prometheus_metric_to_string(&channel_metrics[j], response);
+ }
+
+ ast_free(channel_metrics);
+ ao2_ref(channels, -1);
+}
+
+struct prometheus_callback channels_callback = {
+ .name = "Channels callback",
+ .callback_fn = channels_scrape_cb,
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when the core module is unloaded
+ */
+static void channel_metrics_unload_cb(void)
+{
+ prometheus_callback_unregister(&channels_callback);
+}
+
+/*!
+ * \internal
+ * \brief Metrics provider definition
+ */
+static struct prometheus_metrics_provider provider = {
+ .name = "channels",
+ .unload_cb = channel_metrics_unload_cb,
+};
+
+int channel_metrics_init(void)
+{
+ prometheus_metrics_provider_register(&provider);
+ prometheus_callback_register(&channels_callback);
+
+ return 0;
+}
\ No newline at end of file
diff --git a/res/prometheus/prometheus_internal.h b/res/prometheus/prometheus_internal.h
new file mode 100644
index 0000000000..06cff989e2
--- /dev/null
+++ b/res/prometheus/prometheus_internal.h
@@ -0,0 +1,41 @@
+/*
+ * Prometheus Internal API
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+#ifndef PROMETHEUS_INTERNAL_H__
+#define PROMETHEUS_INTERNAL_H__
+
+/*!
+ * \file prometheus_internal
+ *
+ * \brief Prometheus Metric Internal API
+ *
+ * This module provides internal APIs for \file res_prometheus.
+ * It should not be used outsize of that module, and should
+ * typically only provide intialization functions for units that
+ * want to register metrics / handlers with the core API.
+ */
+
+/*!
+ * \brief Initialize channel metrics
+ *
+ * \retval 0 success
+ * \retval -1 error
+ */
+int channel_metrics_init(void);
+
+#endif /* #define PROMETHEUS_INTERNAL_H__ */
diff --git a/res/res_prometheus.c b/res/res_prometheus.c
index 1f4e635252..4533e1b0da 100644
--- a/res/res_prometheus.c
+++ b/res/res_prometheus.c
@@ -128,6 +128,8 @@
#include "asterisk/buildinfo.h"
#include "asterisk/res_prometheus.h"
+#include "prometheus/prometheus_internal.h"
+
/*! \brief Lock that protects data structures during an HTTP scrape */
AST_MUTEX_DEFINE_STATIC(scrape_lock);
@@ -135,6 +137,8 @@ AST_VECTOR(, struct prometheus_metric *) metrics;
AST_VECTOR(, struct prometheus_callback *) callbacks;
+AST_VECTOR(, const struct prometheus_metrics_provider *) providers;
+
/*! \brief The actual module config */
struct module_config {
/*! \brief General settings */
@@ -812,6 +816,11 @@ static void prometheus_config_post_apply(void)
}
}
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider)
+{
+ AST_VECTOR_APPEND(&providers, provider);
+}
+
static int unload_module(void)
{
SCOPED_MUTEX(lock, &scrape_lock);
@@ -819,6 +828,16 @@ static int unload_module(void)
ast_http_uri_unlink(&prometheus_uri);
+ for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+ const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+ if (!provider->unload_cb) {
+ continue;
+ }
+
+ provider->unload_cb();
+ }
+
for (i = 0; i < AST_VECTOR_SIZE(&metrics); i++) {
struct prometheus_metric *metric = AST_VECTOR_GET(&metrics, i);
@@ -827,7 +846,7 @@ static int unload_module(void)
AST_VECTOR_FREE(&metrics);
AST_VECTOR_FREE(&callbacks);
-
+ AST_VECTOR_FREE(&providers);
aco_info_destroy(&cfg_info);
ao2_global_obj_release(global_config);
@@ -836,11 +855,31 @@ static int unload_module(void)
static int reload_module(void) {
SCOPED_MUTEX(lock, &scrape_lock);
+ int i;
+ struct prometheus_general_config *general_config;
ast_http_uri_unlink(&prometheus_uri);
if (aco_process_config(&cfg_info, 1) == ACO_PROCESS_ERROR) {
return -1;
}
+
+ /* Our config should be all reloaded now */
+ general_config = prometheus_general_config_get();
+ for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+ const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+ if (!provider->reload_cb) {
+ continue;
+ }
+
+ if (provider->reload_cb(general_config)) {
+ ast_log(AST_LOG_WARNING, "Failed to reload metrics provider %s\n", provider->name);
+ ao2_ref(general_config, -1);
+ return -1;
+ }
+ }
+ ao2_ref(general_config, -1);
+
if (ast_http_uri_link(&prometheus_uri)) {
ast_log(AST_LOG_WARNING, "Failed to re-register Prometheus Metrics URI during reload\n");
return -1;
@@ -861,6 +900,10 @@ static int load_module(void)
goto cleanup;
}
+ if (AST_VECTOR_INIT(&providers, 8)) {
+ goto cleanup;
+ }
+
if (aco_info_init(&cfg_info)) {
goto cleanup;
}
@@ -874,6 +917,10 @@ static int load_module(void)
goto cleanup;
}
+ if (channel_metrics_init()) {
+ goto cleanup;
+ }
+
if (ast_http_uri_link(&prometheus_uri)) {
goto cleanup;
}
@@ -885,6 +932,7 @@ cleanup:
aco_info_destroy(&cfg_info);
AST_VECTOR_FREE(&metrics);
AST_VECTOR_FREE(&callbacks);
+ AST_VECTOR_FREE(&providers);
return AST_MODULE_LOAD_DECLINE;
}
diff --git a/tests/test_res_prometheus.c b/tests/test_res_prometheus.c
index 01279bed72..2719267d08 100644
--- a/tests/test_res_prometheus.c
+++ b/tests/test_res_prometheus.c
@@ -173,13 +173,13 @@ AST_TEST_DEFINE(metric_values)
}
ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
- ast_test_validate_cleanup(test, strcmp(ast_str_buffer(buffer),
+ ast_test_validate_cleanup(test, strstr(ast_str_buffer(buffer),
"# HELP test_counter_one A test counter\n"
"# TYPE test_counter_one counter\n"
"test_counter_one 1\n"
"# HELP test_counter_two A test counter\n"
"# TYPE test_counter_two counter\n"
- "test_counter_two 2\n") == 0, result, metric_values_cleanup);
+ "test_counter_two 2\n") != NULL, result, metric_values_cleanup);
metric_values_cleanup:
prometheus_metric_unregister(&test_counter_one);
@@ -247,10 +247,10 @@ AST_TEST_DEFINE(metric_callback_register)
}
ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
- ast_test_validate(test, strcmp(ast_str_buffer(buffer),
+ ast_test_validate(test, strstr(ast_str_buffer(buffer),
"# HELP test_counter A test counter\n"
"# TYPE test_counter counter\n"
- "test_counter 0\n") == 0);
+ "test_counter 0\n") != NULL);
prometheus_callback_unregister(&callback);
--
GitLab