1 /*******************************************************************************
2 
3     Contains methods that collect stats from primitive data members of structs
4     or classes to respond to Prometheus queries with.
5 
6     The metric designs specified by Prometheus
7     (`https://prometheus.io/docs/concepts/metric_types/`) have not been
8     implemented yet. So, as of now, this collector can process only primitive
9     data-type members from a struct or a class. However, the current Prometheus
10     stat collection framework could be enhanced to support metrics with a very
11     little effort.
12 
13     In Prometheus' data model, the stats that are measured are called Metrics,
14     and the dimensions along which stats are measured are called Labels.
15 
16     Metrics can be any measurable value, e.g., CPU or memory consumption.
17 
18     Labels resemble key-value pairs, where the key is referred to as a label's
19     name, and the value as a label's value. A label name would refer to the name
20     of a dimension across which we want to measure stats. Correspondingly, a
21     label value would refer to a point along the said dimension. A stat can have
22     more than one label, if it is intended to be measured across multiple
23     dimensions.
24 
25     Stats with labels look like the following example
26     `
27     promhttp_metric_handler_requests_total{code="200"} 3
28     promhttp_metric_handler_requests_total{code="500"} 0
29     promhttp_metric_handler_requests_total{code="503"} 0
30     `
31     Here `promhttp_metric_handler_requests_total` is the metric, `code` is
32     the label name, `"200"`, `"500"` and `"503"` are the label values, and `3`,
33     `0` and `0` are the respective metric values.
34 
35     On the data visualization side of Prometheus, fetching stats using queries
36     is analogous to calling functions. A metric name is analogous to a function
37     name, and a label is analogous to a function parameter. Continuing with the
38     above example, the following query will return `3`:
39     `
40     promhttp_metric_handler_requests_total {code="200"}
41     `
42 
43     (For detailed information about Prometheus-specific terminology, e.g.,
44     Metrics and Labels, please refer to
45     `https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels`.)
46 
47     This module contains a class which can be used to collect metrics as well as
48     labels from composite-type values, and format them in a way acceptable for
49     Prometheus.
50 
51     Copyright:
52         Copyright (c) 2019 dunnhumby Germany GmbH.
53         All rights reserved
54 
55     License:
56         Boost Software License Version 1.0. See LICENSE.txt for details.
57 
58 *******************************************************************************/
59 
60 module ocean.util.prometheus.collector.Collector;
61 
62 /*******************************************************************************
63 
64     This class provides methods to collect metrics with no label, one label, or
65     multiple labels, using overloaded definitions of the `collect` method.
66 
67     Once the desired stats have been collected, the accumulated response
68     string can be collected using the `getCollection` method.
69 
70     Additionally, the `reset` method can be used to reset stat
71     collection, when the desired stats have been collected from a Collector
72     instance.
73 
74 *******************************************************************************/
75 
76 public class Collector
77 {
78     import core.stdc.time;
79     import ocean.math.IEEE;
80     import ocean.text.convert.Formatter;
81     import ocean.meta.types.Qualifiers;
82     import StatFormatter = ocean.util.prometheus.collector.StatFormatter;
83 
84     /// A buffer used for storing collected stats. Is cleared when the `reset`
85     /// method is called.
86     private mstring collect_buf;
87 
88     /***************************************************************************
89 
90         Returns:
91             The stats collected since the last call to the `reset` method, in a
92             textual respresentation that can be readily added to a response
93             message body.
94             The specifications of the format of the collected stats can be found
95             at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
96 
97     ***************************************************************************/
98 
99     public cstring getCollection ( )
100     {
101         return this.collect_buf;
102     }
103 
104     /// Reset the length of the stat collection buffer to 0.
105     public void reset ( )
106     {
107         this.collect_buf.length = 0;
108         assumeSafeAppend(this.collect_buf);
109     }
110 
111     /***************************************************************************
112 
113         Collect stats from the data members of a struct or a class and prepare
114         them to be fetched upon the next call to `getCollection`. The
115         specifications of the format of the collected stats can be found at
116         `https://prometheus.io/docs/instrumenting/exposition_formats/`.
117 
118         Params:
119             ValuesT = The struct or class type to fetch the stat names from.
120             values  = The struct or class to fetch stat values from.
121 
122     ***************************************************************************/
123 
124     public void collect ( ValuesT ) ( ValuesT values )
125     {
126         static assert (is(ValuesT == struct) || is(ValuesT == class),
127             "'values' parameter must be a struct or a class.");
128 
129         StatFormatter.formatStats(values, this.collect_buf);
130     }
131 
132     /***************************************************************************
133 
134         Collect stats from the data members of a struct or a class, annotate
135         them with a given label name and value, and prepare them to be fetched
136         upon the next call to `getCollection`.
137         The specifications of the format of the collected stats can be found
138         at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
139 
140         Params:
141             LabelName = The name of the label to annotate the stats with.
142             ValuesT   = The struct or class type to fetch the stat names from.
143             LabelT    = The type of the label's value.
144             values    = The struct or class to fetch stat values from.
145             label_val = The label value to annotate the stats with.
146 
147     ***************************************************************************/
148 
149     public void collect ( istring LabelName, ValuesT, LabelT ) (
150         ValuesT values, LabelT label_val )
151     {
152         static assert (is(ValuesT == struct) || is(ValuesT == class),
153             "'values' parameter must be a struct or a class.");
154 
155         StatFormatter.formatStats!(LabelName)(values, label_val,
156             this.collect_buf);
157     }
158 
159     /***************************************************************************
160 
161         Collect stats from the data members of a struct or a class, annotate
162         them with labels from the data members of another struct or class, and
163         prepare them to be fetched upon the next call to `getCollection`.
164         The specifications of the format of the collected stats can be found
165         at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
166 
167         Params:
168             ValuesT = The struct or class type to fetch the stat names from.
169             LabelsT = The struct or class type to fetch the label names from.
170             values  = The struct or class to fetch stat values from.
171             labels  = The struct or class holding the label values to annotate
172                       the stats with.
173 
174     ***************************************************************************/
175 
176     public void collect ( ValuesT, LabelsT ) ( ValuesT values, LabelsT labels )
177     {
178         static assert (is(ValuesT == struct) || is(ValuesT == class),
179             "'values' parameter must be a struct or a class.");
180         static assert (is(LabelsT == struct) || is(LabelsT == class),
181             "'labels' parameter must be a struct or a class.");
182 
183         StatFormatter.formatStats(values, labels, this.collect_buf);
184     }
185 }
186 
187 version (unittest)
188 {
189     import ocean.core.Test;
190     import ocean.meta.types.Qualifiers;
191 
192     struct Statistics
193     {
194         ulong up_time_s;
195         size_t count;
196         float ratio;
197         double fraction;
198         real very_real;
199 
200         // The following should not be collected as stats
201         int delegate ( int ) a_delegate;
202         void function ( ) a_function;
203     }
204 
205     struct Labels
206     {
207         hash_t id;
208         cstring job;
209         float perf;
210     }
211 }
212 
213 /// Test collecting populated stats, but without any label.
214 unittest
215 {
216     auto test_collector = new Collector();
217     test_collector.collect(Statistics(3600, 347, 3.14, 6.023, 0.43));
218 
219     test!("==")(test_collector.getCollection(),
220         "up_time_s 3600\ncount 347\nratio 3.14\nfraction 6.023\n" ~
221         "very_real 0.43\n");
222 }
223 
224 /// Test collecting populated stats with one label
225 unittest
226 {
227     auto test_collector = new Collector();
228     test_collector.collect!("id")(
229         Statistics(3600, 347, 3.14, 6.023, 0.43), 123.034);
230 
231     test!("==")(test_collector.getCollection(),
232         "up_time_s {id=\"123.034\"} 3600\ncount {id=\"123.034\"} 347\n" ~
233         "ratio {id=\"123.034\"} 3.14\nfraction {id=\"123.034\"} 6.023\n" ~
234         "very_real {id=\"123.034\"} 0.43\n");
235 }
236 
237 /// Test collecting stats having initial values with multiple labels
238 unittest
239 {
240     auto test_collector = new Collector();
241     test_collector.collect(Statistics(3600, 347, 3.14, 6.023, 0.43),
242         Labels(1_235_813, "ocean", 3.14159));
243 
244     test!("==")(test_collector.getCollection(),
245         "up_time_s {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 3600\n" ~
246         "count {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 347\n" ~
247         "ratio {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 3.14\n" ~
248         "fraction {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 6.023\n" ~
249         "very_real {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 0.43\n");
250 }
251 
252 /// Test resetting collected stats
253 unittest
254 {
255     auto test_collector = new Collector();
256     test_collector.collect!("id")(Statistics.init, 123);
257 
258     test!("==")(test_collector.getCollection(),
259         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
260         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
261         "very_real {id=\"123\"} 0\n");
262 
263     test_collector.reset();
264 
265     test!("==")(test_collector.getCollection(), "");
266 }
267 
268 // Test collecting stats having initial values, but without any label.
269 unittest
270 {
271     auto test_collector = new Collector();
272     test_collector.collect(Statistics.init);
273 
274     test!("==")(test_collector.getCollection(),
275         "up_time_s 0\ncount 0\nratio 0\nfraction 0\nvery_real 0\n");
276 }
277 
278 // Test collecting stats having initial values with one label
279 unittest
280 {
281     auto test_collector = new Collector();
282     test_collector.collect!("id")(Statistics.init, 123);
283 
284     test!("==")(test_collector.getCollection(),
285         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
286         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
287         "very_real {id=\"123\"} 0\n");
288 }
289 
290 // Test collecting stats having initial values with multiple labels
291 unittest
292 {
293     auto test_collector = new Collector();
294     test_collector.collect(Statistics.init, Labels.init);
295 
296     test!("==")(test_collector.getCollection(),
297         "up_time_s {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
298         "count {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
299         "ratio {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
300         "fraction {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
301         "very_real {id=\"0\",job=\"\",perf=\"0\"} 0\n");
302 }
303 
304 // Test accumulation of collected stats
305 unittest
306 {
307     auto test_collector = new Collector();
308     test_collector.collect!("id")(Statistics.init, 123);
309 
310     test!("==")(test_collector.getCollection(),
311         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
312         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
313         "very_real {id=\"123\"} 0\n");
314 
315     test_collector.collect!("id")(
316         Statistics(3600, 347, 3.14, 6.023, 0.43), 123.034);
317 
318     test!("==")(test_collector.getCollection(),
319         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
320         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
321         "very_real {id=\"123\"} 0\n" ~
322 
323         "up_time_s {id=\"123.034\"} 3600\ncount {id=\"123.034\"} 347\n" ~
324         "ratio {id=\"123.034\"} 3.14\nfraction {id=\"123.034\"} 6.023\n" ~
325         "very_real {id=\"123.034\"} 0.43\n");
326 }