1 /*******************************************************************************
2 
3     Contains methods that collect stats from primitive data members of structs
4     or classes to respond to Prometheus queries with.
5 
6     The metric designs specified by Prometheus
7     (`https://prometheus.io/docs/concepts/metric_types/`) have not been
8     implemented yet. So, as of now, this collector can process only primitive
9     data-type members from a struct or a class. However, the current Prometheus
10     stat collection framework could be enhanced to support metrics with a very
11     little effort.
12 
13     In Prometheus' data model, the stats that are measured are called Metrics,
14     and the dimensions along which stats are measured are called Labels.
15 
16     Metrics can be any measurable value, e.g., CPU or memory consumption.
17 
18     Labels resemble key-value pairs, where the key is referred to as a label's
19     name, and the value as a label's value. A label name would refer to the name
20     of a dimension across which we want to measure stats. Correspondingly, a
21     label value would refer to a point along the said dimension. A stat can have
22     more than one label, if it is intended to be measured across multiple
23     dimensions.
24 
25     Stats with labels look like the following example
26     `
27     promhttp_metric_handler_requests_total{code="200"} 3
28     promhttp_metric_handler_requests_total{code="500"} 0
29     promhttp_metric_handler_requests_total{code="503"} 0
30     `
31     Here `promhttp_metric_handler_requests_total` is the metric, `code` is
32     the label name, `"200"`, `"500"` and `"503"` are the label values, and `3`,
33     `0` and `0` are the respective metric values.
34 
35     On the data visualization side of Prometheus, fetching stats using queries
36     is analogous to calling functions. A metric name is analogous to a function
37     name, and a label is analogous to a function parameter. Continuing with the
38     above example, the following query will return `3`:
39     `
40     promhttp_metric_handler_requests_total {code="200"}
41     `
42 
43     (For detailed information about Prometheus-specific terminology, e.g.,
44     Metrics and Labels, please refer to
45     `https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels`.)
46 
47     This module contains a class which can be used to collect metrics as well as
48     labels from composite-type values, and format them in a way acceptable for
49     Prometheus.
50 
51     Copyright:
52         Copyright (c) 2019 dunnhumby Germany GmbH.
53         All rights reserved
54 
55     License:
56         Boost Software License Version 1.0. See LICENSE.txt for details.
57 
58 *******************************************************************************/
59 
60 module ocean.util.prometheus.collector.Collector;
61 
62 /*******************************************************************************
63 
64     This class provides methods to collect metrics with no label, one label, or
65     multiple labels, using overloaded definitions of the `collect` method.
66 
67     Once the desired stats have been collected, the accumulated response
68     string can be collected using the `getCollection` method.
69 
70     Additionally, the `reset` method can be used to reset stat
71     collection, when the desired stats have been collected from a Collector
72     instance.
73 
74 *******************************************************************************/
75 
76 public class Collector
77 {
78     import core.stdc.time;
79     import Traits = ocean.core.Traits;
80     import ocean.math.IEEE;
81     import ocean.text.convert.Formatter;
82     import ocean.transition;
83     import StatFormatter = ocean.util.prometheus.collector.StatFormatter;
84 
85     /// A buffer used for storing collected stats. Is cleared when the `reset`
86     /// method is called.
87     private mstring collect_buf;
88 
89     /***************************************************************************
90 
91         Returns:
92             The stats collected since the last call to the `reset` method, in a
93             textual respresentation that can be readily added to a response
94             message body.
95             The specifications of the format of the collected stats can be found
96             at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
97 
98     ***************************************************************************/
99 
100     public cstring getCollection ( )
101     {
102         return this.collect_buf;
103     }
104 
105     /// Reset the length of the stat collection buffer to 0.
106     public void reset ( )
107     {
108         this.collect_buf.length = 0;
109         enableStomping(this.collect_buf);
110     }
111 
112     /***************************************************************************
113 
114         Collect stats from the data members of a struct or a class and prepare
115         them to be fetched upon the next call to `getCollection`. The
116         specifications of the format of the collected stats can be found at
117         `https://prometheus.io/docs/instrumenting/exposition_formats/`.
118 
119         Params:
120             ValuesT = The struct or class type to fetch the stat names from.
121             values  = The struct or class to fetch stat values from.
122 
123     ***************************************************************************/
124 
125     public void collect ( ValuesT ) ( ValuesT values )
126     {
127         static assert (is(ValuesT == struct) || is(ValuesT == class),
128             "'values' parameter must be a struct or a class.");
129 
130         StatFormatter.formatStats(values, this.collect_buf);
131     }
132 
133     /***************************************************************************
134 
135         Collect stats from the data members of a struct or a class, annotate
136         them with a given label name and value, and prepare them to be fetched
137         upon the next call to `getCollection`.
138         The specifications of the format of the collected stats can be found
139         at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
140 
141         Params:
142             LabelName = The name of the label to annotate the stats with.
143             ValuesT   = The struct or class type to fetch the stat names from.
144             LabelT    = The type of the label's value.
145             values    = The struct or class to fetch stat values from.
146             label_val = The label value to annotate the stats with.
147 
148     ***************************************************************************/
149 
150     public void collect ( istring LabelName, ValuesT, LabelT ) (
151         ValuesT values, LabelT label_val )
152     {
153         static assert (is(ValuesT == struct) || is(ValuesT == class),
154             "'values' parameter must be a struct or a class.");
155 
156         StatFormatter.formatStats!(LabelName)(values, label_val,
157             this.collect_buf);
158     }
159 
160     /***************************************************************************
161 
162         Collect stats from the data members of a struct or a class, annotate
163         them with labels from the data members of another struct or class, and
164         prepare them to be fetched upon the next call to `getCollection`.
165         The specifications of the format of the collected stats can be found
166         at `https://prometheus.io/docs/instrumenting/exposition_formats/`.
167 
168         Params:
169             ValuesT = The struct or class type to fetch the stat names from.
170             LabelsT = The struct or class type to fetch the label names from.
171             values  = The struct or class to fetch stat values from.
172             labels  = The struct or class holding the label values to annotate
173                       the stats with.
174 
175     ***************************************************************************/
176 
177     public void collect ( ValuesT, LabelsT ) ( ValuesT values, LabelsT labels )
178     {
179         static assert (is(ValuesT == struct) || is(ValuesT == class),
180             "'values' parameter must be a struct or a class.");
181         static assert (is(LabelsT == struct) || is(LabelsT == class),
182             "'labels' parameter must be a struct or a class.");
183 
184         StatFormatter.formatStats(values, labels, this.collect_buf);
185     }
186 }
187 
188 version (UnitTest)
189 {
190     import ocean.core.Test;
191     import ocean.transition;
192 
193     struct Statistics
194     {
195         ulong up_time_s;
196         size_t count;
197         float ratio;
198         double fraction;
199         real very_real;
200 
201         // The following should not be collected as stats
202         int delegate ( int ) a_delegate;
203         void function ( ) a_function;
204     }
205 
206     struct Labels
207     {
208         hash_t id;
209         cstring job;
210         float perf;
211     }
212 }
213 
214 /// Test collecting populated stats, but without any label.
215 unittest
216 {
217     auto test_collector = new Collector();
218     test_collector.collect(Statistics(3600, 347, 3.14, 6.023, 0.43));
219 
220     test!("==")(test_collector.getCollection(),
221         "up_time_s 3600\ncount 347\nratio 3.14\nfraction 6.023\n" ~
222         "very_real 0.43\n");
223 }
224 
225 /// Test collecting populated stats with one label
226 unittest
227 {
228     auto test_collector = new Collector();
229     test_collector.collect!("id")(
230         Statistics(3600, 347, 3.14, 6.023, 0.43), 123.034);
231 
232     test!("==")(test_collector.getCollection(),
233         "up_time_s {id=\"123.034\"} 3600\ncount {id=\"123.034\"} 347\n" ~
234         "ratio {id=\"123.034\"} 3.14\nfraction {id=\"123.034\"} 6.023\n" ~
235         "very_real {id=\"123.034\"} 0.43\n");
236 }
237 
238 /// Test collecting stats having initial values with multiple labels
239 unittest
240 {
241     auto test_collector = new Collector();
242     test_collector.collect(Statistics(3600, 347, 3.14, 6.023, 0.43),
243         Labels(1_235_813, "ocean", 3.14159));
244 
245     test!("==")(test_collector.getCollection(),
246         "up_time_s {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 3600\n" ~
247         "count {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 347\n" ~
248         "ratio {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 3.14\n" ~
249         "fraction {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 6.023\n" ~
250         "very_real {id=\"1235813\",job=\"ocean\",perf=\"3.14159\"} 0.43\n");
251 }
252 
253 /// Test resetting collected stats
254 unittest
255 {
256     auto test_collector = new Collector();
257     test_collector.collect!("id")(Statistics.init, 123);
258 
259     test!("==")(test_collector.getCollection(),
260         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
261         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
262         "very_real {id=\"123\"} 0\n");
263 
264     test_collector.reset();
265 
266     test!("==")(test_collector.getCollection(), "");
267 }
268 
269 // Test collecting stats having initial values, but without any label.
270 unittest
271 {
272     auto test_collector = new Collector();
273     test_collector.collect(Statistics.init);
274 
275     test!("==")(test_collector.getCollection(),
276         "up_time_s 0\ncount 0\nratio 0\nfraction 0\nvery_real 0\n");
277 }
278 
279 // Test collecting stats having initial values with one label
280 unittest
281 {
282     auto test_collector = new Collector();
283     test_collector.collect!("id")(Statistics.init, 123);
284 
285     test!("==")(test_collector.getCollection(),
286         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
287         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
288         "very_real {id=\"123\"} 0\n");
289 }
290 
291 // Test collecting stats having initial values with multiple labels
292 unittest
293 {
294     auto test_collector = new Collector();
295     test_collector.collect(Statistics.init, Labels.init);
296 
297     test!("==")(test_collector.getCollection(),
298         "up_time_s {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
299         "count {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
300         "ratio {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
301         "fraction {id=\"0\",job=\"\",perf=\"0\"} 0\n" ~
302         "very_real {id=\"0\",job=\"\",perf=\"0\"} 0\n");
303 }
304 
305 // Test accumulation of collected stats
306 unittest
307 {
308     auto test_collector = new Collector();
309     test_collector.collect!("id")(Statistics.init, 123);
310 
311     test!("==")(test_collector.getCollection(),
312         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
313         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
314         "very_real {id=\"123\"} 0\n");
315 
316     test_collector.collect!("id")(
317         Statistics(3600, 347, 3.14, 6.023, 0.43), 123.034);
318 
319     test!("==")(test_collector.getCollection(),
320         "up_time_s {id=\"123\"} 0\ncount {id=\"123\"} 0\n" ~
321         "ratio {id=\"123\"} 0\nfraction {id=\"123\"} 0\n" ~
322         "very_real {id=\"123\"} 0\n" ~
323 
324         "up_time_s {id=\"123.034\"} 3600\ncount {id=\"123.034\"} 347\n" ~
325         "ratio {id=\"123.034\"} 3.14\nfraction {id=\"123.034\"} 6.023\n" ~
326         "very_real {id=\"123.034\"} 0.43\n");
327 }