1 /******************************************************************************* 2 3 Classes that can be used to parse the graphite stats. 4 5 Parses an `InputStream` to `StatsLine`. The lines can be iterated using a 6 foreach. Sometimes only the last stat line is important. 7 The `StatsLogReader.last` method will return only that line. 8 9 The `StatsLine` struct parses one stat line. 10 11 Refer to the class' description for information about their actual usage. 12 13 Copyright: 14 Copyright (C) 2018 dunnhumby Germany GmbH. All rights reserved 15 16 License: 17 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details. 18 Alternatively, this file may be distributed under the terms of the Tango 19 3-Clause BSD License (see LICENSE_BSD.txt for details). 20 21 *******************************************************************************/ 22 23 module ocean.util.log.StatsReader; 24 25 import ocean.meta.types.Qualifiers; 26 import ocean.core.Buffer; 27 import ocean.core.Enforce; 28 import ocean.core.array.Search; 29 import ocean.io.model.IConduit; 30 import ocean.io.stream.Iterator; 31 import ocean.io.stream.Lines; 32 33 version (unittest) 34 { 35 import ocean.io.device.Array; 36 import ocean.util.app.DaemonApp; 37 import ocean.core.Test; 38 } 39 40 /************************************************************************** 41 42 Struct that parses a stats line 43 44 **************************************************************************/ 45 46 public struct StatsLine 47 { 48 /************************************************************************** 49 50 The stats after the date and time were removed 51 52 **************************************************************************/ 53 54 private cstring line; 55 56 /************************************************************************** 57 58 The line date 59 60 **************************************************************************/ 61 62 public cstring date; 63 64 /************************************************************************** 65 66 The line time 67 68 **************************************************************************/ 69 70 public cstring time; 71 72 /************************************************************************** 73 74 Create an StatsLine 75 76 Params: 77 line = a line stat 78 79 Returns: 80 A StatsLine 81 82 **************************************************************************/ 83 84 static public StatsLine opCall (cstring line) 85 { 86 enforce(line.length > 0, "Can not parse an empty line"); 87 StatsLine stats_line; 88 auto len = line.length; 89 90 auto date_end_position = find(line, " "); 91 stats_line.date = line[0 .. date_end_position]; 92 93 auto time_begin_position = date_end_position + 1; 94 auto time_end_position = find(line[time_begin_position .. len], " ") + time_begin_position; 95 stats_line.time = line[time_begin_position .. time_end_position]; 96 97 stats_line.line = line[time_end_position .. len]; 98 99 return stats_line; 100 } 101 102 /************************************************************************** 103 104 Returns the value associated to a key 105 106 Params: 107 key = the stats key 108 109 Return: 110 the value associated with the key 111 112 **************************************************************************/ 113 114 public cstring opIndex (cstring key) const 115 { 116 auto position = find(line[], key); 117 118 enforce(position < line.length, idup(key ~ " is not present in the stats")); 119 120 enforce(line[position - 1 .. position] == " ", 121 idup(key ~ " is not present in the stats")); 122 enforce(line[position + key.length .. position + key.length + 1] == ":", 123 idup(key ~ " is not present in the stats")); 124 125 auto rest = line[position .. line.length]; 126 127 auto begin = find(rest, ':') + 1; 128 auto end = find(rest, ' '); 129 130 return rest[begin .. end]; 131 } 132 133 /************************************************************************** 134 135 Copy the structure 136 137 Return: 138 a new copy of the structure 139 140 **************************************************************************/ 141 142 public StatsLine dup() 143 { 144 StatsLine stats_line; 145 146 stats_line.line = this.line.dup; 147 stats_line.date = this.date.dup; 148 stats_line.time = this.time.dup; 149 150 return stats_line; 151 } 152 } 153 154 /// Parsing a valid stat line 155 unittest 156 { 157 auto line = StatsLine("2018-09-12 10:03:07,598 cpu_usage:64.96 memory:330.19"); 158 159 /// It should not extract a missing key 160 testThrown!(Exception)(line["missing_key"]); 161 162 /// It should not extract a key when the name is not fully provided 163 testThrown!(Exception)(line["pu_usage"]); 164 testThrown!(Exception)(line["cpu_usag"]); 165 166 /// Valid indexes 167 test!("==")(line["cpu_usage"], "64.96"); 168 test!("==")(line["memory"], "330.19"); 169 test!("==")(line.date, "2018-09-12"); 170 test!("==")(line.time, "10:03:07,598"); 171 } 172 173 /// Parsing a stat line with missing values 174 unittest 175 { 176 auto line = StatsLine("2018-09-12 10:03:07,598 cpu_usage: memory:"); 177 178 /// It should return an empty string 179 test!("==")(line["cpu_usage"], ""); 180 test!("==")(line["memory"], ""); 181 } 182 183 /// Parsing a stat line with missing space separator 184 unittest 185 { 186 auto line = StatsLine("2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:64.96memory:330.19"); 187 188 /// It should return the value after column 189 test!("==")(line["cpu_usage"], "64.96memory:330.19"); 190 191 /// It should not find the key memory, since is parsed as a value for cpu_usage 192 testThrown!(Exception)(line["memory"]); 193 } 194 195 /// Duplicating a stat line 196 unittest 197 { 198 auto line = StatsLine("2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:64.96 memory:330.19"); 199 auto line_copy = line.dup; 200 201 line.line.length = 0; 202 line.date.length = 0; 203 line.time.length = 0; 204 205 /// It should return an empty string 206 test!("==")(line_copy["cpu_usage"], "64.96"); 207 test!("==")(line_copy.date, "2018-09-12"); 208 test!("==")(line_copy.time, "10:03:07,598"); 209 } 210 211 /****************************************************************************** 212 213 Class that iterarates through a char stream and extracts the StatsLines 214 215 ******************************************************************************/ 216 217 public class StatsLogReader 218 { 219 /************************************************************************** 220 221 Struct used to iterate through stat lines that are not empty 222 223 **************************************************************************/ 224 225 private struct StatLinesIterator 226 { 227 /********************************************************************** 228 229 The line stream 230 231 **********************************************************************/ 232 233 private Lines lines; 234 235 /*************************************************************************** 236 237 Enables 'foreach' iteration over the stat lines. 238 239 Params: 240 dg = delegate called for each argument 241 242 ***************************************************************************/ 243 244 public int opApply ( scope int delegate(ref const(char[])) dg ) 245 { 246 int result; 247 248 foreach (line; this.lines) 249 { 250 if (line.length == 0) 251 { 252 continue; 253 } 254 255 result = dg(line); 256 257 if ( result != 0 ) 258 { 259 break; 260 } 261 } 262 263 return result; 264 } 265 } 266 267 /****************************************************************************** 268 269 Line iterator 270 271 ******************************************************************************/ 272 273 private StatLinesIterator lines; 274 275 /************************************************************************** 276 277 Constructor 278 279 Params: 280 stream = a char stream that contains stats 281 282 **************************************************************************/ 283 284 this (InputStream stream) 285 { 286 this.lines = StatLinesIterator(new Lines(stream)); 287 } 288 289 /*************************************************************************** 290 291 Get the last line from the stats 292 293 Returns: 294 The last line 295 296 ***************************************************************************/ 297 298 public const(StatsLine) last ( ) 299 { 300 auto last_line = Buffer!(char)(); 301 302 foreach (line; this.lines) 303 { 304 last_line = line; 305 } 306 307 enforce(last_line.length > 0, "The stats are empty"); 308 309 const(StatsLine) stats_line = StatsLine(last_line[]); 310 311 return stats_line; 312 } 313 314 315 /*************************************************************************** 316 317 Enables 'foreach' iteration over the stat lines. 318 319 Params: 320 dg = delegate called for each argument 321 322 ***************************************************************************/ 323 324 public int opApply ( scope int delegate(ref const(StatsLine)) dg ) 325 { 326 int result; 327 328 foreach (line; this.lines) 329 { 330 const(StatsLine) stats_line = StatsLine(line); 331 result = dg(stats_line); 332 333 if ( result != 0 ) 334 { 335 break; 336 } 337 } 338 339 return result; 340 } 341 342 /****************************************************************************** 343 344 ditto 345 346 ******************************************************************************/ 347 348 public int opApply ( scope int delegate(ref size_t index, ref const(StatsLine)) dg ) 349 { 350 int result; 351 size_t index; 352 353 foreach (line; this.lines) 354 { 355 const(StatsLine) stats_line = StatsLine(line); 356 result = dg(index, stats_line); 357 index++; 358 359 if ( result != 0 ) 360 { 361 break; 362 } 363 } 364 365 return result; 366 } 367 } 368 369 /// Read a list of stats 370 unittest 371 { 372 auto data = new Array( 373 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:1 memory:3\n" ~ 374 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:2 memory:4\n".dup); 375 376 auto reader = new StatsLogReader(data); 377 378 /// Iteration without index 379 size_t index; 380 foreach (line; reader) 381 { 382 if (index == 0) 383 { 384 test!("==")(line["cpu_usage"], "1"); 385 test!("==")(line["memory"], "3"); 386 } 387 else 388 { 389 test!("==")(line["cpu_usage"], "2"); 390 test!("==")(line["memory"], "4"); 391 } 392 index++; 393 } 394 395 test!("==")(index, 2); 396 397 /// Iteration with index 398 data = new Array( 399 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:1 memory:3\n" ~ 400 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:2 memory:4\n".dup); 401 reader = new StatsLogReader(data); 402 index = 0; 403 foreach (i, line; reader) 404 { 405 if (i == 0) 406 { 407 test!("==")(line["cpu_usage"], "1"); 408 test!("==")(line["memory"], "3"); 409 } 410 else 411 { 412 test!("==")(line["cpu_usage"], "2"); 413 test!("==")(line["memory"], "4"); 414 } 415 416 index = i; 417 } 418 419 test!("==")(index, 1); 420 } 421 422 /// Get the last line 423 unittest 424 { 425 auto data = new Array( 426 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:1 memory:3\n" ~ 427 "2018-09-12 10:03:07,598 2018-09-12 10:03:07,598 cpu_usage:2 memory:4\n".dup); 428 429 /// It should be able to get the last line with a function call 430 auto reader = new StatsLogReader(data); 431 auto line = reader.last(); 432 433 test!("==")(line["cpu_usage"], "2"); 434 test!("==")(line["memory"], "4"); 435 436 /// It should raise an error for an empty string 437 data = new Array("".dup); 438 reader = new StatsLogReader(data); 439 440 testThrown!(Exception)(reader.last()); 441 }