Unify metrics_collection and metrics_daemon into metrics.

Tested new binaries on the target.
Tested incremental build.
Tested arm-generic build.

Review URL: http://codereview.chromium.org/1650006

metrics/Makefile

@@ -0,0 +1,91 @@
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Makefile for metrics utilities -- library, client and daemon
+#
+
+CCONFIG = $(shell $(PKG_CONFIG) --cflags dbus-1 glib-2.0 dbus-glib-1)
+LDCONFIG = $(shell $(PKG_CONFIG) --libs dbus-1 glib-2.0 gthread-2.0 dbus-glib-1)
+
+CFLAGS = -Wall -Werror -I/usr/include -fpic -O2 $(CCONFIG)
+CXXFLAGS = $(CFLAGS) -fno-exceptions
+
+CLIENT = metrics_client
+DAEMON = metrics_daemon
+TESTDAEMON = test_daemon
+LIB = libmetrics.a
+SHAREDLIB = libmetrics.so
+
+CLIENT_OBJS = \
+	metrics_client.o
+LIB_OBJS = \
+	metrics_library.o
+DAEMON_OBJS = \
+	marshal_void__string_boxed.o \
+	metrics_daemon.o \
+	metrics_daemon_main.o
+TESTDAEMON_OBJS = \
+	marshal_void__string_boxed.o \
+	metrics_daemon.o \
+	metrics_daemon_unittest.o
+
+DAEMON_LDFLAGS = $(LDCONFIG) -lrt -lbase -lpthread -lgflags
+TESTDAEMON_LIBS = -lgtest
+
+all: $(LIB) $(SHAREDLIB) $(CLIENT) $(DAEMON) $(TESTDAEMON)
+
+$(CLIENT): $(CLIENT_OBJS) $(SHAREDLIB)
+	$(CXX) $(LDFLAGS) $^ -o $@
+
+$(DAEMON): $(DAEMON_OBJS) $(SHAREDLIB)
+	$(CXX) -o $@ $^ $(DAEMON_LDFLAGS)
+
+$(TESTDAEMON): $(TESTDAEMON_OBJS) $(SHAREDLIB)
+	$(CXX) -o $@ $^ $(DAEMON_LDFLAGS) $(TESTDAEMON_LIBS)
+
+$(LIB): $(LIB_OBJS)
+	ar rcs $@ $^
+
+$(SHAREDLIB): $(LIB_OBJS)
+	$(CXX) $(LDFLAGS) -shared $^ -o $@
+
+%.o: %.cc
+	$(CXX) $(CXXFLAGS) -c $< -o $@
+
+%.o: %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.c: %.list
+	glib-genmarshal --body --prefix=marshal $< > $@
+
+%.h: %.list
+	glib-genmarshal --header --prefix=marshal $< > $@
+
+# dependencies in addition to those defined by the rules
+
+metrics_daemon.o: \
+	marshal_void__string_boxed.h \
+	metrics_daemon.h \
+	network_states.h
+metrics_daemon_unittest.o: \
+	marshal_void__string_boxed.h \
+	metrics_daemon.h \
+	network_states.h
+marshal_void__string_boxed.o: \
+	marshal_void__string_boxed.h
+
+.PRECIOUS: marshal_void__string_boxed.c  # keep around for debugging
+
+install:
+	install $(CLIENT) $(DESTDIR)/usr/bin
+	install $(DAEMON) $(DESTDIR)/usr/bin
+	install $(LIB) $(DESTDIR)/usr/lib
+	install $(SHAREDLIB) $(DESTDIR)/usr/lib
+	install metrics_library.h $(DESTDIR)/usr/include
+	install syslog_parser.sh $(DESTDIR)/usr/bin
+	install omaha_tracker.sh $(DESTDIR)/usr/sbin
+
+clean:
+	rm -f $(CLIENT) $(DAEMON) $(LIB) $(SHAREDLIB) $(TESTDAEMON)
+	rm -f *.o marshal_void__string_boxed.[ch]

metrics/README

@@ -0,0 +1,8 @@
+This packages contains all scripts and programs assoicated with metrics
+collection for both Chrome's User Metrics Server and automated performance
+metrics collection via Autotest. 
+
+The package includes the metrics daemon for Chrome OS.  This program
+runs as a daemon and collects events by polling and listening for
+d-bus signals. It then adds timing (if needed) and sends the events
+to Chrome for transport to the UMA server at Google.

metrics/marshal_void__string_boxed.list

@@ -0,0 +1 @@
+VOID:STRING,BOXED

metrics/metrics_client.cc

@@ -0,0 +1,73 @@
+// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <errno.h>
+#include <sys/file.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <cstdlib>
+#include <iostream>
+
+#include "metrics_library.h"
+
+using namespace std;
+
+// Usage:  metrics_client [-ab] metric_name metric_value
+int main(int argc, char** argv) {
+  bool send_to_autotest = false;
+  bool send_to_chrome = true;
+  int metric_name_index = 1;
+  int metric_value_index = 2;
+  bool print_usage = false;
+
+  if (argc >= 3) {
+    // Parse arguments
+    int flag;
+    while ((flag = getopt(argc, argv, "ab")) != -1) {
+      switch (flag) {
+        case 'a':
+          send_to_autotest = true;
+          send_to_chrome = false;
+          break;
+        case 'b':
+          send_to_chrome = true;
+          send_to_autotest = true;
+          break;
+        default:
+          print_usage = true;
+          break;
+      }
+    }
+    metric_name_index = optind;
+    metric_value_index = optind + 1;
+  } else {
+    print_usage = true;
+  }
+
+  // Metrics value should be the last argument passed
+  if ((metric_value_index + 1) != argc) {
+    print_usage = true;
+  }
+
+  if (print_usage) {
+    cerr << "Usage:  metrics_client [-ab] name value" << endl;
+    cerr << endl;
+    cerr << "  default: send metric to chrome only" << endl;
+    cerr << "  -a: send metric to autotest only" << endl;
+    cerr << "  -b: send metric to both chrome and autotest" << endl;
+    return 1;
+  }
+
+  // Send metrics
+  if (send_to_autotest) {
+    MetricsLibrary::SendToAutotest(argv[metric_name_index],
+                                   argv[metric_value_index]);
+  }
+  if (send_to_chrome) {
+    MetricsLibrary::SendToChrome(argv[metric_name_index],
+                                 argv[metric_value_index]);
+  }
+  return 0;
+}

metrics/metrics_daemon.cc

@@ -0,0 +1,144 @@
+// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "metrics_daemon.h"
+#include "metrics_library.h"
+
+#include <glib-object.h>
+
+extern "C" {
+#include "marshal_void__string_boxed.h"
+}
+
+#include <base/logging.h>
+
+#define SAFE_MESSAGE(e) ((e && e->message) ? e->message : "unknown error")
+
+MetricsDaemon::NetworkState
+MetricsDaemon::network_states_[MetricsDaemon::kNumberNetworkStates] = {
+#define STATE(name, capname) { #name, "Connman" # capname },
+#include "network_states.h"
+};
+
+void MetricsDaemon::Run(bool run_as_daemon, bool testing) {
+  Init(testing);
+  if (!run_as_daemon || daemon(0, 0) == 0) {
+    Loop();
+  }
+}
+
+void MetricsDaemon::Init(bool testing) {
+  testing_ = testing;
+  network_state_id_ = kUnknownNetworkStateId;
+
+  ::g_thread_init(NULL);
+  ::g_type_init();
+  ::dbus_g_thread_init();
+
+  ::GError* error = NULL;
+  ::DBusGConnection* dbc = ::dbus_g_bus_get(DBUS_BUS_SYSTEM, &error);
+  // Note that LOG(FATAL) terminates the process; otherwise we'd have to worry
+  // about leaking |error|.
+  LOG_IF(FATAL, dbc == NULL) <<
+    "cannot connect to dbus: " << SAFE_MESSAGE(error);
+
+  ::DBusGProxy* net_proxy = ::dbus_g_proxy_new_for_name(
+      dbc, "org.moblin.connman", "/", "org.moblin.connman.Metrics");
+  LOG_IF(FATAL, net_proxy == NULL) << "no dbus proxy for network";
+
+#if 0
+  // Unclear how soon one can call dbus_g_type_get_map().  Doing it before the
+  // call to dbus_g_bus_get() results in a (non-fatal) assertion failure.
+  // GetProperties returns a hash table.
+  hashtable_gtype = ::dbus_g_type_get_map("GHashTable", G_TYPE_STRING,
+                                          G_TYPE_VALUE);
+#endif
+
+  dbus_g_object_register_marshaller(marshal_VOID__STRING_BOXED,
+                                    G_TYPE_NONE,
+                                    G_TYPE_STRING,
+                                    G_TYPE_VALUE,
+                                    G_TYPE_INVALID);
+  ::dbus_g_proxy_add_signal(net_proxy, "ConnectionStateChanged",
+                            G_TYPE_STRING, G_TYPE_VALUE, G_TYPE_INVALID);
+  ::dbus_g_proxy_connect_signal(net_proxy, "ConnectionStateChanged",
+                                G_CALLBACK(&StaticNetSignalHandler),
+                                this, NULL);
+}
+
+void MetricsDaemon::Loop() {
+  ::GMainLoop* loop = ::g_main_loop_new(NULL, false);
+  ::g_main_loop_run(loop);
+}
+
+void MetricsDaemon::StaticNetSignalHandler(::DBusGProxy* proxy,
+                                           const char* property,
+                                           const ::GValue* value,
+                                           void *data) {
+  (static_cast<MetricsDaemon*>(data))->NetSignalHandler(proxy, property, value);
+}
+
+void MetricsDaemon::NetSignalHandler(::DBusGProxy* proxy,
+                                     const char* property,
+                                     const ::GValue* value) {
+  if (strcmp("ConnectionState", property) != 0) {
+    return;
+  }
+
+  const char* newstate = static_cast<const char*>(g_value_get_string(value));
+  LogNetworkStateChange(newstate);
+}
+
+void MetricsDaemon::LogNetworkStateChange(const char* newstate) {
+  NetworkStateId new_id = GetNetworkStateId(newstate);
+  if (new_id == kUnknownNetworkStateId) {
+    LOG(WARNING) << "unknown network connection state " << newstate;
+    return;
+  }
+  NetworkStateId old_id = network_state_id_;
+  if (new_id == old_id) {  // valid new state and no change
+    return;
+  }
+  struct timeval now;
+  if (gettimeofday(&now, NULL) != 0) {
+    PLOG(WARNING) << "gettimeofday";
+  }
+  if (old_id != kUnknownNetworkStateId) {
+    struct timeval diff;
+    timersub(&now, &network_state_start_, &diff);
+    int diff_ms = diff.tv_usec / 1000 + diff.tv_sec * 1000;
+    // Saturates rather than overflowing.  We expect this to be statistically
+    // insignificant, since INT_MAX milliseconds is 24.8 days.
+    if (diff.tv_sec >= INT_MAX / 1000) {
+      diff_ms = INT_MAX;
+    }
+    char buffer[100];
+    snprintf(buffer, sizeof(buffer), "%d", diff_ms);
+    if (testing_) {
+      TestPublishMetric(network_states_[old_id].stat_name, buffer);
+    } else {
+      ChromePublishMetric(network_states_[old_id].stat_name, buffer);
+    }
+  }
+  network_state_id_ = new_id;
+  network_state_start_ = now;
+}
+
+MetricsDaemon::NetworkStateId
+MetricsDaemon::GetNetworkStateId(const char* state_name) {
+  for (int i = 0; i < kNumberNetworkStates; i++) {
+    if (strcmp(state_name, network_states_[i].name) == 0) {
+      return static_cast<NetworkStateId>(i);
+    }
+  }
+  return static_cast<NetworkStateId>(-1);
+}
+
+void MetricsDaemon::ChromePublishMetric(const char* name, const char* value) {
+  MetricsLibrary::SendToChrome(name, value);
+}
+
+void MetricsDaemon::TestPublishMetric(const char* name, const char* value) {
+  LOG(INFO) << "received metric: " << name << " " << value;
+}

metrics/metrics_daemon.h

@@ -0,0 +1,88 @@
+// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef METRICS_DAEMON_H_
+#define METRICS_DAEMON_H_
+
+#include <dbus/dbus-glib.h>
+#include <sys/time.h>
+#include <time.h>
+
+class MetricsDaemon {
+
+ public:
+  MetricsDaemon()
+      : network_state_id_(kUnknownNetworkStateId) {
+  }
+  ~MetricsDaemon() {}
+
+  // Does all the work.  If |run_as_daemon| is true, daemonize by forking.  If
+  // |testing| is true, log the stats instead of sending them to Chrome.
+  void Run(bool run_as_daemon, bool testing);
+
+ private:
+  // Shared with Chrome for transport.
+  static const char* kMetricsFilePath;
+  static const int kMetricsMessageMaxLength = 4096;
+
+  // The network states.  See network_states.h.
+  typedef enum {
+    // Initial/unknown network state id.
+    kUnknownNetworkStateId = -1,
+#define STATE(name, capname) kNetworkState ## capname,
+#include "network_states.h"
+    kNumberNetworkStates
+  } NetworkStateId;
+
+  typedef struct {
+    const char* name;
+    const char* stat_name;
+  } NetworkState;
+
+  // Initializes.
+  void Init(bool testing);
+
+  // Creates the event loop and enters it.
+  void Loop();
+
+  // Static callback for network events on DBus.
+  static void StaticNetSignalHandler(::DBusGProxy* proxy, const char* property,
+                                     const ::GValue* value, void* data);
+
+  // Callback for network events on DBus.
+  void NetSignalHandler(::DBusGProxy* proxy, const char* property,
+                        const ::GValue* value);
+
+  // This is called at each network state change.  The new state is identified
+  // by the string @newstate.  As a side effect, this method ships to Chrome
+  // (or prints to stdout when testing) the name and duration of the state
+  // that has ended.
+  void LogNetworkStateChange(const char* newstate);
+
+  // Given a string with the name of a state, returns the id for the state.
+  NetworkStateId GetNetworkStateId(const char* state_name);
+
+  // Sends a stat to Chrome for transport to UMA.
+  void ChromePublishMetric(const char* name, const char* value);
+
+  // Prints a stat for testing.
+  void TestPublishMetric(const char* name, const char* value);
+
+#if 0
+  // Fetches a name-value hash table from DBus.
+  bool GetProperties(::DBusGProxy* proxy, ::GHashTable** table);
+
+  // The type descriptor for a glib hash table.
+  GType hashtable_gtype;
+#endif
+
+  // Array of network states of interest.
+  static NetworkState network_states_[kNumberNetworkStates];
+
+  bool testing_;                           // just testing
+  NetworkStateId network_state_id_;        // id of current state
+  struct timeval network_state_start_;     // when current state was entered
+};
+
+#endif  // METRICS_DAEMON_H_

metrics/metrics_daemon_main.cc

@@ -0,0 +1,16 @@
+// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+#include <gflags/gflags.h>
+
+#include "metrics_daemon.h"
+
+DEFINE_bool(daemon, true, "run as daemon (use -nodaemon for debugging)");
+
+int main(int argc, char** argv) {
+  MetricsDaemon::MetricsDaemon d;
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  d.Run(FLAGS_daemon, false);
+}

metrics/metrics_daemon_unittest.cc

@@ -0,0 +1,10 @@
+// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "metrics_daemon.h"
+
+int main(int argc, char** argv) {
+  MetricsDaemon d;
+  d.Run(false, true);
+}

metrics/metrics_library.cc

@@ -0,0 +1,100 @@
+// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/*
+ * metrics_library.cc
+ *
+ *  Created on: Dec 1, 2009
+ *      Author: sosa
+ */
+
+#include "metrics_library.h"
+
+#include <errno.h>
+#include <sys/file.h>
+#include <string.h>
+#include <stdio.h>
+
+#define READ_WRITE_ALL_FILE_FLAGS \
+  (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
+
+static const char kAutotestPath[] = "/tmp/.chromeos-metrics-autotest";
+static const char kChromePath[] = "/tmp/.chromeos-metrics";
+static const int kBufferSize = 4096;
+
+using namespace std;
+
+// TODO(sosa@chromium.org) - use Chromium logger instead of stderr
+void MetricsLibrary::PrintError(const char *message, const char *file,
+                               int code) {
+  const char *kProgramName = "metrics_library";
+  if (code == 0) {
+    fprintf(stderr, "%s: %s\n", kProgramName, message);
+  } else if (file == NULL) {
+    fprintf(stderr, "%s: ", kProgramName);
+    perror(message);
+  } else {
+    fprintf(stderr, "%s: %s: ", kProgramName, file);
+    perror(message);
+  }
+}
+
+void MetricsLibrary::SendToAutotest(string name, string value) {
+  FILE *autotest_file = fopen(kAutotestPath, "a+");
+  if (autotest_file == NULL) {
+    PrintError("fopen", kAutotestPath, errno);
+    return;
+  }
+
+  fprintf(autotest_file, "%s=%s\n", name.c_str(), value.c_str());
+  fclose(autotest_file);
+}
+
+void MetricsLibrary::SendToChrome(string name, string value) {
+  int chrome_fd = open(kChromePath,
+                       O_WRONLY | O_APPEND | O_CREAT,
+                       READ_WRITE_ALL_FILE_FLAGS);
+  // If we failed to open it, return
+  if (chrome_fd < 0) {
+    PrintError("open", kChromePath, errno);
+    return;
+  }
+
+  // Need to chmod because open flags are anded with umask.
+  if (fchmod(chrome_fd, READ_WRITE_ALL_FILE_FLAGS) < 0) {
+    PrintError("fchmod", kChromePath, errno);
+    close(chrome_fd);
+    return;
+  }
+
+  // Grab an exclusive lock to protect Chrome from truncating underneath us
+  if (flock(chrome_fd, LOCK_EX) < 0) {
+    PrintError("flock", kChromePath, errno);
+    close(chrome_fd);
+    return;
+  }
+
+  // Message format is: LENGTH (binary), NAME, VALUE
+  char message[kBufferSize];
+  char *curr_ptr = message;
+  int32_t message_length =
+      name.length() + value.length() + 2 + sizeof(message_length);
+  if (message_length > static_cast<int32_t>(sizeof(message)))
+    PrintError("name/value too long", NULL, 0);
+
+  // Make sure buffer is blanked
+  memset(message, 0, sizeof(message));
+  memcpy(curr_ptr, &message_length, sizeof(message_length));
+  curr_ptr += sizeof(message_length);
+  strncpy(curr_ptr, name.c_str(), name.length());
+  curr_ptr += name.length() + 1;
+  strncpy(curr_ptr, value.c_str(), value.length());
+  if (write(chrome_fd, message, message_length) != message_length)
+    PrintError("write", kChromePath, errno);
+
+  // Release the file lock and close file
+  if (flock(chrome_fd, LOCK_UN) < 0)
+    PrintError("unlock", kChromePath, errno);
+  close(chrome_fd);
+}

metrics/metrics_library.h

@@ -0,0 +1,33 @@
+// Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/*
+ * metrics_library.h
+ *
+ *  Created on: Dec 1, 2009
+ *      Author: sosa
+ */
+
+#ifndef METRICS_LIBRARY_H_
+#define METRICS_LIBRARY_H_
+
+#include <stdio.h>
+#include <string>
+
+// TODO(sosa@chromium.org): Add testing for send methods
+
+// Library used to send metrics both Autotest and Chrome
+class MetricsLibrary {
+ public:
+  // Sends histogram data to Chrome.
+  static void SendToChrome(std::string name, std::string value);
+  // Sends to Autotest.
+  static void SendToAutotest(std::string name, std::string value);
+
+ private:
+  // Prints message to stderr
+  static void PrintError(const char *message, const char *file, int code);
+};
+
+#endif /* METRICS_LIBRARY_H_ */

metrics/network_states.h

@@ -0,0 +1,34 @@
+// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// A table of network states, to be included when building tabular things.
+//
+// This file is used to construct two things: an enumerated type in
+// metrics_daemon.h, and a table of structures with state names in
+// metrics_daemon.cc.  Including this file ensures that the two tables are
+// always in sync (and saves typing).  I don't know of other ways of achieving
+// the same result in C/C++, but it doesn't mean there isn't one.
+
+// Before you include this file, define STATE to do something useful, or else
+// if will be a no-op.  STATE will be undefined on exit.  Don't worry about
+// collisions for the STATE macro (as long as it's a macro) because the
+// compiler will flag them---in that case, just change the name.  If someone is
+// misguided enough to use STATE for something other than a macro, the error
+// messages will be slightly more complicated.
+
+
+#ifndef STATE
+#define STATE(name, capname)
+#endif
+
+STATE(association, Association)
+STATE(configuration, Configuration)
+STATE(disconnect, Disconnect)
+STATE(failure, Failure)
+STATE(idle, Idle)
+STATE(offline, Offline)
+STATE(online, Online)
+STATE(ready, Ready)
+
+#undef STATE

metrics/syslog_parser.sh

@@ -0,0 +1,69 @@
+#! /bin/sh
+
+# This script parses /var/log/syslog for messages from programs that log
+# uptime and disk stats (number of sectors read).  It then outputs
+# these stats in a format usable by the metrics collector, which forwards
+# them to autotest and UMA.
+
+# To add a new metric add a line below, as PROGRAM_NAME  METRIC_NAME.
+# PROGRAM_NAME is the name of the job whose start time we
+# are interested in.  METRIC_NAME is the prefix we want to use for
+# reporting to UMA and autotest.  The script prepends "Time" and
+# "Sectors" to METRIC_NAME for the two available measurements, uptime
+# and number of sectors read thus far.
+
+# You will need to emit messages similar to the following in order to add a 
+# a metric using this process.  You will need to emit both a start and stop
+# time and the metric reported will be the difference in values
+
+# Nov 15 08:05 localhost PROGRAM_NAME[822]: start METRIC_NAME time 12 sectors 56
+# Nov 15 08:05 localhost PROGRAM_NAME[822]: stop METRIC_NAME time 24 sectors 68
+
+# If you add metrics without a start, it is assumed you are requesting the
+# time differece from system start
+
+# Metrics we are interested in measuring
+METRICS="
+upstart start_x
+"
+
+first=1
+program=""
+
+# Get the metrics for all things
+for m in $METRICS
+do
+  if [ $first -eq 1 ]
+  then
+    first=0
+    program_name=$m 
+  else
+    first=1
+    metrics_name=$m       
+         
+    # Example of line from /var/log/messages:
+    # Nov 15 08:05:42 localhost connmand[822]: start metric time 12 sectors 56
+    # "upstart:" is $5, 1234 is $9, etc.
+    program="${program}/$program_name([[0-9]+]:|:) start $metrics_name/\
+    {
+      metrics_start[\"${metrics_name}Time\"] = \$9;
+      metrics_start[\"${metrics_name}Sectors\"] = \$11;            
+    }"
+    program="${program}/$program_name([[0-9]+]:|:) stop $metrics_name/\
+    { 
+        metrics_stop[\"${metrics_name}Time\"] = \$9;
+        metrics_stop[\"${metrics_name}Sectors\"] = \$11;
+    }"
+  fi      
+done
+
+# Do all the differencing here
+program="${program}\
+END{
+  for (i in metrics_stop) {
+    value_time = metrics_stop[i] - metrics_start[i];
+    print i \"=\" value_time;
+  }
+}"
+
+exec awk "$program" /var/log/syslog