Continue on error when processing series

Previously, if we encountered an error while trying to update our series
list, we'd return an error, aborting the processing of the entire batch.
This could lead to the list of available metrics being severely out of
date.  Instead, we simply log an error when we fail to process a metric
name, and skip it.
This commit is contained in:
Solly Ross 2017-06-27 19:59:39 -04:00
parent 01755d5acb
commit 823b8051c9

View file

@ -86,13 +86,13 @@ func (r *basicSeriesRegistry) SetSeries(newSeries []prom.Series) error {
} else if namespaceLabel, hasNamespaceLabel := series.Labels["namespace"]; hasNamespaceLabel && namespaceLabel != "" { } else if namespaceLabel, hasNamespaceLabel := series.Labels["namespace"]; hasNamespaceLabel && namespaceLabel != "" {
// TODO: handle metrics describing a namespace // TODO: handle metrics describing a namespace
if err := r.namer.processNamespacedSeries(series, newInfo); err != nil { if err := r.namer.processNamespacedSeries(series, newInfo); err != nil {
// TODO: do we want to log this and continue, or abort? glog.Errorf("Unable to process namespaced series %q: %v", series.Name, err)
return err continue
} }
} else { } else {
if err := r.namer.processRootScopedSeries(series, newInfo); err != nil { if err := r.namer.processRootScopedSeries(series, newInfo); err != nil {
// TODO: do we want to log this and continue, or abort? glog.Errorf("Unable to process root-scoped series %q: %v", series.Name, err)
return err continue
} }
} }
} }
@ -263,6 +263,7 @@ func (n *metricNamer) processContainerSeries(series prom.Series, infos map[provi
// processNamespacedSeries adds the metric info for the given generic namespaced series to // processNamespacedSeries adds the metric info for the given generic namespaced series to
// the map of metric info. // the map of metric info.
func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error { func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error {
// NB: all errors must occur *before* we save the series info
name, metricKind := n.metricNameFromSeries(series) name, metricKind := n.metricNameFromSeries(series)
resources, err := n.groupResourcesFromSeries(series) resources, err := n.groupResourcesFromSeries(series)
if err != nil { if err != nil {
@ -294,6 +295,7 @@ func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[prov
// processesRootScopedSeries adds the metric info for the given generic namespaced series to // processesRootScopedSeries adds the metric info for the given generic namespaced series to
// the map of metric info. // the map of metric info.
func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error { func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error {
// NB: all errors must occur *before* we save the series info
name, metricKind := n.metricNameFromSeries(series) name, metricKind := n.metricNameFromSeries(series)
resources, err := n.groupResourcesFromSeries(series) resources, err := n.groupResourcesFromSeries(series)
if err != nil { if err != nil {