trim PagerDuty message summary to 1024 chars, add PagerDuty debug log (…

…prometheus#1701)

PagerDuty Alerts are rejected (a 400 BadRequest is sent back from PagerDuty)
when the summary field is longer than 1024 characters
(https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2).

Signed-off-by: Stefan Bueringer <sbueringer@gmail.com>

notify/impl.go

@@ -579,6 +579,12 @@ func (n *PagerDuty) notifyV2(
 		n.conf.Severity = "error"
 	}
 
+	summary := tmpl(n.conf.Description)
+	summaryRunes := []rune(summary)
+	if len(summaryRunes) > 1024 {
+		summary = string(summaryRunes[:1018]) + " [...]"
+	}
+
 	msg := &pagerDutyMessage{
 		Client:      tmpl(n.conf.Client),
 		ClientURL:   tmpl(n.conf.ClientURL),
@@ -588,7 +594,7 @@ func (n *PagerDuty) notifyV2(
 		Images:      make([]pagerDutyImage, len(n.conf.Images)),
 		Links:       make([]pagerDutyLink, len(n.conf.Links)),
 		Payload: &pagerDutyPayload{
-			Summary:       tmpl(n.conf.Description),
+			Summary:       summary,
 			Source:        tmpl(n.conf.Client),
 			Severity:      tmpl(n.conf.Severity),
 			CustomDetails: details,
@@ -615,15 +621,24 @@ func (n *PagerDuty) notifyV2(
 
 	var buf bytes.Buffer
 	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
-		return false, err
+		return false, fmt.Errorf("failed to encode PagerDuty v2 message: %v", err)
 	}
 
 	resp, err := post(ctx, c, n.conf.URL.String(), contentTypeJSON, &buf)
 	if err != nil {
-		return true, err
+		return true, fmt.Errorf("failed to post message to PagerDuty: %v", err)
 	}
 	defer resp.Body.Close()
 
+	// See: https://v2.developer.pagerduty.com/docs/events-api-v2#api-response-codes--retry-logic
+	if resp.StatusCode == http.StatusBadRequest {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return false, fmt.Errorf("failed to read error response from PagerDuty (status: %d): %v", resp.StatusCode, err)
+		}
+		level.Debug(n.logger).Log("msg", "Received error response from PagerDuty", "incident", key, "code", resp.StatusCode, "body", string(body))
+	}
+
 	return n.retryV2(resp.StatusCode)
 }