Make config reloader file writes atomic (#962)

* Make config reloader file writes atomic

This addresses an issue found in the Prometheus Operator, which reuses
this reloader sidecar, but which then also has a second sidecar which
may trigger rule-based reloads while the config sidecar is in the middle
of writing out its config (in a non-atomic way):

prometheus-operator/prometheus-operator#2501

I didn't add a test for this because it's hard to catch the original
problem to begin with, but it has happened.

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Explicitly ignore os.Remove() error

Signed-off-by: Julius Volz <julius.volz@gmail.com>

pkg/reloader/reloader.go

@@ -199,9 +199,16 @@ func (r *Reloader) apply(ctx context.Context) error {
 				return errors.Wrap(err, "expand environment variables")
 			}
 
-			if err := ioutil.WriteFile(r.cfgOutputFile, b, 0666); err != nil {
+			tmpFile := r.cfgOutputFile + ".tmp"
+			defer func() {
+				_ = os.Remove(tmpFile)
+			}()
+			if err := ioutil.WriteFile(tmpFile, b, 0666); err != nil {
 				return errors.Wrap(err, "write file")
 			}
+			if err := os.Rename(tmpFile, r.cfgOutputFile); err != nil {
+				return errors.Wrap(err, "rename file")
+			}
 		}
 	}