Add script to notify about systemd service failures; add a README

README.md

@@ -0,0 +1,9 @@
+# README
+
+A collection of scripts and systemd services which might be useful for servers.
+
+After you pulled, use `./sync` and pick an action which should be self-explaining.
+
+Services and scripts don't depend on each other but have hints that they can be combined (out-commented line in service files for `systemd_failure_notify@.service` for example).
+
+Use at your own risk.

etc/systemd/system/docker_housekeeping.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run docker_housekeeping
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/system/pkgfile.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run pkgfile
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/system/systemd_failure_notify@.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Systemd Failure via mail for %i
+
+[Service]
+Type=oneshot
+ExecStart=systemd_failure_notify %i

etc/systemd/user/borgwrapper.service

@@ -1,10 +1,11 @@
 [Unit]
 Description=Run borgwrapper
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot
-#ExecStart=borgwrapper /home/myUser/borgwrapper-custom.conf
 ExecStart=borgwrapper
+#ExecStart=borgwrapper /home/myUser/borgwrapper-custom.conf
 #Environment="PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/home/myUser/.customBinaryPathWithBorgSymlinkForVirtualenv"
 #Environment="SSH_AUTH_SOCK=/run/user/myUserId/keyring/ssh"
 

etc/systemd/user/check_updates.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run check_updates
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/disk_space_alert.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run disk_space_alert
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/docker_check.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run docker_check
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/memory_usage_alert.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run memory_usage_alert
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/smartctl_tests_long.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run smartctl_tests_long
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/systemd_check.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Run systemd_check
+#OnFailure=systemd_failure_notify@%n.service
 
 [Service]
 Type=oneshot

etc/systemd/user/systemd_failure_notify@.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Systemd Failure via mail for %i
+
+[Service]
+Type=oneshot
+ExecStart=systemd_failure_notify %i

usr/local/bin/systemd_failure_notify

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# usage
+usage() {
+USAGE=$(cat <<EOF
+Usage: systemd_failure_notify [CONFIG_FILE (absolute path)]
+
+Notifies via mail when a service has failed. You need to place the following in the service which should be checked:
+
+[Unit]
+Description=...
+OnFailure=systemd_failure_notify@%n.service
+...
+
+You should place the <scriptname>'s service into systemd's user and systemd's system folder so it can be used by global and user services.
+
+If no CONFIG_FILE is given, HOME/.systemd_failure_notify.conf is used. This fallback option
+has to exist or the script will exit.
+
+Configuration can be done in any file and any pre-defined variable can be overwritten.
+
+The following are at least required for the script to work:
+- SYSTEMD_FAILURE_NOTIFY_MAIL_ADDRESS="" // mail.rc has to be configured
+
+You can copy this script to '/usr/local/bin' and use create a custom CONFIG_FILE as user. Examples can be found in '/usr/share/doc/<scriptname>'.
+EOF
+)
+	echo "$USAGE";
+}
+
+set -e;
+
+SYSTEMD_FAILURE_NOTFY_SERVICE=$1
+if [ -z "$SYSTEMD_FAILURE_NOTFY_SERVICE" ]; then
+	echo "No service given";
+	echo "";
+	usage;
+	exit 1;
+fi
+
+SYSTEMD_FAILURE_NOTFY_HOSTNAME=$(hostname)
+SYSTEMD_FAILURE_NOTFY_USER=$(whoami)
+SYSTEMD_FAILURE_NOTIFY_MAIL_ADDRESS="";
+
+# check for config file
+source_config() {
+	local config=$1;
+	local configFallback=$2;
+
+	if [[ ! -f "$config" ]]; then
+		if [[ ! -f "$configFallback" ]]; then
+			echo "No config file specified and could not find default in '$configFallback'!";
+			echo "";
+			usage;
+			exit 1;
+		else
+			config=$configFallback;
+		fi
+	fi
+
+	set -a;
+	source "$config";
+	set +a;
+}
+source_config "$2" "$HOME/.systemd_failure_notify.conf"
+
+SUBJECT="[systemd $SYSTEMD_FAILURE_NOTFY_HOSTNAME for $SYSTEMD_FAILURE_NOTFY_USER] $SYSTEMD_FAILURE_NOTFY_SERVICE RUN FAILED"
+MESSAGE="$SYSTEMD_FAILURE_NOTFY_SERVICE run failed"
+echo $MESSAGE|mailx -Ssendwait -s "$SUBJECT" $SYSTEMD_FAILURE_NOTIFY_MAIL_ADDRESS;

usr/share/doc/systemd_failure_notify/.systemd_failure_notify.conf.example

@@ -0,0 +1 @@
+SYSTEMD_FAILURE_NOTIFY_MAIL_ADDRESS="alias@domain.tld"