moar features!

README.md

@@ -22,7 +22,7 @@ Example Playbook
         - role: SphericalElephant.borgbackup
           borgbackup_client: True
           borgbackup_client_backup_server: backup01.example.com
-          borgbackup_client_jobs:
+          borgbackup_create_jobs:
             - name: system
               day: "*"
               hour: "0"

defaults/main.yml

@@ -1,52 +1,101 @@
 ---
 
+##
+## borgbackup package/binary installation
+##
 borgbackup_install_from_binary: True
 borgbackup_install_from_repo: False
-
-borgbackup_binary_version: "1.0.9"
+borgbackup_binary_version: "1.1.4"
 borgbackup_binary_platform: "borg-linux64"
 borgbackup_binary_uri: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_binary_version }}/{{ borgbackup_binary_platform }}"
 borgbackup_binary: "/usr/local/bin/borg"
 
+##
+## borgbackup server configuration
+##
+borgbackup_server: False
+borgbackup_server_user: "borgbackup"
+borgbackup_server_group: "borgbackup"
+borgbackup_server_home: "/srv/borgbackup"
+borgbackup_server_pool: "{{ borgbackup_server_home }}/repos"
+# ssh pubkey list of sysadmins that manage the borg server
+borgbackup_server_admins: []
+
+##
+## borgbackup general configuration
+##
+# encryption
+borgbackup_encryption_mode: "none"
+borgbackup_passphrase: "yoursecret"
+
+##
+## borg client configuration
+##
 borgbackup_client: False
+
 # Specify the SSH private key type to use. By default role will generate
 # ED25519 keys if they are supported by the remote host, otherwise RSA keys
 # will be generated.
 borgbackup_client_ssh_key_type: '{{ "ed25519"
                             if ("ssh-ed25519" in borgbackup_register_key_types.stdout_lines)
                             else "rsa" }}'
+
 borgbackup_client_ssh_key_file: "/root/.ssh/id_{{ borgbackup_client_ssh_key_type }}-backup"
 borgbackup_client_ssh_key_comment: 'root@{{ ansible_hostname }} generated by Ansible'
 borgbackup_client_scripts_dir: "/etc/borg"
-borgbackup_encryption_mode: "none"
-borgbackup_passphrase: "yoursecret"
-#borgbackup_client_backup_server: backup01.example.com
-#borgbackup_client_jobs:
+
+# lastlog parsed by supervision
+borgbackup_client_lastlog_dir: "/var/cache/borg"
+
+# backup server IP or FQDN used during ansible installation AND backup operation.
+borgbackup_client_backup_server:
+
+# if defined, IP or FQDN used on backup operation. Usefull in case of LAN-free backup
+borgbackup_client_backup_server_lanfreebackup:
+
+# you have to set at least a "create" job.
+# "prune" and "checks" jobs are optionnal, but you should use it too.
+borgbackup_create_jobs:
+
+# borgbackup jobs examples :
+#borgbackup_create_jobs:
 #  - name: system
-#    options: "--compression zlib,6"
+#    options: "--lock-wait 7200 --compression lzma"
 #    day: "*"
-#    hour: "{{ 6 | random }}"
-#    minute: "{{ 59 | random }}"
+#    hour: 0                                                     # default value = 1
+#    minute: 0                                                   # default value = 0
+#    random_hour: 5                                              # default value : ignore randomization
+#    random_minute: 59                                           # default value : ignore randomization
 #    directories:
 #      - "/etc/"
 #      - "/home"
 #    excludes: []
 
-borgbackup_server: False
-borgbackup_server_user: "borgbackup"
-borgbackup_server_group: "borgbackup"
-borgbackup_server_home: "/srv/borgbackup"
-borgbackup_server_pool: "{{ borgbackup_server_home }}/repos"
-
 borgbackup_prune_enabled: yes
-borgbackup_prune_jobs:
-  - name: system
-    prune_options: "--keep-daily=7 --keep-weekly=4"
-    day: "*"
-    hour: "{{ 4 }}"
-    minute: "{{ 0 }}"
-  - name: scripts
-    prune_options: "--keep-within=10d"
-    day: "*"
-    hour: "{{ 4 }}"
-    minute: "{{ 0 }}"
+#borgbackup_prune_jobs:
+#  - name: system
+#    prune_options: "--lock-wait 7200 --keep-daily=7 --keep-weekly=4 --keep-monthly=12 --keep-yearly=-1"
+#    day: "*"
+#    hour: 12                                                     # default value = 1
+#    minute: 0                                                    # default value = 0
+#    random_hour: 5                                               # default value : ignore randomization
+#    random_minute: 59                                            # default value : ignore randomization
+
+borgbackup_check_enabled: yes
+#borgbackup_check_jobs:
+#  - name: system
+#    check_options: "--lock-wait 28800"
+#    day: 1
+#    hour: 12                                                     # default value = 1
+#    minute: 0                                                    # default value = 0
+#    random_hour: 5                                               # default value : ignore randomization
+#    random_minute: 59                                            # default value : ignore randomization
+#    random_day: 27                                               # default value : ignore randomization
+
+# about random :
+# TL;DR : if 'random_hour' is set, then 'hour' will be set by a random value within range 'hour' up to 'hour+random_hour'
+# purpose is to set indempotent random crontab values. Usefull when many jobs are croned toward a small number of repository servers.
+# random_hour : if defined, a indempotent random value is computed between 0 and the value specified.
+# Then, the crontab hour will be addition of hour and random_hour.
+# a modulo 24 is applied in order to ensure valid hour are specified.
+# random_minute and random_day works the same way.

tasks/borgbackup_client.yml

@@ -1,5 +1,9 @@
 ---
 
+#
+# client installation task
+#
+
 - name: check available SSH key types
   shell: ssh -Q key 2>/dev/null || echo "ssh-rsa"
   register: borgbackup_register_key_types
@@ -49,12 +53,20 @@
     mode: 0750
     state: directory
 
+- name: backup supervision lastlog dir
+  file:
+    path: "{{ borgbackup_client_lastlog_dir }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+
 - name: check if the repositories already exist
-  command: "{{ borgbackup_binary }} list {{ borgbackup_server_user }}@{{ borgbackup_client_backup_server }}:{{ item.name }}"
+  command: "{{ borgbackup_binary }} list {{ borgbackup_server_user }}@{{ borgbackup_client_backup_server_lanfreebackup | default(borgbackup_client_backup_server) }}:{{ item.name }}"
   environment:
     - BORG_RSH: "ssh -o StrictHostKeyChecking=no -i {{ borgbackup_client_ssh_key_file }}"
     - BORG_PASSPHRASE: "{{ borgbackup_passphrase }}"
-  with_items: "{{ borgbackup_client_jobs }}"
+  with_items: "{{ borgbackup_create_jobs }}"
   register: list_repos
   failed_when: False
   changed_when: False
@@ -71,55 +83,114 @@
   when: item.rc == 0
 
 - name: initialize repositories
-  command: "{{ borgbackup_binary }} init --encryption {{ borgbackup_encryption_mode }} {{ borgbackup_server_user }}@{{ borgbackup_client_backup_server }}:{{ item.name }}"
+  command: "{{ borgbackup_binary }} init --encryption {{ borgbackup_encryption_mode }} {{ borgbackup_server_user }}@{{ borgbackup_client_backup_server_lanfreebackup | default(borgbackup_client_backup_server) }}:{{ item.name }}"
   environment:
     - BORG_RSH: "ssh -o StrictHostKeyChecking=no -i {{ borgbackup_client_ssh_key_file }}"
     - BORG_PASSPHRASE: "{{ borgbackup_passphrase }}"
-  with_items: "{{ borgbackup_client_jobs }}"
+  with_items: "{{ borgbackup_create_jobs }}"
   register: borgbackup_initialize_result
   failed_when: (borgbackup_initialize_result.rc != 0) and (borgbackup_initialize_result.stderr != "") and ('already exists' not in borgbackup_initialize_result.stderr)
   when: item.name not in initialized_repos
 
-- name: backup scripts
+- name: generate filename for create / prune / check scripts
+  set_fact:
+    create_suffix_script_filename: "create_{{ borgbackup_client_backup_server.split('.')[0] }}"
+    prune_suffix_script_filename: "prune_{{ borgbackup_client_backup_server.split('.')[0] }}"
+    check_suffix_script_filename: "check_{{ borgbackup_client_backup_server.split('.')[0] }}"
+
+#
+# borg create scripts
+#
+- name: deploy borg create scripts
   template:
-    dest: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}.sh"
-    src: job.sh.j2
+    dest: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ create_suffix_script_filename }}.sh"
+    src: create_job.sh.j2
     owner: root
     group: root
     mode: 0700
-  with_items: "{{ borgbackup_client_jobs }}"
+  with_items: "{{ borgbackup_create_jobs }}"
 
-- name: backup cronjobs
+- name: schedule borg create scripts on cron
   cron:
     name: "borg backup {{ item.name }}"
     user: root
-    job: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}.sh 2>&1 | /usr/bin/logger -t borgbackup"
-    day: "{{ item.day | default('*') }}"
-    hour: "{{ item.hour | default('1') }}"
-    minute: "{{ item.minute | default('0') }}"
+    job: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ create_suffix_script_filename }}.sh 2>&1 | /usr/bin/logger -t borgbackup"
+    day: "{{ (item.day | default(1)) + (item.random_day | random(seed=item.name + check_suffix_script_filename + ansible_host))%28 if item.random_day is defined else item.day | default('*') }}"
+    hour: "{{ (item.hour | default(1)) + (item.random_hour | random(seed=item.name + create_suffix_script_filename + ansible_host))%24 if item.random_hour is defined else item.hour | default(1) }}"
+    minute: "{{ (item.minute | default(0)) + (item.random_minute | random(seed=ansible_host + item.name + create_suffix_script_filename))%60 if item.random_minute is defined else item.minute | default(0) }}"
     state: present
-    cron_file: "borgbackup_backup"
-  with_items: "{{ borgbackup_client_jobs }}"
+    cron_file: "borgbackup_{{ item.name }}_{{ create_suffix_script_filename }}"
+  with_items: "{{ borgbackup_create_jobs }}"
+
+- name: deploy borg create fake logs, when no log yet
+  shell: echo -ne "FAKE LOG\nterminating with success status, rc 0\n" | tee "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ create_suffix_script_filename }}.lastlog"
+  args:
+    chdir: "{{ borgbackup_client_lastlog_dir }}"
+    creates: "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ create_suffix_script_filename }}.lastlog"
+  with_items: "{{ borgbackup_create_jobs }}"
 
-- name: deploy prune scripts
+#
+# borg prune scripts
+#
+- name: deploy borg prune scripts
   template:
-    dest: "{{ borgbackup_client_scripts_dir }}/prune_{{ item.name }}.sh"
+    dest: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ prune_suffix_script_filename }}.sh"
     src: prune_job.sh.j2
     owner: root
     group: root
     mode: 0700
   with_items: "{{ borgbackup_prune_jobs }}"
   when: borgbackup_prune_enabled
 
-- name: schedule prune scripts on cron
+- name: schedule borg prune scripts on cron
   cron:
     name: "borg prune {{ item.name }}"
     user: root
-    job: "{{ borgbackup_client_scripts_dir }}/prune_{{ item.name }}.sh 2>&1 | /usr/bin/logger -t borgbackup"
-    day: "{{ item.day | default('*') }}"
-    hour: "{{ item.hour | default('1') }}"
-    minute: "{{ item.minute | default('0') }}"
+    job: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ prune_suffix_script_filename }}.sh 2>&1 | /usr/bin/logger -t borgbackup"
+    day: "{{ (item.day | default(1)) + (item.random_day | random(seed=item.name + check_suffix_script_filename + ansible_host))%28 if item.random_day is defined else item.day | default('*') }}"
+    hour: "{{ (item.hour | default(1)) + (item.random_hour | random(seed=item.name + prune_suffix_script_filename + ansible_host))%24 if item.random_hour is defined else item.hour | default(2) }}"
+    minute: "{{ (item.minute | default(0)) + (item.random_minute | random(seed=ansible_host + item.name + prune_suffix_script_filename))%60 if item.random_minute is defined else item.minute | default(0) }}"
     state: present
-    cron_file: "borgbackup_prune"
+    cron_file: "borgbackup_{{ item.name }}_{{ prune_suffix_script_filename }}"
   with_items: "{{ borgbackup_prune_jobs }}"
   when: borgbackup_prune_enabled
+
+- name: deploy borg prune fake logs, when no log yet
+  shell: echo -ne "FAKE LOG\nterminating with success status, rc 0\n" | tee "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ prune_suffix_script_filename }}.lastlog"
+  args:
+    chdir: "{{ borgbackup_client_lastlog_dir }}"
+    creates: "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ prune_suffix_script_filename }}.lastlog"
+  with_items: "{{ borgbackup_prune_jobs }}"
+
+#
+# borg check scripts
+#
+- name: deploy borg check scripts
+  template:
+    dest: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ check_suffix_script_filename }}.sh"
+    src: check_job.sh.j2
+    owner: root
+    group: root
+    mode: 0700
+  with_items: "{{ borgbackup_check_jobs }}"
+  when: borgbackup_check_enabled
+
+- name: schedule borg check scripts on cron
+  cron:
+    name: "borg check {{ item.name }}"
+    user: root
+    job: "{{ borgbackup_client_scripts_dir }}/{{ item.name }}_{{ check_suffix_script_filename }}.sh 2>&1 | /usr/bin/logger -t borgbackup"
+    day: "{{ (item.day | default(1)) + (item.random_day | random(seed=item.name + check_suffix_script_filename + ansible_host))%28 if item.random_day is defined else item.day | default(1) }}"
+    hour: "{{ (item.hour | default(1)) + (item.random_hour | random(seed=item.name + check_suffix_script_filename + ansible_host))%24 if item.random_hour is defined else item.hour | default(3) }}"
+    minute: "{{ (item.minute | default(0)) + (item.random_minute | random(seed=ansible_host + item.name + check_suffix_script_filename))%60 if item.random_minute is defined else item.minute | default(0) }}"
+    state: present
+    cron_file: "borgbackup_{{ item.name }}_{{ check_suffix_script_filename }}"
+  with_items: "{{ borgbackup_check_jobs }}"
+  when: borgbackup_check_enabled
+
+- name: deploy borg check fake logs, when no log yet
+  shell: echo -ne "FAKE LOG\nterminating with success status, rc 0\n" | tee "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ check_suffix_script_filename }}.lastlog"
+  args:
+    chdir: "{{ borgbackup_client_lastlog_dir }}"
+    creates: "{{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ check_suffix_script_filename }}.lastlog"
+  with_items: "{{ borgbackup_check_jobs }}"

tasks/borgbackup_server.yml

@@ -1,5 +1,9 @@
 ---
 
+#
+# server installation task
+#
+
 - name: create group for borg
   group:
     name: "{{ borgbackup_server_group }}"
@@ -32,3 +36,9 @@
     group: "{{ borgbackup_server_group }}"
     mode: 0600
     state: touch
+
+- name: add ssh-pubkey admin users to backup server
+  authorized_key:
+    user: "{{ borgbackup_server_user }}"
+    key: "{{ item }}"
+  with_items: "{{ borgbackup_server_admins }}"

tasks/install_binary.yml

@@ -1,4 +1,14 @@
 ---
 
 - name: Download borg binary
-  get_url: url="{{ borgbackup_binary_uri }}" dest="{{ borgbackup_binary }}" mode=755
+  get_url:
+    url: "{{ borgbackup_binary_uri }}"
+    dest: "{{ borgbackup_binary }}-{{ borgbackup_binary_version }}"
+    mode: "755"
+
+- name: Link install
+  file:
+    src: "{{ borgbackup_binary }}-{{ borgbackup_binary_version }}"
+    dest: "{{ borgbackup_binary }}"
+    state: link
+    force: yes

templates/check_job.sh.j2

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+export BORG_RSH="ssh -i {{ borgbackup_client_ssh_key_file }}"
+export BORG_PASSPHRASE="{{ borgbackup_passphrase }}"
+
+REPOSITORY={{ borgbackup_server_user }}@{{ borgbackup_client_backup_server_lanfreebackup | default(borgbackup_client_backup_server) }}:{{ item.name }}
+
+{{ borgbackup_binary }} check --show-rc --verbose \
+  {{ item.check_options }} \
+  $REPOSITORY \
+  2>&1 | tee {{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ check_suffix_script_filename }}.lastlog \
+  | /usr/bin/logger -t borgbackup_check_{{ item.name }}_{{ borgbackup_client_backup_server }}

templates/create_job.sh.j2

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+export BORG_RSH="ssh -i {{ borgbackup_client_ssh_key_file }}"
+export BORG_PASSPHRASE="{{ borgbackup_passphrase }}"
+
+REPOSITORY={{ borgbackup_server_user }}@{{ borgbackup_client_backup_server_lanfreebackup | default(borgbackup_client_backup_server) }}:{{ item.name }}
+
+{{ borgbackup_binary }} create --show-rc --verbose --stats \
+  {{ item.options | default('--compression zlib,6') }} \
+  $REPOSITORY::{{ ansible_hostname }}-$(date +%Y-%m-%d_%H-%M) \
+  {{ item.directories | join(' ') }} \
+  {% for e in item.excludes %}
+  --exclude '{{ e }}' \
+  {% endfor %}
+  2>&1 | tee {{ borgbackup_client_lastlog_dir }}/{{ item.name }}_{{ create_suffix_script_filename }}.lastlog \
+  | /usr/bin/logger -t borgbackup_backup_{{ item.name }}_{{ borgbackup_client_backup_server }}