diff --git a/group_vars/monitoring.yaml b/group_vars/monitoring.yaml new file mode 100644 index 0000000..80bdc83 --- /dev/null +++ b/group_vars/monitoring.yaml @@ -0,0 +1,37 @@ +monitoring_domain: monitoring.bitlair.nl +monitoring_bootstrap_cert: no +acme_domains: + - "{{ monitoring_domain }}" + +prometheus_scrape_configs: + - job_name: "node" + static_configs: + - targets: + - "localhost:9100" + - "bitlair.nl:9100" + - "git.bitlair.nl:9100" + - "mqtt.bitlair.nl:9100" + - "pad.bitlair.nl:9100" + - job_name: "mqtt" + static_configs: + - targets: [ "localhost:9883" ] + - job_name: "blackbox" + scrape_interval: 5m + metrics_path: /probe + params: + module: [http_2xx] # Look for a HTTP 200 response. + static_configs: + - targets: + - https://bitlair.nl + - https://git.bitlair.nl + - https://pad.bitlair.nl + # Legacy + - https://dashboard.bitlair.nl + - https://wiki.bitlair.nl + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: "{{ blackbox_exporter_web_listen_address }}" diff --git a/hosts.yaml b/hosts.yaml index b7b9930..dbd5690 100644 --- a/hosts.yaml +++ b/hosts.yaml @@ -26,6 +26,9 @@ all: mqtt_internal: hosts: mqtt.bitlair.nl: + monitoring: + hosts: + monitoring.bitlair.nl: music: hosts: music.bitlair.nl: diff --git a/monitoring.yaml b/monitoring.yaml new file mode 100644 index 0000000..7ac7e32 --- /dev/null +++ b/monitoring.yaml @@ -0,0 +1,6 @@ +--- +- hosts: monitoring + roles: + - common + - acme + - monitoring diff --git a/roles/monitoring/defaults/main.yaml b/roles/monitoring/defaults/main.yaml new file mode 100644 index 0000000..11facb6 --- /dev/null +++ b/roles/monitoring/defaults/main.yaml @@ -0,0 +1,11 @@ +monitoring_mqtt_exporter_port: 9883 +prometheus_config_dir: /etc/prometheus +prometheus_web_listen_address: "[::1]:9090" +prometheus_storage_retention: 730d +blackbox_exporter_web_listen_address: "[::1]:9115" +grafana_url: "https://{{ monitoring_domain }}" +grafana_domain: "{{ monitoring_domain }}" +grafana_address: "::1" +grafana_port: 9000 +grafana_admin_password: "{{ lookup('password', '/tmp/monitoring_grafana_password length=32') }}" +grafana_data_dir: /var/lib/grafana diff --git a/roles/monitoring/handlers/main.yaml b/roles/monitoring/handlers/main.yaml new file mode 100644 index 0000000..ee9948d --- /dev/null +++ b/roles/monitoring/handlers/main.yaml @@ -0,0 +1,17 @@ +--- +- import_tasks: ../../common/handlers/main.yaml + +- name: restart mqtt_exporter + systemd: + name: mqtt_exporter + state: restarted + +- name: restart prometheus + systemd: + name: prometheus + state: restarted + +- name: restart grafana + systemd: + name: grafana-server + state: restarted diff --git a/roles/monitoring/tasks/blackbox.yaml b/roles/monitoring/tasks/blackbox.yaml new file mode 100644 index 0000000..2385022 --- /dev/null +++ b/roles/monitoring/tasks/blackbox.yaml @@ -0,0 +1,4 @@ +--- +- name: Install blackbox exporter + apt: + name: prometheus-blackbox-exporter diff --git a/roles/monitoring/tasks/grafana.yaml b/roles/monitoring/tasks/grafana.yaml new file mode 100644 index 0000000..b730f17 --- /dev/null +++ b/roles/monitoring/tasks/grafana.yaml @@ -0,0 +1,46 @@ +--- +- name: Add key + get_url: + url: https://apt.grafana.com/gpg.key + dest: /etc/apt/keyrings/grafana.asc + notify: apt update + +- name: Grafana source + copy: + dest: /etc/apt/sources.list.d/grafana.list + content: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" + notify: apt update + +- meta: flush_handlers + +- name: Install Grafana + apt: + name: grafana + +- name: Configure grafana + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: 0644 + notify: restart grafana + with_items: + - { src: grafana.ini, dest: /etc/grafana/grafana.ini } + - { src: grafana-ldap.toml, dest: /etc/grafana/ldap.toml } + +- name: Configure grafana data source + template: + src: grafana-data-source.yml + dest: "/etc/grafana/provisioning/datasources/{{ item.name | lower }}.yaml" + owner: root + group: root + mode: 0644 + notify: restart grafana + with_items: + - name: Prometheus + type: prometheus + access: proxy + url: 'http://{{ prometheus_web_listen_address }}/prometheus' + basicAuth: false + isDefault: true diff --git a/roles/monitoring/tasks/main.yaml b/roles/monitoring/tasks/main.yaml new file mode 100644 index 0000000..f629658 --- /dev/null +++ b/roles/monitoring/tasks/main.yaml @@ -0,0 +1,66 @@ +--- +- name: monitoring + tags: monitoring + block: + - name: Install dependencies + apt: + name: nginx + state: present + + - name: Clear default nginx site + file: + state: absent + path: /etc/nginx/sites-enabled/default + notify: reload nginx + + - name: Install nginx site + template: + src: nginx-site.conf + dest: /etc/nginx/sites-available/monitoring + owner: root + group: root + mode: 0644 + notify: reload nginx + + - name: Enable nginx site + file: + src: /etc/nginx/sites-available/monitoring + dest: /etc/nginx/sites-enabled/monitoring + state: link + notify: reload nginx + + - name: Start nginx + systemd: + name: nginx + state: started + enabled: yes + + - name: Allow HTTP/HTTPS + iptables: + chain: INPUT + protocol: tcp + destination_port: "{{ item.port }}" + ctstate: NEW + jump: ACCEPT + ip_version: "{{ item.ip }}" + action: insert + with_items: + - { ip: ipv6, port: 80 } + - { ip: ipv6, port: 443 } + notify: persist iptables + +- name: mqtt_exporter + tags: mqtt_exporter + import_tasks: mqtt_exporter.yaml + +- name: blackbox + tags: blackbox + import_tasks: blackbox.yaml + +- name: prometheus + tags: prometheus + import_tasks: prometheus.yaml + +- name: grafana + tags: grafana + import_tasks: grafana.yaml diff --git a/roles/monitoring/tasks/mqtt_exporter.yaml b/roles/monitoring/tasks/mqtt_exporter.yaml new file mode 100644 index 0000000..2fee482 --- /dev/null +++ b/roles/monitoring/tasks/mqtt_exporter.yaml @@ -0,0 +1,46 @@ +--- +- name: Clone source + git: + repo: https://github.com/polyfloyd/mqtt-exporter.git + version: main + dest: /opt/mqtt_exporter + accept_hostkey: yes + notify: restart mqtt_exporter + +- name: Install apt dependencies + apt: + name: + - python3-paho-mqtt + - python3-prometheus-client + - python3-yaml + state: present + +- name: Install service + template: + src: mqtt_exporter.service + dest: /etc/systemd/system/mqtt_exporter.service + owner: root + group: root + mode: 0644 + notify: + - daemon reload + - restart mqtt_exporter + +- name: Install config file + template: + src: mqtt_exporter_config.yaml + dest: /etc/mqtt_exporter.yaml + owner: root + group: root + mode: 0644 + notify: + - daemon reload + - restart mqtt_exporter + +- meta: flush_handlers + +- name: Start service + systemd: + name: mqtt_exporter + state: started + enabled: true diff --git a/roles/monitoring/tasks/prometheus.yaml b/roles/monitoring/tasks/prometheus.yaml new file mode 100644 index 0000000..f6c5865 --- /dev/null +++ b/roles/monitoring/tasks/prometheus.yaml @@ -0,0 +1,27 @@ +--- +- name: Install dependencies + apt: + name: prometheus + +- name: Configure Prometheus + template: + src: prometheus.yml + dest: "{{ prometheus_config_dir }}/prometheus.yml" + owner: root + group: root + mode: 0644 + notify: restart prometheus + +- name: Configure Prometheus args + lineinfile: + path: /etc/default/prometheus + line: >- + ARGS=" + --storage.tsdb.retention.time={{ prometheus_storage_retention }} + --storage.tsdb.retention.size=0 + --web.listen-address={{ prometheus_web_listen_address }} + --web.external-url=prometheus + --config.file={{ prometheus_config_dir }}/prometheus.yml + " + regexp: '^ARGS' + notify: restart prometheus diff --git a/roles/monitoring/templates/grafana-data-source.yml b/roles/monitoring/templates/grafana-data-source.yml new file mode 100644 index 0000000..6f9f460 --- /dev/null +++ b/roles/monitoring/templates/grafana-data-source.yml @@ -0,0 +1,4 @@ +apiVersion: 1 +deleteDatasources: [] +datasources: + {{ [item] | to_nice_yaml | indent(2) }} diff --git a/roles/monitoring/templates/grafana-ldap.toml b/roles/monitoring/templates/grafana-ldap.toml new file mode 100644 index 0000000..3e03e8d --- /dev/null +++ b/roles/monitoring/templates/grafana-ldap.toml @@ -0,0 +1,85 @@ +# Managed by Ansible + +# Set to true to log user information returned from LDAP +verbose_logging = true + +[[servers]] +# Ldap server host (specify multiple hosts space separated) +host = "ldap.bitlair.nl" +# Default port is 389 or 636 if use_ssl = true +port = 636 +# Set to true if ldap server supports TLS +use_ssl = true +# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) +start_tls = false +# set to true if you want to skip ssl cert validation +ssl_skip_verify = false +# set to the path to your root CA certificate or leave unset to use system defaults +# root_ca_cert = /path/to/certificate.crt + +# Search user bind dn +bind_dn = "cn=grafana,ou=System,dc=bitlair,dc=nl" +# Search user bind password +bind_password = 'VHfVwAYB6tsEZHuX' + +# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" +search_filter = "(uid=%s)" + +# An array of base dns to search through +search_base_dns = ["dc=bitlair,dc=nl"] + +# In POSIX LDAP schemas, without memberOf attribute a secondary query must be made for groups. +# This is done by enabling group_search_filter below. You must also set member_of= "cn" +# in [servers.attributes] below. + +# Users with nested/recursive group membership and an LDAP server that supports LDAP_MATCHING_RULE_IN_CHAIN +# can set group_search_filter, group_search_filter_user_attribute, group_search_base_dns and member_of +# below in such a way that the user's recursive group membership is considered. +# +# Nested Groups + Active Directory (AD) Example: +# +# AD groups store the Distinguished Names (DNs) of members, so your filter must +# recursively search your groups for the authenticating user's DN. For example: +# +# group_search_filter = "(member:1.2.840.113556.1.4.1941:=%s)" +# group_search_filter_user_attribute = "distinguishedName" +# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] +# +# [servers.attributes] +# ... +# member_of = "distinguishedName" + +## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) +group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +## Group search filter user attribute defines what user attribute gets substituted for %s in group_search_filter. +## Defaults to the value of username in [server.attributes] +## Valid options are any of your values in [servers.attributes] +## If you are using nested groups you probably want to set this and member_of in +## [servers.attributes] to "distinguishedName" +# group_search_filter_user_attribute = "distinguishedName" +## An array of the base DNs to search through for groups. Typically uses ou=groups +group_search_base_dns = ["ou=Groups,dc=bitlair,dc=nl"] + +# Specify names of the ldap attributes your ldap uses +[servers.attributes] +name = "givenName" +surname = "sn" +username = "uid" +email = "mail" +member_of = "cn" + +# Map ldap groups to grafana org roles +[[servers.group_mappings]] +group_dn = "Admins" +org_role = "Admin" +# The Grafana organization database id, optional, if left out the default org (id 1) will be used +# org_id = 1 + +[[servers.group_mappings]] +group_dn = "Members" +org_role = "Editor" + +[[servers.group_mappings]] +# If you want to match all (or no ldap groups) then you can use wildcard +group_dn = "*" +org_role = "Viewer" diff --git a/roles/monitoring/templates/grafana.ini b/roles/monitoring/templates/grafana.ini new file mode 100644 index 0000000..3b2bbd0 --- /dev/null +++ b/roles/monitoring/templates/grafana.ini @@ -0,0 +1,85 @@ +# Managed by Asnible + +app_mode = production +instance_name = monitoring + +# Directories +[paths] +data = /var/lib/grafana +logs = /var/log/grafana +plugins = /var/lib/grafana/plugins + +# HTTP options +[server] +http_addr = {{ grafana_address }} +http_port = {{ grafana_port }} +domain = {{ grafana_domain }} +root_url = {{ grafana_url }} +protocol = http +enforce_domain = False +socket = +cert_key = +cert_file = +enable_gzip = False +static_root_path = public +router_logging = False +serve_from_sub_path = False + +# Database +[database] +type = sqlite3 + +# Remote cache +[remote_cache] + +# Security +[security] +admin_user = admin +admin_password = {{ grafana_admin_password }} + +# Users management and registration +[users] +allow_sign_up = False +auto_assign_org_role = Viewer +default_theme = dark + +[emails] +welcome_email_on_sign_up = False + +# Analytics +[analytics] +reporting_enabled = "True" + +# Dashboards +[dashboards] +versions_to_keep = 20 + +[dashboards.json] +enabled = true +path = /var/lib/grafana/dashboards + +# Alerting +[alerting] +enabled = true +execute_alerts = True + +# SMTP and email config + +# Logging +[log] +mode = console, file +level = info + +# Grafana.com configuration +[grafana_com] +url = https://grafana.com + +[auth.anonymous] +enabled = true +org_name = Bitlair +org_role = Viewer + +[auth.ldap] +enabled = true +config_file = /etc/grafana/ldap.toml +allow_sign_up = true diff --git a/roles/monitoring/templates/mqtt_exporter.service b/roles/monitoring/templates/mqtt_exporter.service new file mode 100644 index 0000000..25b0294 --- /dev/null +++ b/roles/monitoring/templates/mqtt_exporter.service @@ -0,0 +1,14 @@ +[Unit] +Description=Prometheus exporter for MQTT +After=network.target + +[Service] +Type=simple +ExecStart=/opt/mqtt_exporter/mqtt-exporter.py /etc/mqtt_exporter.yaml +Restart=always +RestartSec=10 +DynamicUser=true +AmbientCapabilities=CAP_NET_RAW,CAP_NET_ADMIN+eip + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/templates/mqtt_exporter_config.yaml b/roles/monitoring/templates/mqtt_exporter_config.yaml new file mode 100644 index 0000000..553883d --- /dev/null +++ b/roles/monitoring/templates/mqtt_exporter_config.yaml @@ -0,0 +1,25 @@ +mqtt: + host: {{ mqtt_public_host }} + port: 1883 + +prometheus: + port: {{ monitoring_mqtt_exporter_port }} + +export: + - subscribe: bitlair/# + - subscribe: bitlair/climate/+location/# + - subscribe: bitlair/wifi/+ssid/# + - subscribe: bitlair/state + value_map: + open: 1 + closed: 0 + - subscribe: bitlair/pos/product + metric_type: counter + labels: + product: enum + - subscribe: bitlair/collectd/bitlair-5406/snmp/if_octets-traffic.D15 + metric_name: bitlair_internet_rx + value_regex: "^.+:(.+):" + - subscribe: bitlair/collectd/bitlair-5406/snmp/if_octets-traffic.D15 + metric_name: bitlair_internet_tx + value_regex: "^.+:.+:([\\d\\.]+)" diff --git a/roles/monitoring/templates/nginx-site.conf b/roles/monitoring/templates/nginx-site.conf new file mode 100644 index 0000000..b2f21fc --- /dev/null +++ b/roles/monitoring/templates/nginx-site.conf @@ -0,0 +1,45 @@ +server { + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2; + server_name {{ monitoring_domain }}; + + {% if monitoring_bootstrap_cert %} + include "snippets/snakeoil.conf"; + {% else %} + ssl_certificate "/var/lib/dehydrated/certs/{{ monitoring_domain }}/fullchain.pem"; + ssl_certificate_key "/var/lib/dehydrated/certs/{{ monitoring_domain }}/privkey.pem"; + {% endif %} + + add_header X-Robots-Tag noindex; + + location / { + proxy_pass http://localhost:9000/; + include proxy_params; + } + + location /prometheus/ { + proxy_pass http://localhost:9090/prometheus/; + include proxy_params; + + {% for range in trusted_ranges %} + allow "{{ range.cidr }}"; + {% endfor %} + allow "127.0.0.1"; + allow "::1"; + deny all; + } + + include "snippets/acme.conf"; +} + +server { + listen 80 default_server; + listen [::]:80; + server_name {{ monitoring_domain }}; + + location / { + rewrite ^/(.*) https://$server_name$request_uri? redirect; + } + + include "snippets/acme.conf"; +} diff --git a/roles/monitoring/templates/prometheus.yml b/roles/monitoring/templates/prometheus.yml new file mode 100644 index 0000000..c512949 --- /dev/null +++ b/roles/monitoring/templates/prometheus.yml @@ -0,0 +1,18 @@ +# Managed by Ansible + +global: + evaluation_interval: 1m + scrape_interval: 1m + + external_labels: + environment: monitoring + +rule_files: + - /etc/prometheus/rules/*.rules + +scrape_configs: + - job_name: "prometheus" + metrics_path: "/prometheus/metrics" + static_configs: + - targets: [ "localhost:9090" ] + {{ prometheus_scrape_configs | to_nice_yaml | indent(2) }}