Separate traefik and grafana prometheus

roles/metrics/files/grafana.env

@@ -0,0 +1,6 @@
+GF_AUTH_ANONYMOUS_ENABLED=true
+GF_AUTH_BASIC_ENABLED=false
+GF_AUTH_PROXY_ENABLED=false
+GF_USERS_ALLOW_SIGN_UP=false
+GF_INSTALL_PLUGINS=grafana-piechart-panel
+GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s:%(http_port)s/

roles/metrics/files/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,21 @@
+apiVersion: 1
+
+providers:
+  # <string> provider name
+- name: 'default'
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string, required> name of the dashboard folder. Required
+  folder: ''
+  # <string> folder UID. will be automatically generated if not specified
+  folderUid: ''
+  # <string, required> provider type. Required
+  type: file
+  # <bool> disable dashboard deletion
+  disableDeletion: false
+  # <bool> enable dashboard editing
+  editable: true
+  # <int> how often Grafana will scan for changed dashboards
+  updateIntervalSeconds: 10
+  options:
+    path: /etc/grafana/provisioning/dashboards

roles/metrics/files/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <string> basic auth username
+  basicAuthUser:
+  # <string> basic auth password
+  basicAuthPassword:
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault: true
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

roles/metrics/files/prometheus/alert.rules

@@ -0,0 +1,11 @@
+groups:
+- name: traefik
+  rules:
+  - alert: service_down
+    expr: up == 0
+    for: 2m
+    labels:
+      severity: page
+    annotations:
+      summary: "Instance {{ $labels.instance }} down"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes"

roles/metrics/files/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+rule_files:
+  - 'alert.rules'
+
+scrape_configs:
+  - job_name: 'traefik'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['traefik:8080']

roles/metrics/tasks/base.yml

@@ -0,0 +1,28 @@
+---
+
+- name: Check metrics directory exist
+  ansible.builtin.file:
+    path: metrics
+    state: directory
+
+- name: Copy metrics conf
+  ansible.builtin.copy:
+    src: .
+    dest: metrics/
+  register: metrics_copy_files_results
+
+- name: Copy template conf
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "metrics/{{ item.dest }}"
+  loop:
+    - { src: 'docker-compose.yml', dest: 'docker-compose.yml' }
+  register: metrics_copy_templates_results
+
+- name: Update and restart container
+  community.docker.docker_compose:
+    project_src: metrics
+    state: present
+    pull: true
+    restarted: "{{ metrics_copy_files_results.changed or metrics_copy_templates_results.changed }}"
+  become: true

roles/metrics/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+
+- ansible.builtin.import_tasks: base.yml
+  name: base

roles/metrics/templates/docker-compose.yml

@@ -0,0 +1,65 @@
+version: '3'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    container_name: prometheus
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+      - {{ server.work_dir }}/traefik/prometheus:/prometheus
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+    command:
+      - "--web.route-prefix=/"
+      - "--web.external-url=https://prometheus.{{ server.domain }}/"
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+    networks:
+      - metrics
+      - proxy
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.prometheus-secure.entrypoints=https
+      - traefik.http.routers.prometheus-secure.rule=Host(`prometheus.{{ server.domain }}`)
+      - traefik.http.routers.prometheus-secure.middlewares=private-network@file
+      - traefik.http.routers.prometheus-secure.tls=true
+      - traefik.http.routers.prometheus-secure.tls.certresolver=sslResolver
+      - traefik.http.routers.prometheus-secure.service=prometheus
+      - traefik.http.services.prometheus.loadbalancer.server.port=9090
+      - traefik.docker.network=proxy
+
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    container_name: grafana
+    volumes:
+      - {{ server.work_dir }}/traefik/grafana:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - grafana.env
+    depends_on:
+      - prometheus
+    networks:
+      - proxy
+      - metrics
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.grafana-secure.entrypoints=https
+      - traefik.http.routers.grafana-secure.rule=Host(`grafana.{{ server.domain }}`)
+      - traefik.http.routers.grafana-secure.middlewares=private-network@file
+      - traefik.http.routers.grafana-secure.tls=true
+      - traefik.http.routers.grafana-secure.tls.certresolver=sslResolver
+      - traefik.http.routers.grafana-secure.service=grafana
+      - traefik.http.services.grafana.loadbalancer.server.port=3000
+      - traefik.docker.network=proxy
+
+networks:
+  metrics:
+    external: true
+  proxy:
+    external: true