Skip to content

Traefik

Traefik Doc: https://doc.traefik.io/traefik/routing/providers/docker/

Config acme.json - Setup
  docker network create zabra # will be used by traefik and all containers behind it
  touch acme.json
  chmod 600 acme.json
  touch /var/log/traefik-access.log
  # Create user/password for traefik GUI basic authentification
  # And store the value in variable CREDS=myser:$$xyxxx in .env file
  # Sed command is used to double all $ for escaping
  echo $(htpasswd -nB $USER) | sed -e s/\\$/\\$\\$/g

Create traefik.toml file with Let's Encrypt as certificate resolver.

traefik.toml
  [api]
    dashboard = true
   # insecure = true   ## using https

  [entryPoints]
    [entryPoints.web]
      address = ":80"
      [entryPoints.web.http]
        [entryPoints.web.http.redirections]
          [entryPoints.web.http.redirections.entryPoint]
            to = "websecure"
            scheme = "https"
            permanent = true

    [entryPoints.websecure]
      address = ":443"
        [entryPoints.websecure.http.tls]
          certResolver = "default"

  [accessLog]
    filePath = "/var/log/access.log"
    format = "json"

  [http.middlewares]
    [http.middlewares.test-auth.basicAuth]
      usersFile = "/usersfile"

  [providers]
    [providers.docker]
      watch = true
      exposedByDefault = false
      network = "zabra"
    [providers.file]
      filename =  "/etc/traefik/tls_config.toml"  # For TLS Hardening

  [certificatesResolvers.letsencrypt.acme]
    email = "user@email"
    storage = "acme.json"
    [certificatesResolvers.letsencrypt.acme.httpChallenge]
      # used during the challenge
      entryPoint = "web"

TLS Hardening

Traefik TLS Hardening to not support TLS 1.0, TLS 1.1 and enforce some ciphers.

tls_config.toml
  # https://doc.traefik.io/traefik/https/tls/
  # check results : nmap -Pn --script ssl-enum-ciphers -p 443 doc.enoks.fr  or https://www.ssllabs.com/ssltest/analyze.html
  [tls.options]

    [tls.options.default]
      sniStrict = true
      curvePreferences = ["CurveP521", "CurveP384"]
      minVersion = "VersionTLS12"
      cipherSuites = [
          "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
          "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
          "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
          "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
          "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
          "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
      ]

    [tls.options.mintls13]
      minVersion = "VersionTLS13"

???+ example "docker-compose.yml with the tls config

          # Adapt traefik_url

          version: "3.3"

          networks:
            zabra:
              external: true

          services:
            traefik:
              image: "traefik:v2.9"
              container_name: "traefik"
              hostname: "traefik"
              restart: always
              networks:
                - zabra
              labels:
                # dashboard access without port 8080
                - "traefik.enable=true"
                - "traefik.http.routers.api.rule=Host(`traefik_url`)"
                - "traefik.http.routers.api.service=api@internal"
                # https
                - "traefik.http.routers.api.entrypoints=websecure"
                - "traefik.http.routers.api.tls=true"
                - "traefik.http.routers.api.tls.certresolver=letsencrypt"
                # auth
                - "traefik.http.routers.api.middlewares=auth"
                - "traefik.http.middlewares.auth.basicauth.users=${CREDS}"
                # Enforce HSTS (HTTP Strict Transport Security) & STS Headers for the  UI.
                - "traefik.http.middlewares.servicests.headers.stsseconds=31536000"
                - "traefik.http.middlewares.servicests.headers.stspreload=true"
                - "traefik.http.middlewares.servicests.headers.stsincludesubdomains=true"
                - "traefik.http.middlewares.servicests.headers.isdevelopment=false"
                # Disable STS service for traefik dashboard as it prevents basic HTTP authentification to work
                # Need to check if there is an option to enable the both to work as the same time
                # - "traefik.http.routers.api.middlewares=servicests"
              ports:
                - "80:80"
                - "443:443"
              volumes:
                - /var/run/docker.sock:/var/run/docker.sock:ro
                - ./traefik.toml:/etc/traefik/traefik.toml
                # Mount TLS config
                - ./tls_config.toml:/etc/traefik/tls_config.toml
                - ./acme.json:/acme.json
                - /var/log/traefik-access.log:/var/log/access.log

NextCloud + MariaDB

Config - Setup
  mdkir /opt/application/nextcloud
  mkdir -p /opt/data/mariadb /opt/data/nextcloud/data /opt/data/nextcloud/html

  # After docker-compose up -d #we use different root folder from /Var/www for security reasons
  docker exec -it cloud chown -R www-data /opt/data
docker-compose.yml with Treafik options
---
version: "3.3"

networks:
  zabra:
    external: true

services:
  db:
    image: mariadb:10.8.8
    container_name: mariadb
    restart: always
    networks:
      - zabra
    command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW
    volumes:
      - /opt/data/mariadb:/var/lib/mysql
    environment:
      - MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD}"
      - MYSQL_PASSWORD="${MYSQL_PASSWORD}"
      - MYSQL_DATABASE=nextcloud
      - MYSQL_USER=nextcloud
      # added to avoid error after upgrading to 10.8.8
      # https://mariadb.com/kb/en/incorrect-definition-of-table-mysql-column_stats-after-upgrade-from-10-6-5-/
      - MARIADB_AUTO_UPGRADE=1
  nextcloud:
    image: nextcloud:29.0.4
    container_name: cloud
    hostname: cloud.enoks.fr
    restart: always
    networks:
      - zabra
    links:
      - db
    volumes:
      - /opt/data/nextcloud/html:/var/www/html
      - /opt/data/nextcloud/data:/opt/data:rw
    environment:
      - NEXTCLOUD_DATA_DIR=/opt/data #security rec: dont use a default /var/www
      - NEXTCLOUD_TRUSTED_DOMAINS="cloud.enoks.fr localhost"
      - TRUSTED_PROXIES=traefik
      - OVERWRITEPROTOCOL=https
      - MYSQL_PASSWORD="${MYSQL_PASSWORD}"
      - MYSQL_DATABASE=nextcloud
      - MYSQL_USER=nextcloud
      - MYSQL_HOST=db
    labels:
      # Enable proxy through traefik and https
      - "traefik.enable=true"
      - "traefik.http.routers.nextcloud.rule=Host(`cloud.enoks.fr`)"
      - "traefik.http.routers.nextcloud.entrypoints=websecure"
      - "traefik.http.routers.nextcloud.tls=true"
      - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt"
      - "traefik.http.services.nextcloud.loadbalancer.server.port=80"
      # Additionall security options
      - "traefik.http.middlewares.nextcloudredir.redirectregex.permanent=true"
      - "traefik.http.middlewares.nextcloudredir.redirectregex.regex=https://(.*)/.well-known/(card|cal)dav"
      - "traefik.http.middlewares.nextcloudredir.redirectregex.replacement=https://$$1/remote.php/dav/"
      - "traefik.http.middlewares.nextcloudsts.headers.stsincludesubdomains=false"
      - "traefik.http.middlewares.nextcloudsts.headers.stspreload=true"
      - "traefik.http.middlewares.nextcloudsts.headers.stsseconds=31536000"
      - "traefik.http.middlewares.nextcloudsts.headers.isdevelopment=false"
      - "traefik.http.routers.nextcloud.middlewares=nextcloudredir,nextcloudsts"
old docker-compose.yml with Treafik options
      ---
      version: "3.3"

      networks:
        zabra:
          external: true

      services:
        db:
          image: mariadb:10.5.11
          container_name: mariadb
          restart: always
          networks:
            - zabra
          command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW
          volumes:
            - /opt/data/mariadb:/var/lib/mysql
          environment:
            - MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD}"
            - MYSQL_PASSWORD="${MYSQL_PASSWORD}"
            - MYSQL_DATABASE=nextcloud
            - MYSQL_USER=nextcloud

        nextcloud:
          image: nextcloud:22.2.7
          container_name: cloud
          hostname: nextcloud.example.com
          restart: always
          networks:
            - zabra
          links:
            - db
          volumes:
            - /opt/data/nextcloud/html:/var/www/html
            - /opt/data/nextcloud/data:/opt/data:rw
          environment:
            - NEXTCLOUD_DATA_DIR=/opt/data #security rec: dont use a default /var/www
            - NEXTCLOUD_TRUSTED_DOMAINS="nextcloud.example.com localhost"
            - TRUSTED_PROXIES=traefik
            - OVERWRITEPROTOCOL=https
            - MYSQL_PASSWORD="${MYSQL_PASSWORD}"
            - MYSQL_DATABASE=nextcloud
            - MYSQL_USER=nextcloud
            - MYSQL_HOST=db
          labels:
            # Enable proxy through traefik and https
            - "traefik.enable=true"
            - "traefik.http.routers.nextcloud.rule=Host(`netxcloud.example.com`)"
            - "traefik.http.routers.nextcloud.entrypoints=websecure"
            - "traefik.http.routers.nextcloud.tls=true"
            - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt"
            - "traefik.http.services.nextcloud.loadbalancer.server.port=80"
            # Additionall security options
            - "traefik.http.middlewares.nextcloudredir.redirectregex.permanent=true"
            - "traefik.http.middlewares.nextcloudredir.redirectregex.regex=https://(.*)/.well-known/(card|cal)dav"
            - "traefik.http.middlewares.nextcloudredir.redirectregex.replacement=https://$$1/remote.php/dav/"
            - "traefik.http.middlewares.nextcloudsts.headers.stsincludesubdomains=false"
            - "traefik.http.middlewares.nextcloudsts.headers.stspreload=true"
            - "traefik.http.middlewares.nextcloudsts.headers.stsseconds=31536000"
            - "traefik.http.middlewares.nextcloudsts.headers.isdevelopment=false"
            - "traefik.http.routers.nextcloud.middlewares=nextcloudredir,nextcloudsts"

Operations

Background Jobs configuration

Enable CRON JOBS: https://docs.nextcloud.com/server/21/admin_manual/configuration_server/background_jobs_configuration.html

Need to adapt to a docker case
  # Added in root crontab to run docker command
  #  container name is: cloud
  */5  *  *  *  * docker exec -i -u www-data cloud php -f /var/www/html/cron.php

Upgrade

Nextcloud Upgrade Documentation : https://docs.nextcloud.com/server/latest/admin_manual/maintenance/upgrade.html

To ensure you can upgrade your current version to the target version like 24.0.12,

  • Pull the tag 24.0.12 image.
  • Run a temporary container with the image : docker run --name upgrade-check --rm -dit nextcloud:24.0.12
  • Check the variable $OC_VersionCanBeUpgradedFrom value in the the version.php file : docker exec -it upgrade-check cat version.php, then docker stop upgrade-check
version.php content
<?php
$OC_Version = array(24,0,12,1);
$OC_VersionString = '24.0.12';
$OC_Edition = '';
$OC_Channel = 'stable';
$OC_VersionCanBeUpgradedFrom = array (
  'nextcloud' =>
  array (
    '23.0' => true,
    '24.0' => true,
  ),
  'owncloud' =>
  array (
    '10.5' => true,
  ),
);
$OC_Build = '2023-04-19T16:04:20+00:00 5b79bb15b510f52ab598fa45e6977857a9d4895a';
$vendor = 'nextcloud';

After the upgrade, check the administrator's dashboard, sometimes additional DB tasks are needed.

  • Add missing indices : docker exec -i -u www-data cloud ./occ db:add-missing-indices
  • Convert data type: docker exec -i -u www-data cloud ./occ db:convert-filecache-bigint

php-imagick SVG support

To remove the warning php-imagick SVG support, install libmagickcore-6.q16-6-extra : apt install libmagickcore-6.q16-6-extra

To remove the warning ISO add 'default_phone_region' => 'fr' in config.php

Traefik v2 enable HSTS for Nextcloud Container

At first time, it is not easy to understand how to configure Traefik v2 for NextCloud
to meet the security recommendations : HSTS (HTTP Strict Transport Security) , STS (Strict Transport Security), proxy trusted.

Here is conf to make it fine
# Additionall security options
- "traefik.http.middlewares.nextcloudredir.redirectregex.permanent=true"
- "traefik.http.middlewares.nextcloudredir.redirectregex.regex=https://(.*)/.well-known/(card|cal)dav"
- "traefik.http.middlewares.nextcloudredir.redirectregex.replacement=https://$$1/remote.php/dav/"
- "traefik.http.middlewares.nextcloudsts.headers.stsincludesubdomains=false"
- "traefik.http.middlewares.nextcloudsts.headers.stspreload=true"
- "traefik.http.middlewares.nextcloudsts.headers.stsseconds=31536000"
- "traefik.http.middlewares.nextcloudsts.headers.isdevelopment=false"
- "traefik.http.routers.nextcloud.middlewares=nextcloudredir,nextcloudsts"

Gitlab

docker-compose.yml with Treafik options
    ---
    version: "3.3"

    networks:
      zabra:
        external: true

    services:
      gitlab:
        image: "gitlab/gitlab-ce:15.3.0-ce.0"
        container_name: gitlab
        hostname: "gitlab.enoks.fr"
        restart: always
        networks:
          - zabra
        environment:
          GITLAB_OMNIBUS_CONFIG: |
            external_url "https://gitlab.enoks.fr"
            # Disable some services
            registry['enable'] = false
            prometheus_monitoring['enable'] = false
            grafana['enable'] = false
            # Set ssh external port
            #gitlab_rails['gitlab_shell_ssh_port'] = 2221
            # Adapt nginx conf because we have traefik above
            letsencrypt['enable'] = false # ssl is managed by traefik
            nginx['listen_port'] = 80
            nginx['listen_https'] = false
            nginx['proxy_set_headers'] = {
              "X-Forwarded-Proto" => "https",
              "X-Forwarded-Ssl" => "on"
            }
        labels:
          # Enable proxy through traefik and https
          - "traefik.enable=true"
          - "traefik.http.routers.gitlab.rule=Host(`gitlab.enoks.fr`)"
          - "traefik.http.routers.gitlab.entrypoints=websecure"
          - "traefik.http.routers.gitlab.tls=true"
          - "traefik.http.routers.gitlab.tls.certresolver=letsencrypt"
          # Tell Traefik to use the port 80 to connect to container/service: gitlab
          # because image has more than one exposed port. trafefik take the first by default
          - "traefik.http.services.gitlab.loadbalancer.server.port=80"
        ports:
        - "2221:22"
        volumes:
          # GITLAB_HOME is defined in .env file:
          - "$GITLAB_HOME/config:/etc/gitlab"
          - "$GITLAB_HOME/logs:/var/log/gitlab"
          - "$GITLAB_HOME/data:/var/opt/gitlab"
Setup - Deploy
  ## Create folders to mount them as volumes
  mkdir -p /opt/data/gitlab/config /opt/data/gitlab/data   /opt/data/gitlab/logs
  # create docker-compose.yml in /opt/data/gitlab
  docker-compose up -d
  # Get default password
  cat /opt/data/gitlab/config/initial_root_password

Gitlab runner

  • Install gitlab-runner Service : https://docs.gitlab.com/runner/install/linux-manually.html
  • Register a runner executor : https://docs.gitlab.com/runner/register/
Install
  ## Ubuntu
  sudo curl -L --output /usr/local/bin/gitlab-runner "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64"
  sudo chmod +x /usr/local/bin/gitlab-runner
  #Create a GitLab CI user
  sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash
  #Install and run as service:
  sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner
  sudo gitlab-runner start
Register a runner executor
    # docker executor to build doc.enoks.fr
    gitlab-runner register -h  # to see all options

    sudo gitlab-runner register \
      --non-interactive \
      --url "https://gitlab.example.com/" \
      --registration-token "YOUR_TOKEN" \
      --executor "docker" \
      --docker-image alpine:latest \
      --docker-volumes /opt/application/mk:/mkdocs \
      --description "docker-runner to build mkdocs site" \
      --tag-list "docker,mk" \
      --run-untagged="true" \
      --locked="false" \
      --access-level="not_protected"

    # conf will be saved here: /etc/gitlab-runner/config.toml for next update
    # Add 'pull_policy = "if-not-present" ' to avoid pulling image each time

If you set a Shell runner executor and you got ERROR: Job failed (system failure): prepare environment: exit status 1
You should comment the content of /home/gitlab-runner/.bash_logout

Dynhost

To update the Dynamic DNS entries when the WAN IP changes(Internet Box IP)

mkdir -p /opt/application/dynhost; cd /opt/application/dynhost; echo 127.0.0.1 > current.txt

domains.txt

doc.enoks.fr
xxx.xxxx

update.sh
      #/bin/bash

      #to add in crontab with crontab -e
      #*/30 * * * * /bin/bash /opt/application/dynhost/update.sh >> /var/log/dynhost.txt 2>&1

      printf "\n\n"
      date
      cd /opt/application/dynhost

      #Get credentials : USER & PASSWORD from file .env
      source .env

      # Get current public IP
      ip=$(curl --fail --silent  https://ipinfo.io/ip)
      if [ "$?" != "0" ]
      then
              printf "\n https://ipinfo.io/ip has failed. Trying https://ifconfig.me/"
              ip=$(curl --fail --silent https://ifconfig.me/)
      fi
      printf "\nGot IP as: $ip"
      current_ip=$(cat current.txt)
      printf "\nCurrent IP: $current_ip"

      if [ "$ip" != "$current_ip" ]
      then
        echo "$ip" > current.txt
        printf "\nNeed to update  domains IP as $ip"
        domains=$(cat domains.txt)
        for domain in ${domains[@]}
        do
              printf "\nUpdating domain $domain ..."
              curl --silent  -u $USER:$PASSWORD "http://www.ovh.com/nic/update?system=dyndns&hostname=$domain&myip=$ip"
        done
      else
        printf "\nNothing to do. Ip is the same..."
      fi

Data Backup

rsync
      # r=recursive, l=copy symlink, p=preserve permissions , t=preserve modif times
      # o=preserves owner, g=preserves group , v=verbose, z=compress during transfer
      # -D same as --devices --specials = preserves device and special files
      # --delete-after :receiver deletes after transfer, not during

      # mkdir -p ${BasePath}/opt-data/gitlab ${BasePath}/opt-app

      BasePath="/media/zabra/ExtremeSSD/.zabraHost"

      # Deployment files
      printf "\n\nBackup Application files...\n"
      rsync -rtD -vz --delete-after /opt/application/ ${BasePath}/opt-app/

      printf "\n\nBackup NextCloud + mariaDB...\n"
      rsync -rlptD -H -S -vz --delete-after --exclude=gitlab /opt/data/ ${BasePath}/opt-data/

      printf "\n\nCreate Gitlab Backup...\n"
      rm -rf /opt/data/gitlab/data/backups/*
      docker exec -t gitlab gitlab-backup create
      printf "\n\nBackup Gitlab"
      rsync -rtD -vz --delete-after /opt/data/gitlab/data/backups ${BasePath}/opt-data/gitlab/
      rsync -rtD -vz --delete-after /opt/data/gitlab/config ${BasePath}/opt-data/gitlab/

Troubleshooting

Ubuntu stucked at boot due to files cleanup: /dev/mapper/vgubuntu-root: clean xx/xx*

Issue noticed: 01/30/2022

Various ...
    # reboot the machine and enter in the boot menu
    # choose a recovery mode and enter in admin/root console mode

    # IF internet connection works:
        apt-get update
        apt-get install --reinstall ubuntu-desktop
        # Then reboot a machine
        # if the issue is not fixed, reboot and enter in console mode again

        apt install aptitude
        aptitude update
        aptitude dist-upgrade

    # ELSE
    #  Set Up Internet connection and follow the step above after

        # In current case if does not work.
        # There is no wifi card, server should use ethernet connection

        lshw -C network   # notice the Ethernet Interface is named eno1 and it is NOT ACTIVE

        ifconfig eno1 up
        ifconfig eno1     # interface is now Up but there is only Ipv6 assigned

        dhclient eno1 -v # will display the ipv4 that server should have

        ifconfig eno1    # again ..if ipv4 is still missing, add it manually

        ifconfig eno1 $iv4_to_add
        ifconfig eno1 netmask ...
        ifconfig eno1 broadcast ...

To disable vino (remote desktop) encryption (Non root user) gsettings set org.gnome.Vino require-encryption false