LAB: Notizen zum skalierten Nextcloud-System

Ziel dieses LAB ist es, die Ausfallsicherheit des Nextcloud Systems zu erhöhen, also eine skalierte und somit resistentere Nextcloud Umgebung zu betreiben, die sowohl ein Loadbalancing nutzt, als auch redundante Backendsysteme (MariaDB Master/Slave und Redis-Server Master/Slave) und ein zentrales NFS berücksichtigt.

Aufbau der Umgebung, basierend auf Ubunu 18.04.x LTS

nginx Loadbalancer: Externe/Öffentlich IP und 192.168.2.10
Nextcloud Backend 1: 192.168.2.11
Nextcloud Backend 2: 192.168.2.12
Master MariaDB und Redis-Slave: 192.168.2.20
Redis-Master und Slave MariaDB: 192.168.2.21
NFS Server für Daten: 192.168.2.30

Hinweis: Ich empfehle dringend den SSH Standardport 22 zu ändern – berücksichtigen Sie das bitte in den exemplarisch dargestellten Firewallsettings. Zudem empfehle ich auch den Einsatz von fail2ban und weitere Härtungsmaßnahmen, die nicht Gegenstand dieses LABs sind.

Installationsreihenfolge:

  1. MariaDB Master/Slave
  2. Redis Master/Slave
  3. NFS
  4. Nextcloud-Server
  5. nginx Loadbalancer

1. MariaDB Master/Slave

Server: MariaDB Master- und Redis-Master

sudo -s
apt-get install software-properties-common -y
apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
add-apt-repository 'deb [arch=amd64] http://mirror.lstn.net/mariadb/repo/10.4/ubuntu bionic main'
apt update && apt install mariadb-server -y
ufw allow 3306/tcp
mysql_secure_installation

Server: MariaDB-Master und -Slave

systemctl enable mariadb
service mariadb stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client]
default-character-set = utf8mb4
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
log_error=/var/log/mysql/mysql_error.log
nice = 0
socket = /var/run/mysqld/mysqld.sock
[mysqld]
server-id       = 01
replicate-do-db = nextcloud
basedir = /usr
#bind-address = 127.0.0.1
bind-address  = 192.168.2.20
binlog_format   = mixed
# binlog_format = ROW
bulk_insert_buffer_size = 16M
character-set-server = utf8mb4
collation-server = utf8mb4_general_ci
concurrent_insert = 2
connect_timeout = 5
datadir = /var/lib/mysql
default_storage_engine = InnoDB
expire_logs_days = 2
general_log_file = /var/log/mysql/mysql.log
general_log = 0
innodb_buffer_pool_size = 1024M
innodb_buffer_pool_instances = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 32M
innodb_max_dirty_pages_pct = 90
innodb_file_per_table = 1
innodb_open_files = 400
innodb_io_capacity = 4000
innodb_flush_method = O_DIRECT
key_buffer_size = 128M
lc_messages_dir = /usr/share/mysql
lc_messages = en_US
log_bin   = /var/log/mysql/master-bin
# log_bin = /var/log/mysql/mariadb-bin
log_bin_index   = /var/log/mysql/master-bin.index
# log_bin_index = /var/log/mysql/mariadb-bin.index
log_error = /var/log/mysql/mysql_error.log
log_slow_verbosity = query_plan
log_warnings = 2
long_query_time = 1
max_allowed_packet = 16M
max_binlog_size = 100M
max_connections = 300
max_heap_table_size = 64M
myisam_recover_options = BACKUP
myisam_sort_buffer_size = 512M
port = 3306
pid-file = /var/run/mysqld/mysqld.pid
query_cache_limit = 2M
query_cache_size = 64M
query_cache_type = 1
query_cache_min_res_unit = 2k
read_buffer_size = 2M
read_rnd_buffer_size = 1M
relay_log = /var/log/mysql/master-relay-bin
relay_log_index = /var/log/mysql/master-relay-bin.index
skip-external-locking
skip-name-resolve
slow_query_log_file = /var/log/mysql/mariadb-slow.log
slow-query-log = 1
socket = /var/run/mysqld/mysqld.sock
sort_buffer_size = 4M
table_open_cache = 400
thread_cache_size = 128
tmp_table_size = 64M
tmpdir = /tmp
transaction_isolation = READ-COMMITTED
# unix_socket=OFF
user = mysql
wait_timeout = 600
[mysqldump]
max_allowed_packet = 16M
quick
quote-names
[isamchk]
key_buffer = 16M
service mariadb restart
mysql -e "CREATE USER 'nextcloud'@'localhost' identified by 'nextcloud'; flush privileges;"
mysql -e "CREATE USER 'nextcloud'@'192.168.2.21' identified by 'nextcloud'; flush privileges;"
mysql -e "CREATE USER 'nextcloud'@'192.168.2.20' identified by 'nextcloud'; flush privileges;"
mysql -e "grant replication slave on *.* to 'nextcloud'@'192.168.2.21';"
mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'192.168.2.20'; FLUSH privileges;"
mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'192.168.2.21'; FLUSH privileges;"
mysql -e "grant ALL PRIVILEGES on nextcloud.* to nextcloud@'localhost'; FLUSH privileges;"
mysql -e "SHOW MASTER STATUS\G;"

Server: MariaDB-Slave

systemctl enable mariadb
service mariad stop
cd /etc/mysql/
cp my.cnf my.cnf.bak
vi my.cnf
[client]
default-character-set = utf8mb4
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
log_error=/var/log/mysql/mysql_error.log
nice = 0
socket = /var/run/mysqld/mysqld.sock
[mysqld]
server-id       = 02
replicate-do-db = nextcloud
basedir = /usr
#bind-address = 127.0.0.1
bind-address  = 192.168.2.21
binlog_format   = mixed
# binlog_format = ROW
bulk_insert_buffer_size = 16M
character-set-server = utf8mb4
collation-server = utf8mb4_general_ci
concurrent_insert = 2
connect_timeout = 5
datadir = /var/lib/mysql
default_storage_engine = InnoDB
expire_logs_days = 2
general_log_file = /var/log/mysql/mysql.log
general_log = 0
innodb_buffer_pool_size = 1024M
innodb_buffer_pool_instances = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 32M
innodb_max_dirty_pages_pct = 90
innodb_file_per_table = 1
innodb_open_files = 400
innodb_io_capacity = 4000
innodb_flush_method = O_DIRECT
key_buffer_size = 128M
lc_messages_dir = /usr/share/mysql
lc_messages = en_US
log_bin   = /var/log/mysql/slave-bin
# log_bin = /var/log/mysql/mariadb-bin
log_bin_index   = /var/log/mysql/slave-bin.index
# log_bin_index = /var/log/mysql/mariadb-bin.index
log_error = /var/log/mysql/mysql_error.log
log_slow_verbosity = query_plan
log_warnings = 2
long_query_time = 1
max_allowed_packet = 16M
max_binlog_size = 100M
max_connections = 300
max_heap_table_size = 64M
myisam_recover_options = BACKUP
myisam_sort_buffer_size = 512M
port = 3306
pid-file = /var/run/mysqld/mysqld.pid
query_cache_limit = 2M
query_cache_size = 64M
query_cache_type = 1
query_cache_min_res_unit = 2k
read_buffer_size = 2M
read_rnd_buffer_size = 1M
relay_log = /var/log/mysql/slave-relay-bin
relay_log_index = /var/log/mysql/slave-relay-bin.index
skip-external-locking
skip-name-resolve
slow_query_log_file = /var/log/mysql/mariadb-slow.log
slow-query-log = 1
socket = /var/run/mysqld/mysqld.sock
sort_buffer_size = 4M
table_open_cache = 400
thread_cache_size = 128
tmp_table_size = 64M
tmpdir = /tmp
transaction_isolation = READ-COMMITTED
# unix_socket=OFF
user = mysql
wait_timeout = 600
[mysqldump]
max_allowed_packet = 16M
quick
quote-names
[isamchk]
key_buffer = 16M
service mariadb restart

Server: MariaDB Master

mysql -uroot -p
CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
flush tables with read lock;
show master status;

WICHTIG: diese mysql-Verbindung offen und bestehen lassen und eine weitere SSH Session starten:

mysqldump -uroot -p*SECRET* nextcloud > nextcloud-dump.sql
mysql -e "SHOW MASTER STATUS\G;"
root@DBMASTER:~# mysql -e "SHOW MASTER STATUS\G;"
*************************** 1. row ***************************
            File: master-bin.000001
        Position: 8233
    Binlog_Do_DB:
Binlog_Ignore_DB:
root@DBMASTER:~#

Server: MariaDB Slave

mysql -e "CREATE DATABASE nextcloud CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;"
mysql -uroot -p*SECRET* nextcloud < nextcloud-dump.sql
service mariadb restart
mysql -e "change master 'master01' to master_host='192.168.2.20', master_user='nextcloud', master_password='nextcloud', master_port=3306, master_log_file='master-bin.000001, master_log_pos=8233, master_connect_retry=10, master_use_gtid=slave_pos;"
mysql -e "start slave 'master01';"
mysql -e "show slave 'master01' status\G;"

Prüfen der Werte „Slave_IO_Running: Yes undSlave_SQL_Running: Yes

root@DBSLAVE:~# mysql -e "show slave 'master01' status\G;"
*************************** 1. row ***************************
                Slave_IO_State: Waiting for master to send event
                   Master_Host: 192.168.2.11
                   Master_User: nextcloud
                   Master_Port: 3306
                 Connect_Retry: 10
               Master_Log_File: master-bin.000001
           Read_Master_Log_Pos: 8233
                Relay_Log_File: slave-relay-bin-master01.000002
                 Relay_Log_Pos: 1160
         Relay_Master_Log_File: master-bin.000001
              Slave_IO_Running: Yes
             Slave_SQL_Running: Yes
               Replicate_Do_DB: nextcloud
           Replicate_Ignore_DB:
           [...]

Prüfen des „synchronen“ Repliaktionsstatus:

Server: MariaDB Master

show variables like 'gtid_binlog_pos';
+-----------------+--------+
| Variable_name   | Value  |
+-----------------+--------+
| gtid_binlog_pos | 0-1-17 |
+-----------------+--------+

Server: MariaDB Slave

show variables like 'gtid_slave_pos';
+----------------+--------+
| Variable_name  | Value  |
+----------------+--------+
| gtid_slave_pos | 0-1-17 |
+----------------+--------+

Die Werte vom Master sollten identisch mit denen des Slaves sein.

Server: MariaDB-Master und -Slave

root@DBMASTER:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip

To Action From
-- ------ ----
22/tcp ALLOW IN Anywhere
3306 ALLOW IN 192.168.2.11
3306 ALLOW IN 192.168.2.12
22/tcp (v6) ALLOW IN Anywhere (v6)

2. Redis Master/Slave

Server: Redis-Master und -Slave

sudo -s
apt install redis-server -y
redis-cli ping
PONG

Server: Redis-Master

vi /etc/redis/redis.conf
bind 127.0.0.1 192.168.2.21
requirepass nextcloud
systemctl restart redis-server.service
systemctl enable redis-server.service

Server: Redis-Slave

vi /etc/redis/redis.conf
bind 127.0.0.1 192.168.2.20
slaveof 192.168.2.21 6379
requirepass nextcloud
masterauth nextcloud
systemctl restart redis-server.service
systemctl enable redis-server.service

Server: Redis-Master

root@DBSLAVE:~# redis-cli
127.0.0.1:6379> auth nextcloud
OK
127.0.0.1:6379> set MasterSlave-TEST 'Carsten Rieger IT-Services - Quelle ist der Master'
OK
127.0.0.1:6379> quit
root@DBSLAVE:~#

Server: Redis-Slave

root@DBMASTER:~# redis-cli
127.0.0.1:6379> auth nextcloud
OK
127.0.0.1:6379> get MasterSlave-TEST
"Carsten Rieger IT-Services - Quelle ist der Master"
127.0.0.1:6379> quit
root@DBMASTER:~#

Server: Redis-Master und -Slave

root@DBMASTER:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip

To Action From
-- ------ ----
22/tcp ALLOW IN Anywhere
3306 ALLOW IN 192.168.2.11
3306 ALLOW IN 192.168.2.12
6379 ALLOW IN 192.168.2.11
6379 ALLOW IN 192.168.2.12
16379 ALLOW IN 192.168.2.11
16379 ALLOW IN 192.168.2.12
22/tcp (v6) ALLOW IN Anywhere (v6)

3. NFS Server

sudo -s
apt install nfs-kernel-server -y
mkdir -p /daten /nextcloud
chown -R www-data:www-data /daten /nextcloud
chmod -R 777 /daten /nextcloud
vi /etc/exports
/daten 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/daten 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/nextcloud 192.168.2.11(rw,sync,anonuid=33,anongid=33,no_subtree_check)
/nextcloud 192.168.2.12(rw,sync,anonuid=33,anongid=33,no_subtree_check)
exportfs -a
systemctl enable nfs-kernel-server
systemctl restart nfs-kernel-server
root@NFS:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip

To Action From
-- ------ ----
22/tcp ALLOW IN Anywhere
2049 ALLOW IN 192.168.2.11
2049 ALLOW IN 192.168.2.12
22/tcp (v6) ALLOW IN Anywhere (v6)

4. Nextcloud-Server

Server: Nextcloud Server 1 und 2

sudo -s
apt install nfs-common -y
mkdir /daten -p
chown -R www-data:www-data /daten
vi /etc/fstab
192.168.2.30:/daten /daten nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0
192.168.2.30:/nextcloud /var/www/nextcloud nfs auto,nofail,noatime,nolock,intr,tcp,actimeo=1800 0 0
mount -a

Server: Nextcloud Server 1

vi /etc/nginx/conf.d/nextcloud-server1.conf
server  {
server_name 192.168.2.11;
listen 192.168.2.11:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}

Server: Nextcloud Server 2

vi /etc/nginx/conf.d/nextcloud-server2.conf
server {
server_name 192.168.2.12;
listen 192.168.2.12:80;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
location ^~ / {
client_max_body_size 10G;
location / {
rewrite ^ /index.php;
}
location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
deny all;
}
location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
deny all;
}
location ~ ^\/(?:index|ipcheck|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
set $path_info $fastcgi_path_info;
try_files $fastcgi_script_name =404;
include fastcgi_params;
include /etc/nginx/php_optimization.conf;
fastcgi_param REMOTE_ADDR $http_x_real_ip;
}
location ~ ^\/(?:updater|ocs-provider|ocm-provider)(?:$|\/) {
try_files $uri/ =404;
index index.php;
}
location ~ ^\/.+[^\/]\.(?:css|js|woff2?|svg|gif)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
location ~ ^\/.+[^\/]\.(?:png|html|ttf|ico|jpg|jpeg)$ {
try_files $uri /index.php$request_uri;
access_log off;
}
}
}

Server: Nextcloud Server 1 und 2

sudo -u www-data vi /var/www/nextcloud/ipcheck.php
<?php
header( 'Content-Type: text/plain' );
echo 'NGINX';
echo 'Host: ' . $_SERVER['HTTP_HOST'] . "\n";
echo 'Remote Address: ' . $_SERVER['REMOTE_ADDR'] . "\n";
echo 'X-Forwarded-For: ' . $_SERVER['HTTP_X_FORWARDED_FOR'] . "\n";
echo 'X-Forwarded-Proto: ' . $_SERVER['HTTP_X_FORWARDED_PROTO'] . "\n";
echo 'Server Address: ' . $_SERVER['SERVER_ADDR'] . "\n";
echo 'Server Port: ' . $_SERVER['SERVER_PORT'] . "\n\n";
?>
root@NEXTCLOUDSERVER#:~# curl https://ihre.domain.de/ipcheck.php
Test im Browser:  curl https://ihre.domain.de/ipcheck.php
root@NEXTCLOUDSERVER#:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip

To Action From
-- ------ ----
22/tcp ALLOW IN Anywhere
80/tcp ALLOW IN 192.168.2.10
22/tcp (v6) ALLOW IN Anywhere (v6)

5. nginx Loadbalancer

sudo -s
vi /etc/nginx/conf.d/gateway.conf
upstream Nextcloud-Loadbalancer {
ip_hash; # Für Sticky-Sessions
server 192.168.2.11; # <- Nextcloud Server 1
server 192.168.2.12; # <- Nextcloud Server 2
}
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 80 default_server;
listen [::]:80 default_server;
location ^~ /.well-known/acme-challenge {
proxy_pass http://127.0.0.1:81;
proxy_set_header Host $host;
}
location / {
return 301 https://$server_name$request_uri;
}
}
server {
server_name ihre.domäne.de 192.168.2.10; # External/Public IP und Loadbalancer-IP
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
root /var/www/nextcloud/;
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
include /etc/nginx/proxy.conf;
include /etc/nginx/header.conf;
client_max_body_size 10240M;
location ^~ / {
client_max_body_size 10G;
proxy_connect_timeout 3600;
proxy_send_timeout 3600;
proxy_read_timeout 3600;
send_timeout 3600;
proxy_buffering off;
proxy_request_buffering off;
proxy_max_temp_file_size 10240m;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://Nextcloud-Loadbalancer;
proxy_redirect off;
}
location = /.well-known/carddav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/caldav {
return 301 $scheme://$host/remote.php/dav;
}
location = /.well-known/webfinger {
return 301 $scheme://$host/public.php?service=webfinger;
}
location ~ /(ocm-provider|ocs-provider)/ {
return 301 $scheme://$host/$1/;
}
}
root@LOADBALANCER:~# ufw status verbose
Status: active
Logging: on (medium)
Default: deny (incoming), allow (outgoing), disabled (routed)
New profiles: skip

To Action From
-- ------ ----
22/tcp ALLOW IN Anywhere
80/tcp ALLOW IN Anywhere
443/tcp ALLOW IN Anywhere
22/tcp (v6) ALLOW IN Anywhere (v6)
80/tcp (v6) ALLOW IN Anywhere (v6)
443/tcp (v6) ALLOW IN Anywhere (v6)

© Carsten Rieger IT-Services

Carsten Rieger

Carsten Rieger ist ein angestellter Senior IT-Systemengineer und zudem auch als Kleinunternehmer (Freelancer) aktiv. Er arbeitet seit mehr als 15 Jahren im Linux- und Microsoftumfeld, ist ein Open Source Enthusiast und hoch motiviert, Linux Installationen und Troubleshooting durchzuführen. Dabei arbeitet er vorrangig mit Debian und Ubuntu Linux, Nginx und Apache Webservern, MariaDB/MySQL/PostgreSQL, PHP, Cloud Infrastrukturen (bspw. Nextcloud) und auch vielen anderen Open Source Projekten (bspw. Roundcube). Zudem engagiert er sich ehrenamtlich für die Dr. Michael & Angela Jacobi Stiftung - und das schon seit mehr als 7 Jahren.